Merge pull request #2821 from alphagov/celery-settings-for-reporting

Reduce concurrency and optimise prefetching behaviour for the celery reporting app

app/config.py

@@ -176,6 +176,9 @@ class Config(object):
     CELERY_TASK_SERIALIZER = 'json'
     # on reporting worker, restart workers after each task is executed to help prevent memory leaks
     CELERYD_MAX_TASKS_PER_CHILD = os.getenv('CELERYD_MAX_TASKS_PER_CHILD')
+    # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
+    if os.getenv('CELERYD_PREFETCH_MULTIPLIER'):
+        CELERYD_PREFETCH_MULTIPLIER = os.getenv('CELERYD_PREFETCH_MULTIPLIER')
     CELERY_IMPORTS = (
         'app.celery.tasks',
         'app.celery.scheduled_tasks',

manifest.yml.j2

@@ -30,7 +30,12 @@
   'notify-delivery-worker-research': {},
   'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'},
   'notify-delivery-worker-periodic': {},
-  'notify-delivery-worker-reporting': {'additional_env_vars': {'CELERYD_MAX_TASKS_PER_CHILD': 1}},
+  'notify-delivery-worker-reporting': {
+    'additional_env_vars': {
+      'CELERYD_MAX_TASKS_PER_CHILD': 1,
+      'CELERYD_PREFETCH_MULTIPLIER': 1,
+    }
+  },
   'notify-delivery-worker-priority': {},
   'notify-delivery-worker-letters': {},
   'notify-delivery-worker-retry-tasks': {},

scripts/paas_app_wrapper.sh

@@ -29,7 +29,7 @@ case $NOTIFY_APP_NAME in
     -Q periodic-tasks 2> /dev/null
     ;;
   delivery-worker-reporting)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
+    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Ofair \
     -Q reporting-tasks 2> /dev/null
     ;;
   delivery-worker-priority)