diff --git a/app/config.py b/app/config.py index 3e38d8521..42902f8e1 100644 --- a/app/config.py +++ b/app/config.py @@ -176,6 +176,9 @@ class Config(object): CELERY_TASK_SERIALIZER = 'json' # on reporting worker, restart workers after each task is executed to help prevent memory leaks CELERYD_MAX_TASKS_PER_CHILD = os.getenv('CELERYD_MAX_TASKS_PER_CHILD') + # we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks + if os.getenv('CELERYD_PREFETCH_MULTIPLIER'): + CELERYD_PREFETCH_MULTIPLIER = os.getenv('CELERYD_PREFETCH_MULTIPLIER') CELERY_IMPORTS = ( 'app.celery.tasks', 'app.celery.scheduled_tasks', diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 918b4b50e..80c991309 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -30,7 +30,12 @@ 'notify-delivery-worker-research': {}, 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'}, 'notify-delivery-worker-periodic': {}, - 'notify-delivery-worker-reporting': {'additional_env_vars': {'CELERYD_MAX_TASKS_PER_CHILD': 1}}, + 'notify-delivery-worker-reporting': { + 'additional_env_vars': { + 'CELERYD_MAX_TASKS_PER_CHILD': 1, + 'CELERYD_PREFETCH_MULTIPLIER': 1, + } + }, 'notify-delivery-worker-priority': {}, 'notify-delivery-worker-letters': {}, 'notify-delivery-worker-retry-tasks': {}, diff --git a/scripts/paas_app_wrapper.sh b/scripts/paas_app_wrapper.sh index d574694f9..9e4c7f3e0 100755 --- a/scripts/paas_app_wrapper.sh +++ b/scripts/paas_app_wrapper.sh @@ -29,7 +29,7 @@ case $NOTIFY_APP_NAME in -Q periodic-tasks 2> /dev/null ;; delivery-worker-reporting) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Ofair \ -Q reporting-tasks 2> /dev/null ;; delivery-worker-priority)