darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-20 23:41:17 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2821 from alphagov/celery-settings-for-reporting

Browse Source 
Reduce concurrency and optimise prefetching behaviour for the celery reporting app
This commit is contained in:
David McDonald
2020-04-28 11:13:21 +01:00
committed by GitHub
parent 466c920e60 a237162106
commit 1d3f9589ea
3 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/config.py
View File

@@ -176,6 +176,9 @@ class Config(object):
CELERY_TASK_SERIALIZER = 'json' CELERY_TASK_SERIALIZER = 'json'
# on reporting worker, restart workers after each task is executed to help prevent memory leaks # on reporting worker, restart workers after each task is executed to help prevent memory leaks
CELERYD_MAX_TASKS_PER_CHILD = os.getenv('CELERYD_MAX_TASKS_PER_CHILD') CELERYD_MAX_TASKS_PER_CHILD = os.getenv('CELERYD_MAX_TASKS_PER_CHILD')
# we can set celeryd_prefetch_multiplier to be 1 for celery apps which handle only long running tasks
if os.getenv('CELERYD_PREFETCH_MULTIPLIER'):
CELERYD_PREFETCH_MULTIPLIER = os.getenv('CELERYD_PREFETCH_MULTIPLIER')
CELERY_IMPORTS = ( CELERY_IMPORTS = (
'app.celery.tasks', 'app.celery.tasks',
'app.celery.scheduled_tasks', 'app.celery.scheduled_tasks',

7
manifest.yml.j2
View File

@@ -30,7 +30,12 @@
'notify-delivery-worker-research': {}, 'notify-delivery-worker-research': {},
'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'}, 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'},
'notify-delivery-worker-periodic': {}, 'notify-delivery-worker-periodic': {},
'notify-delivery-worker-reporting': {'additional_env_vars': {'CELERYD_MAX_TASKS_PER_CHILD': 1}}, 'notify-delivery-worker-reporting': {
'additional_env_vars': {
'CELERYD_MAX_TASKS_PER_CHILD': 1,
'CELERYD_PREFETCH_MULTIPLIER': 1,
}
},
'notify-delivery-worker-priority': {}, 'notify-delivery-worker-priority': {},
'notify-delivery-worker-letters': {}, 'notify-delivery-worker-letters': {},
'notify-delivery-worker-retry-tasks': {}, 'notify-delivery-worker-retry-tasks': {},

2
scripts/paas_app_wrapper.sh
View File

@@ -29,7 +29,7 @@ case $NOTIFY_APP_NAME in
-Q periodic-tasks 2> /dev/null -Q periodic-tasks 2> /dev/null
;; ;;
delivery-worker-reporting) delivery-worker-reporting)
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Ofair \
-Q reporting-tasks 2> /dev/null -Q reporting-tasks 2> /dev/null
;; ;;
delivery-worker-priority) delivery-worker-priority)