Merge pull request #2486 from alphagov/delivery-worker-split

Change distribution of queues among notify delivery workers

README.md

@@ -139,8 +139,7 @@ All commands and command options have a --help command if you need more informat
 ## To create a new worker app
 
 You need to:
-
-1. Create a new entry for your app in manifest-delivery-base.yml ([example](https://github.com/alphagov/notifications-api/commit/131495125e5dfb181010c8595b11b34ab412fc37#diff-a1885d77ffd0a5cb168590428871cd9e))
+1. Create new entries for your app in `manifest.yml.j2` and `scripts/paas_app_wrapper.sh` ([example](https://github.com/alphagov/notifications-api/pull/2486/commits/6163ca8b45813ff59b3a879f9cfcb28e55863e16))
 1. Update the jenkins deployment job in the notifications-aws repo ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-17e034fe6186f2717b77ba277e0a5828))
 1. Add the new worker's log group to the list of logs groups we get alerts about and we ship them to kibana ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-501ffa3502adce988e810875af546b97))
 1. Optionally add it to the autoscaler ([example](https://github.com/alphagov/notifications-paas-autoscaler/commit/16d4cd0bdc851da2fab9fad1c9130eb94acf3d15))

manifest.yml.j2

@@ -8,12 +8,13 @@
   'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0},
 
   'notify-delivery-celery-beat': {'memory': '128M'},
-  'notify-delivery-worker-database': {},
+  'notify-delivery-worker-jobs': {},
   'notify-delivery-worker-research': {'local_statsd': ['preview', 'staging']},
   'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G', 'local_statsd': ['preview', 'staging']},
   'notify-delivery-worker-periodic': {},
   'notify-delivery-worker-priority': {},
-  'notify-delivery-worker': {},
+  'notify-delivery-worker-letters': {},
+  'notify-delivery-worker-retry-tasks': {},
   'notify-delivery-worker-internal': {},
   'notify-delivery-worker-receipts': {},
   'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'},

scripts/paas_app_wrapper.sh

@@ -4,13 +4,17 @@ case $NOTIFY_APP_NAME in
     unset GUNICORN_CMD_ARGS
     scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
     ;;
-  delivery-worker)
+  delivery-worker-retry-tasks)
     scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
-    -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null
+    -Q retry-tasks 2> /dev/null
     ;;
-  delivery-worker-database)
+  delivery-worker-letters)
     scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
-    -Q database-tasks 2> /dev/null
+    -Q create-letters-pdf-tasks,letter-tasks 2> /dev/null
+    ;;
+  delivery-worker-jobs)
+    scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
+    -Q database-tasks,job-tasks 2> /dev/null
     ;;
   delivery-worker-research)
     scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \