From 6163ca8b45813ff59b3a879f9cfcb28e55863e16 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Tue, 30 Apr 2019 11:29:28 +0100 Subject: [PATCH 1/3] Change distribution of queues among notify delivery workers This is so that retry-tasks queue, which can have quite a lot of load, has its own worker, and other queues are paired with queues that flow similarly: - letter-tasks with create-letters-pdf-tasks - job-tasks with database-tasks --- manifest.yml.j2 | 5 +++-- scripts/paas_app_wrapper.sh | 12 ++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 0576fec6a..f54e3d044 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -8,12 +8,13 @@ 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0}, 'notify-delivery-celery-beat': {'memory': '128M'}, - 'notify-delivery-worker-database': {}, + 'notify-delivery-worker-database-and-job-tasks': {}, 'notify-delivery-worker-research': {'local_statsd': ['preview', 'staging']}, 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G', 'local_statsd': ['preview', 'staging']}, 'notify-delivery-worker-periodic': {}, 'notify-delivery-worker-priority': {}, - 'notify-delivery-worker': {}, + 'notify-delivery-worker-letters-tasks': {}, + 'notify-delivery-worker-retry-tasks': {}, 'notify-delivery-worker-internal': {}, 'notify-delivery-worker-receipts': {}, 'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'}, diff --git a/scripts/paas_app_wrapper.sh b/scripts/paas_app_wrapper.sh index 9786df1b6..39322ef63 100755 --- a/scripts/paas_app_wrapper.sh +++ b/scripts/paas_app_wrapper.sh @@ -4,13 +4,17 @@ case $NOTIFY_APP_NAME in unset GUNICORN_CMD_ARGS scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application ;; - delivery-worker) + delivery-worker-retry-tasks) scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ - -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null + -Q retry-tasks 2> /dev/null ;; - delivery-worker-database) + delivery-worker-letters-tasks) scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ - -Q database-tasks 2> /dev/null + -Q create-letters-pdf-tasks,letter-tasks 2> /dev/null + ;; + delivery-worker-database-and-job-tasks) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q database-tasks,job-tasks 2> /dev/null ;; delivery-worker-research) scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \ From 39d521cfc0773870c6a69c61fd084bfb33c3481d Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Tue, 30 Apr 2019 12:07:42 +0100 Subject: [PATCH 2/3] Update instructions about creating a new worker app in the README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index e1ef3be57..3688dee86 100644 --- a/README.md +++ b/README.md @@ -139,8 +139,7 @@ All commands and command options have a --help command if you need more informat ## To create a new worker app You need to: - -1. Create a new entry for your app in manifest-delivery-base.yml ([example](https://github.com/alphagov/notifications-api/commit/131495125e5dfb181010c8595b11b34ab412fc37#diff-a1885d77ffd0a5cb168590428871cd9e)) +1. Create new entries for your app in `manifest.yml.j2` and `scripts/paas_app_wrapper.sh` ([example](https://github.com/alphagov/notifications-api/pull/2486/commits/6163ca8b45813ff59b3a879f9cfcb28e55863e16)) 1. Update the jenkins deployment job in the notifications-aws repo ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-17e034fe6186f2717b77ba277e0a5828)) 1. Add the new worker's log group to the list of logs groups we get alerts about and we ship them to kibana ([example](https://github.com/alphagov/notifications-aws/commit/69cf9912bd638bce088d4845e4b0a3b11a2cb74c#diff-501ffa3502adce988e810875af546b97)) 1. Optionally add it to the autoscaler ([example](https://github.com/alphagov/notifications-paas-autoscaler/commit/16d4cd0bdc851da2fab9fad1c9130eb94acf3d15)) From b59bca0fc2cb9e081ba538e8ad3b50ddd2ac4b87 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Wed, 1 May 2019 14:51:43 +0100 Subject: [PATCH 3/3] Rename workers so they are less wordy xd --- manifest.yml.j2 | 4 ++-- scripts/paas_app_wrapper.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/manifest.yml.j2 b/manifest.yml.j2 index f54e3d044..75b6d3c6a 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -8,12 +8,12 @@ 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0}, 'notify-delivery-celery-beat': {'memory': '128M'}, - 'notify-delivery-worker-database-and-job-tasks': {}, + 'notify-delivery-worker-jobs': {}, 'notify-delivery-worker-research': {'local_statsd': ['preview', 'staging']}, 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G', 'local_statsd': ['preview', 'staging']}, 'notify-delivery-worker-periodic': {}, 'notify-delivery-worker-priority': {}, - 'notify-delivery-worker-letters-tasks': {}, + 'notify-delivery-worker-letters': {}, 'notify-delivery-worker-retry-tasks': {}, 'notify-delivery-worker-internal': {}, 'notify-delivery-worker-receipts': {}, diff --git a/scripts/paas_app_wrapper.sh b/scripts/paas_app_wrapper.sh index 39322ef63..730186b69 100755 --- a/scripts/paas_app_wrapper.sh +++ b/scripts/paas_app_wrapper.sh @@ -8,11 +8,11 @@ case $NOTIFY_APP_NAME in scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ -Q retry-tasks 2> /dev/null ;; - delivery-worker-letters-tasks) + delivery-worker-letters) scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ -Q create-letters-pdf-tasks,letter-tasks 2> /dev/null ;; - delivery-worker-database-and-job-tasks) + delivery-worker-jobs) scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ -Q database-tasks,job-tasks 2> /dev/null ;;