From c108c493e6d4570315d92a9259e715232625d04d Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 31 Aug 2016 11:40:31 +0100 Subject: [PATCH 1/4] Rejigged the queue config. - the two new queues that handle delivery of notifications (db-[type] / send-[type]) Now no longer are processed by the default workers - These workers now to the admin type queues for notify (CSV/Validation codes) etc. - Two new workers are deployed to the AWS environments, one focused on db- tasks and one for send tasks - These will pick up those queues explicitly. --- config.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index 6541b8f65..ee4ac076a 100644 --- a/config.py +++ b/config.py @@ -84,11 +84,7 @@ class Config(object): CELERY_QUEUES = [ Queue('periodic', Exchange('default'), routing_key='periodic'), Queue('sms', Exchange('default'), routing_key='sms'), - Queue('db-sms', Exchange('default'), routing_key='db-sms'), - Queue('send-sms', Exchange('default'), routing_key='send-sms'), Queue('email', Exchange('default'), routing_key='email'), - Queue('db-email', Exchange('default'), routing_key='db-email'), - Queue('send-email', Exchange('default'), routing_key='send-email'), Queue('sms-code', Exchange('default'), routing_key='sms-code'), Queue('email-code', Exchange('default'), routing_key='email-code'), Queue('email-reset-password', Exchange('default'), routing_key='email-reset-password'), @@ -123,6 +119,25 @@ class Development(Config): NOTIFY_ENVIRONMENT = 'development' CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' DEBUG = True + SQLALCHEMY_ECHO = False + CELERY_QUEUES = Config.CELERY_QUEUES + [ + Queue('db-sms', Exchange('default'), routing_key='db-sms'), + Queue('send-sms', Exchange('default'), routing_key='send-sms'), + Queue('db-email', Exchange('default'), routing_key='db-email'), + Queue('send-email', Exchange('default'), routing_key='send-email') + ] + + +class Test(Config): + NOTIFY_ENVIRONMENT = 'test' + CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' + STATSD_PREFIX = "test" + CELERY_QUEUES = Config.CELERY_QUEUES + [ + Queue('db-sms', Exchange('default'), routing_key='db-sms'), + Queue('send-sms', Exchange('default'), routing_key='send-sms'), + Queue('db-email', Exchange('default'), routing_key='db-email'), + Queue('send-email', Exchange('default'), routing_key='send-email') + ] class Preview(Config): @@ -131,12 +146,6 @@ class Preview(Config): STATSD_PREFIX = "preview" -class Test(Development): - NOTIFY_ENVIRONMENT = 'test' - CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' - STATSD_PREFIX = "test" - - class Staging(Config): NOTIFY_ENVIRONMENT = 'staging' CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload' @@ -147,15 +156,14 @@ class Staging(Config): class Live(Config): NOTIFY_ENVIRONMENT = 'live' CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload' - STATSD_ENABLED = True STATSD_PREFIX = os.getenv('STATSD_PREFIX') STATSD_ENABLED = True configs = { - 'development': Development, - 'test': Test, - 'live': Live, - 'staging': Staging, - 'preview': Preview + 'development': Development(), + 'test': Test(), + 'live': Live(), + 'staging': Staging(), + 'preview': Preview() } From ce006bd92d18000c3c30efa883ffc601abaf5edb Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 31 Aug 2016 11:40:47 +0100 Subject: [PATCH 2/4] Start/Stop the new worker scripts on the code deploy actions. --- scripts/aws_start_app.sh | 12 ++++++++++++ scripts/aws_stop_app.sh | 22 ++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scripts/aws_start_app.sh b/scripts/aws_start_app.sh index 7be657a03..47f7103ba 100755 --- a/scripts/aws_start_app.sh +++ b/scripts/aws_start_app.sh @@ -12,6 +12,18 @@ then sudo service notifications-api-celery-worker start fi +if [ -e "/etc/init/notifications-api-celery-worker-sender.conf" ] +then + echo "Starting celery worker" + sudo service notifications-api-celery-worker-sender start +fi + +if [ -e "/etc/init/notifications-api-celery-worker-db.conf" ] +then + echo "Starting celery worker" + sudo service notifications-api-celery-worker-db start +fi + if [ -e "/etc/init/notifications-api-celery-beat.conf" ] then echo "Starting celery beat" diff --git a/scripts/aws_stop_app.sh b/scripts/aws_stop_app.sh index 2f5660ae1..3f9735d4b 100755 --- a/scripts/aws_stop_app.sh +++ b/scripts/aws_stop_app.sh @@ -19,7 +19,7 @@ fi if [ -e "/etc/init/notifications-api-celery-beat.conf" ]; then echo "stopping notifications-api-celery-beat" if sudo service notifications-api-celery-beat stop; then - echo "notifications-api stopped" + echo "notifications-api beat stopped" else error_exit "Could not stop notifications-celery-beat" fi @@ -28,8 +28,26 @@ fi if [ -e "/etc/init/notifications-api-celery-worker.conf" ]; then echo "stopping notifications-api-celery-worker" if sudo service notifications-api-celery-worker stop; then - echo "notifications-api stopped" + echo "notifications-api worker stopped" else error_exit "Could not stop notifications-celery-worker" fi fi + +if [ -e "/etc/init/notifications-api-celery-worker-sender.conf" ]; then + echo "stopping notifications-api-celery-worker-sender" + if sudo service notifications-api-celery-worker-sender stop; then + echo "notifications-api sender worker stopped" + else + error_exit "Could not stop notifications-celery-worker-sender" + fi +fi + +if [ -e "/etc/init/notifications-api-celery-worker-db.conf" ]; then + echo "stopping notifications-api-celery-worker-db" + if sudo service notifications-api-celery-worker-db stop; then + echo "notifications-api db worker stopped" + else + error_exit "Could not stop notifications-celery-worker-db" + fi +fi From 06cb440fd289826bbc2b98ef3b78a41887e8fd25 Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 31 Aug 2016 11:42:05 +0100 Subject: [PATCH 3/4] Not a function call. --- config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index ee4ac076a..fe44862d7 100644 --- a/config.py +++ b/config.py @@ -161,9 +161,9 @@ class Live(Config): configs = { - 'development': Development(), - 'test': Test(), - 'live': Live(), - 'staging': Staging(), - 'preview': Preview() + 'development': Development, + 'test': Test, + 'live': Live, + 'staging': Staging, + 'preview': Preview } From 6858641a3c262b02808e438f76433bab1e68881f Mon Sep 17 00:00:00 2001 From: Martyn Inglis Date: Wed, 31 Aug 2016 12:09:12 +0100 Subject: [PATCH 4/4] Debug mode on for tests. --- config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config.py b/config.py index fe44862d7..d234e8b63 100644 --- a/config.py +++ b/config.py @@ -130,6 +130,7 @@ class Development(Config): class Test(Config): NOTIFY_ENVIRONMENT = 'test' + DEBUG = True CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' STATSD_PREFIX = "test" CELERY_QUEUES = Config.CELERY_QUEUES + [