Add broadcasts worker and queue

This worker will be responsible for handing all broadcasts tasks.

It is based on the internal worker which is currently handling broadcast
tasks.

Concurrency of 2 has been chosen fairly arbitrarily. Gunicorn will be
running 4 worker processes so we will end up with the ability to process
8 tasks per app instance given this.

app/config.py

@@ -33,6 +33,7 @@ class QueueNames(object):
     SANITISE_LETTERS = 'sanitise-letter-tasks'
     SAVE_API_EMAIL = 'save-api-email-tasks'
     SAVE_API_SMS = 'save-api-sms-tasks'
+    BROADCASTS = 'broadcast-tasks'
 
     @staticmethod
     def all_queues():
@@ -52,7 +53,8 @@ class QueueNames(object):
             QueueNames.LETTERS,
             QueueNames.SMS_CALLBACKS,
             QueueNames.SAVE_API_EMAIL,
-            QueueNames.SAVE_API_SMS
+            QueueNames.SAVE_API_SMS,
+            QueueNames.BROADCASTS,
         ]
 
 

manifest.yml.j2

@@ -64,6 +64,7 @@
   'notify-delivery-worker-letters': {'memory': '2G'},
   'notify-delivery-worker-retry-tasks': {},
   'notify-delivery-worker-internal': {},
+  'notify-delivery-worker-broadcasts': {},
   'notify-delivery-worker-receipts': {},
   'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'},
   'notify-delivery-worker-save-api-notifications': {'disk_quota': '2G'},

scripts/paas_app_wrapper.sh

@@ -41,6 +41,10 @@ case $NOTIFY_APP_NAME in
     exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
     -Q notify-internal-tasks 2> /dev/null
     ;;
+  delivery-worker-broadcasts)
+    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \
+    -Q broadcast-tasks 2> /dev/null
+    ;;
   delivery-worker-receipts)
     exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
     -Q ses-callbacks,sms-callbacks 2> /dev/null

tests/app/test_config.py

@@ -60,7 +60,7 @@ def test_load_config_if_cloudfoundry_not_available(reload_config):
 def test_queue_names_all_queues_correct():
     # Need to ensure that all_queues() only returns queue names used in API
     queues = QueueNames.all_queues()
-    assert len(queues) == 16
+    assert len(queues) == 17
     assert set([
         QueueNames.PRIORITY,
         QueueNames.PERIODIC,
@@ -77,5 +77,6 @@ def test_queue_names_all_queues_correct():
         QueueNames.LETTERS,
         QueueNames.SMS_CALLBACKS,
         QueueNames.SAVE_API_EMAIL,
-        QueueNames.SAVE_API_SMS
+        QueueNames.SAVE_API_SMS,
+        QueueNames.BROADCASTS,
     ]) == set(queues)