Move Queuenames in with the celery code, revamp config to allow move to celery 4.x

app/celery/__init__.py

@@ -0,0 +1,27 @@
+
+class QueueNames(object):
+    PERIODIC = 'periodic-tasks'
+    PRIORITY = 'priority-tasks'
+    DATABASE = 'database-tasks'
+    SEND = 'send-tasks'
+    RESEARCH_MODE = 'research-mode-tasks'
+    STATISTICS = 'statistics-tasks'
+    JOBS = 'job-tasks'
+    RETRY = 'retry-tasks'
+    NOTIFY = 'notify-internal-tasks'
+    PROCESS_FTP = 'process-ftp-tasks'
+
+    @staticmethod
+    def all_queues():
+        return [
+            QueueNames.PRIORITY,
+            QueueNames.PERIODIC,
+            QueueNames.DATABASE,
+            QueueNames.SEND,
+            QueueNames.RESEARCH_MODE,
+            QueueNames.STATISTICS,
+            QueueNames.JOBS,
+            QueueNames.RETRY,
+            QueueNames.NOTIFY,
+            QueueNames.PROCESS_FTP
+        ]

app/celery/celery.py

@@ -1,5 +1,6 @@
 from datetime import timedelta
 
+from app.celery import QueueNames
 from celery import Celery
 from celery.schedules import crontab
 from kombu import Queue, Exchange
@@ -8,7 +9,6 @@ from app.config import QueueNames
 
 # BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
 class CeleryConfig(object):
-
     broker_url = 'sqs://'
     broker_transport_options = {
         'region': 'sqs.eu-west-1',
@@ -107,16 +107,11 @@ class CeleryConfig(object):
     }
     task_queues = []
 
-    for queue in QueueNames.all_queues():
-        task_queues.append(
-            Queue(queue, Exchange('default'), routing_key=queue)
-        )
-
 
 class NotifyCelery(Celery):
-
     def init_app(self, app):
         super().__init__(app.import_name, broker=CeleryConfig.broker_url)
+        self.init_queues_if_needed(app.config['NOTIFY_ENVIRONMENT'])
         self.config_from_object(CeleryConfig())
         TaskBase = self.Task
 
@@ -126,4 +121,12 @@ class NotifyCelery(Celery):
             def __call__(self, *args, **kwargs):
                 with app.app_context():
                     return TaskBase.__call__(self, *args, **kwargs)
+
         self.Task = ContextTask
+
+    def init_queues_if_needed(self, environment):
+        if environment in ['development', 'test']:
+            for queue in QueueNames.all_queues():
+                CeleryConfig.task_queues.append(
+                    Queue(queue, Exchange('default'), routing_key=queue)
+                )

app/celery/provider_tasks.py

@@ -3,7 +3,7 @@ from notifications_utils.recipients import InvalidEmailError
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery
-from app.config import QueueNames
+from app.celery import QueueNames
 from app.dao import notifications_dao
 from app.dao.notifications_dao import update_notification_status_by_id
 from app.statsd_decorators import statsd

app/celery/research_mode_tasks.py

@@ -1,7 +1,6 @@
 import json
 
 from flask import current_app
-from app import notify_celery
 from requests import request, RequestException, HTTPError
 
 from app.models import SMS_TYPE

app/celery/scheduled_tasks.py

@@ -28,7 +28,7 @@ from app.models import LETTER_TYPE
 from app.notifications.process_notifications import send_notification_to_queue
 from app.statsd_decorators import statsd
 from app.celery.tasks import process_job
-from app.config import QueueNames
+from app.celery import QueueNames
 
 
 @notify_celery.task(name="remove_csv_files")

app/celery/statistics_tasks.py

@@ -10,7 +10,7 @@ from app.dao.statistics_dao import (
 )
 from app.dao.notifications_dao import get_notification_by_id
 from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED
-from app.config import QueueNames
+from app.celery import QueueNames
 
 
 def create_initial_notification_statistic_tasks(notification):

app/celery/tasks.py

@@ -19,8 +19,8 @@ from app import (
 )
 from app.aws import s3
 from app.celery import provider_tasks
-from app.config import QueueNames
 from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
+from app.celery import QueueNames
 from app.dao.jobs_dao import (
     dao_update_job,
     dao_get_job_by_id,