Move Queuenames in with the celery code, revamp config to allow move to celery 4.x

app/celery/__init__.py

@@ -0,0 +1,27 @@
+
+class QueueNames(object):
+    PERIODIC = 'periodic-tasks'
+    PRIORITY = 'priority-tasks'
+    DATABASE = 'database-tasks'
+    SEND = 'send-tasks'
+    RESEARCH_MODE = 'research-mode-tasks'
+    STATISTICS = 'statistics-tasks'
+    JOBS = 'job-tasks'
+    RETRY = 'retry-tasks'
+    NOTIFY = 'notify-internal-tasks'
+    PROCESS_FTP = 'process-ftp-tasks'
+
+    @staticmethod
+    def all_queues():
+        return [
+            QueueNames.PRIORITY,
+            QueueNames.PERIODIC,
+            QueueNames.DATABASE,
+            QueueNames.SEND,
+            QueueNames.RESEARCH_MODE,
+            QueueNames.STATISTICS,
+            QueueNames.JOBS,
+            QueueNames.RETRY,
+            QueueNames.NOTIFY,
+            QueueNames.PROCESS_FTP
+        ]

app/celery/celery.py

@@ -1,5 +1,6 @@
 from datetime import timedelta
 
+from app.celery import QueueNames
 from celery import Celery
 from celery.schedules import crontab
 from kombu import Queue, Exchange
@@ -8,7 +9,6 @@ from app.config import QueueNames
 
 # BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
 class CeleryConfig(object):
-
     broker_url = 'sqs://'
     broker_transport_options = {
         'region': 'sqs.eu-west-1',
@@ -107,16 +107,11 @@ class CeleryConfig(object):
     }
     task_queues = []
 
-    for queue in QueueNames.all_queues():
-        task_queues.append(
-            Queue(queue, Exchange('default'), routing_key=queue)
-        )
-
 
 class NotifyCelery(Celery):
-
     def init_app(self, app):
         super().__init__(app.import_name, broker=CeleryConfig.broker_url)
+        self.init_queues_if_needed(app.config['NOTIFY_ENVIRONMENT'])
         self.config_from_object(CeleryConfig())
         TaskBase = self.Task
 
@@ -126,4 +121,12 @@ class NotifyCelery(Celery):
             def __call__(self, *args, **kwargs):
                 with app.app_context():
                     return TaskBase.__call__(self, *args, **kwargs)
+
         self.Task = ContextTask
+
+    def init_queues_if_needed(self, environment):
+        if environment in ['development', 'test']:
+            for queue in QueueNames.all_queues():
+                CeleryConfig.task_queues.append(
+                    Queue(queue, Exchange('default'), routing_key=queue)
+                )

app/celery/provider_tasks.py

@@ -3,7 +3,7 @@ from notifications_utils.recipients import InvalidEmailError
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery
-from app.config import QueueNames
+from app.celery import QueueNames
 from app.dao import notifications_dao
 from app.dao.notifications_dao import update_notification_status_by_id
 from app.statsd_decorators import statsd

app/celery/research_mode_tasks.py

@@ -1,7 +1,6 @@
 import json
 
 from flask import current_app
-from app import notify_celery
 from requests import request, RequestException, HTTPError
 
 from app.models import SMS_TYPE

app/celery/scheduled_tasks.py

@@ -28,7 +28,7 @@ from app.models import LETTER_TYPE
 from app.notifications.process_notifications import send_notification_to_queue
 from app.statsd_decorators import statsd
 from app.celery.tasks import process_job
-from app.config import QueueNames
+from app.celery import QueueNames
 
 
 @notify_celery.task(name="remove_csv_files")

app/celery/statistics_tasks.py

@@ -10,7 +10,7 @@ from app.dao.statistics_dao import (
 )
 from app.dao.notifications_dao import get_notification_by_id
 from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED
-from app.config import QueueNames
+from app.celery import QueueNames
 
 
 def create_initial_notification_statistic_tasks(notification):

app/celery/tasks.py

@@ -19,8 +19,8 @@ from app import (
 )
 from app.aws import s3
 from app.celery import provider_tasks
-from app.config import QueueNames
 from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
+from app.celery import QueueNames
 from app.dao.jobs_dao import (
     dao_update_job,
     dao_get_job_by_id,

app/config.py

@@ -1,10 +1,6 @@
-from datetime import timedelta
 import os
 import json
 
-from celery.schedules import crontab
-from kombu import Exchange, Queue
-
 from app.models import (
     EMAIL_TYPE, SMS_TYPE, LETTER_TYPE,
     KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST
@@ -18,36 +14,6 @@ if os.environ.get('VCAP_SERVICES'):
     extract_cloudfoundry_config()
 
 
-
-class QueueNames(object):
-    PERIODIC = 'periodic-tasks'
-    PRIORITY = 'priority-tasks'
-    DATABASE = 'database-tasks'
-    SEND = 'send-tasks'
-    RESEARCH_MODE = 'research-mode-tasks'
-    STATISTICS = 'statistics-tasks'
-    JOBS = 'job-tasks'
-    RETRY = 'retry-tasks'
-    NOTIFY = 'notify-internal-tasks'
-    PROCESS_FTP = 'process-ftp-tasks'
-
-    @staticmethod
-    def all_queues():
-        return [
-            QueueNames.PRIORITY,
-            QueueNames.PERIODIC,
-            QueueNames.DATABASE,
-            QueueNames.SEND,
-            QueueNames.RESEARCH_MODE,
-            QueueNames.STATISTICS,
-            QueueNames.JOBS,
-            QueueNames.RETRY,
-            QueueNames.NOTIFY,
-            QueueNames.PROCESS_FTP
-        ]
-
-
-
 class Config(object):
     # URL of admin app
     ADMIN_BASE_URL = os.environ['ADMIN_BASE_URL']

app/delivery/rest.py

@@ -1,6 +1,6 @@
 from flask import Blueprint, jsonify
 
-from app.config import QueueNames
+from app.celery import QueueNames
 from app.delivery import send_to_providers
 from app.models import EMAIL_TYPE
 from app.celery import provider_tasks

app/invite/rest.py

@@ -4,7 +4,7 @@ from flask import (
     jsonify,
     current_app)
 
-from app.config import QueueNames
+from app.celery import QueueNames
 from app.dao.invited_user_dao import (
     save_invited_user,
     get_invited_user,

app/job/rest.py

@@ -36,7 +36,7 @@ from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANC
 
 from app.utils import pagination_links
 
-from app.config import QueueNames
+from app.celery import QueueNames
 
 job_blueprint = Blueprint('job', __name__, url_prefix='/service/<uuid:service_id>/job')
 

app/letters/send_letter_jobs.py

@@ -2,7 +2,7 @@ from flask import Blueprint, jsonify
 from flask import request
 
 from app import notify_celery
-from app.config import QueueNames
+from app.celery import QueueNames
 from app.dao.jobs_dao import dao_get_all_letter_jobs
 from app.schemas import job_schema
 from app.v2.errors import register_errors

app/notifications/notifications_letter_callback.py

@@ -13,7 +13,7 @@ from app.celery.tasks import update_letter_notifications_statuses
 from app.v2.errors import register_errors
 from app.notifications.utils import autoconfirm_subscription
 from app.schema_validation import validate
-from app.config import QueueNames
+from app.celery import QueueNames
 
 letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__)
 register_errors(letter_callback_blueprint)

app/notifications/process_notifications.py

@@ -12,7 +12,7 @@ from app import redis_store
 from app.celery import provider_tasks
 from notifications_utils.clients import redis
 
-from app.config import QueueNames
+from app.celery import QueueNames
 from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE, ScheduledNotification
 from app.dao.notifications_dao import (dao_create_notification,
                                        dao_delete_notifications_and_history_by_id,

app/notifications/rest.py

@@ -6,7 +6,7 @@ from flask import (
 )
 
 from app import api_user, authenticated_service
-from app.config import QueueNames
+from app.celery import QueueNames
 from app.dao import (
     templates_dao,
     notifications_dao

app/service/sender.py

@@ -1,6 +1,6 @@
 from flask import current_app
 
-from app.config import QueueNames
+from app.celery import QueueNames
 from app.dao.services_dao import dao_fetch_service_by_id, dao_fetch_active_users_for_service
 from app.dao.templates_dao import dao_get_template_by_id
 from app.models import EMAIL_TYPE, KEY_TYPE_NORMAL

app/user/rest.py

@@ -4,7 +4,7 @@ from datetime import datetime
 
 from flask import (jsonify, request, Blueprint, current_app)
 
-from app.config import QueueNames
+from app.celery import QueueNames
 from app.dao.users_dao import (
     get_user_by_id,
     save_model_user,

app/v2/notifications/post_notifications.py

@@ -1,8 +1,8 @@
 from flask import request, jsonify, current_app
 
 from app import api_user, authenticated_service
-from app.config import QueueNames
+from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY
-from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY, SCHEDULE_NOTIFICATIONS
+from app.celery import QueueNames
 from app.notifications.process_notifications import (
     persist_notification,
     send_notification_to_queue,
@@ -11,20 +11,17 @@ from app.notifications.process_notifications import (
 from app.notifications.validators import (
     validate_and_format_recipient,
     check_rate_limiting,
-    service_has_permission,
     check_service_can_schedule_notification,
     check_service_has_permission,
     validate_template
 )
 from app.schema_validation import validate
-from app.utils import get_public_notify_type_text
 from app.v2.notifications import v2_notification_blueprint
 from app.v2.notifications.notification_schemas import (
     post_sms_request,
     create_post_sms_response_from_notification,
     post_email_request,
     create_post_email_response_from_notification)
-from app.v2.errors import BadRequestError
 
 
 @v2_notification_blueprint.route('/<notification_type>', methods=['POST'])