This massive set of changes uses the new queue names object throughout the app and tests.

Lots of changes, all changing the line of code that puts things into queues, and the code that tests that.

app/celery/provider_tasks.py

@@ -3,6 +3,7 @@ from notifications_utils.recipients import InvalidEmailError
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery
+from app.config import QueueNames
 from app.dao import notifications_dao
 from app.dao.notifications_dao import update_notification_status_by_id
 from app.statsd_decorators import statsd
@@ -46,7 +47,7 @@ def deliver_sms(self, notification_id):
             current_app.logger.exception(
                 "SMS notification delivery for id: {} failed".format(notification_id)
             )
-            self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
+            self.retry(queue=QueueNames.RETRY, countdown=retry_iteration_to_delay(self.request.retries))
         except self.MaxRetriesExceededError:
             current_app.logger.exception(
                 "RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification_id),
@@ -70,7 +71,7 @@ def deliver_email(self, notification_id):
             current_app.logger.exception(
                 "RETRY: Email notification {} failed".format(notification_id)
             )
-            self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
+            self.retry(queue=QueueNames.RETRY, countdown=retry_iteration_to_delay(self.request.retries))
         except self.MaxRetriesExceededError:
             current_app.logger.error(
                 "RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification_id)

app/celery/scheduled_tasks.py

@@ -23,6 +23,7 @@ from app.dao.provider_details_dao import (
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
 from app.statsd_decorators import statsd
 from app.celery.tasks import process_job
+from app.config import QueueNames
 
 
 @notify_celery.task(name="remove_csv_files")
@@ -39,7 +40,7 @@ def remove_csv_files():
 def run_scheduled_jobs():
     try:
         for job in dao_set_scheduled_jobs_to_pending():
-            process_job.apply_async([str(job.id)], queue="process-job")
+            process_job.apply_async([str(job.id)], queue=QueueNames.JOBS)
             current_app.logger.info("Job ID {} added to process job queue".format(job.id))
     except SQLAlchemyError as e:
         current_app.logger.exception("Failed to run scheduled jobs")

app/celery/statistics_tasks.py

@@ -3,7 +3,6 @@ from sqlalchemy.exc import SQLAlchemyError
 from app import notify_celery
 from flask import current_app
 
-from app.models import JobStatistics
 from app.statsd_decorators import statsd
 from app.dao.statistics_dao import (
     create_or_update_job_sending_statistics,
@@ -11,16 +10,17 @@ from app.dao.statistics_dao import (
 )
 from app.dao.notifications_dao import get_notification_by_id
 from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED
+from app.config import QueueNames
 
 
 def create_initial_notification_statistic_tasks(notification):
     if notification.job_id and notification.status:
-        record_initial_job_statistics.apply_async((str(notification.id),), queue="statistics")
+        record_initial_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS)
 
 
 def create_outcome_notification_statistic_tasks(notification):
     if notification.job_id and notification.status in NOTIFICATION_STATUS_TYPES_COMPLETED:
-        record_outcome_job_statistics.apply_async((str(notification.id),), queue="statistics")
+        record_outcome_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS)
 
 
 @notify_celery.task(bind=True, name='record_initial_job_statistics', max_retries=20, default_retry_delay=10)
@@ -35,7 +35,7 @@ def record_initial_job_statistics(self, notification_id):
             raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id))
     except SQLAlchemyError as e:
         current_app.logger.exception(e)
-        self.retry(queue="retry")
+        self.retry(queue=QueueNames.RETRY)
     except self.MaxRetriesExceededError:
         current_app.logger.error(
             "RETRY FAILED: task record_initial_job_statistics failed for notification {}".format(
@@ -53,12 +53,12 @@ def record_outcome_job_statistics(self, notification_id):
         if notification:
             updated_count = update_job_stats_outcome_count(notification)
             if updated_count == 0:
-                self.retry(queue="retry")
+                self.retry(queue=QueueNames.RETRY)
         else:
             raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id))
     except SQLAlchemyError as e:
         current_app.logger.exception(e)
-        self.retry(queue="retry")
+        self.retry(queue=QueueNames.RETRY)
     except self.MaxRetriesExceededError:
         current_app.logger.error(
             "RETRY FAILED: task update_job_stats_outcome_count failed for notification {}".format(

app/celery/tasks.py

@@ -16,6 +16,7 @@ from app import (
 )
 from app.aws import s3
 from app.celery import provider_tasks
+from app.config import QueueNames
 from app.dao.jobs_dao import (
     dao_update_job,
     dao_get_job_by_id,
@@ -80,7 +81,7 @@ def process_job(job_id):
         process_row(row_number, recipient, personalisation, template, job, service)
 
     if template.template_type == LETTER_TYPE:
-        build_dvla_file.apply_async([str(job.id)], queue='process-job')
+        build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)
         # temporary logging
         current_app.logger.info("send job {} to build-dvla-file in the process-job queue".format(job_id))
     else:
@@ -112,12 +113,6 @@ def process_row(row_number, recipient, personalisation, template, job, service):
         LETTER_TYPE: persist_letter
     }
 
-    queues = {
-        SMS_TYPE: 'db-sms',
-        EMAIL_TYPE: 'db-email',
-        LETTER_TYPE: 'db-letter',
-    }
-
     send_fn = send_fns[template_type]
 
     send_fn.apply_async(
@@ -127,7 +122,7 @@ def process_row(row_number, recipient, personalisation, template, job, service):
             encrypted,
             datetime.utcnow().strftime(DATETIME_FORMAT)
         ),
-        queue=queues[template_type] if not service.research_mode else 'research-mode'
+        queue=QueueNames.DATABASE if not service.research_mode else QueueNames.RESEARCH_MODE
     )
 
 
@@ -181,7 +176,7 @@ def send_sms(self,
 
         provider_tasks.deliver_sms.apply_async(
             [str(saved_notification.id)],
-            queue='send-sms' if not service.research_mode else 'research-mode'
+            queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE
         )
 
         current_app.logger.info(
@@ -226,7 +221,7 @@ def send_email(self,
 
         provider_tasks.deliver_email.apply_async(
             [str(saved_notification.id)],
-            queue='send-email' if not service.research_mode else 'research-mode'
+            queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE
         )
 
         current_app.logger.info("Email {} created at {}".format(saved_notification.id, created_at))
@@ -284,10 +279,9 @@ def build_dvla_file(self, job_id):
                 file_location="{}-dvla-job.text".format(job_id)
             )
             dao_update_job_status(job_id, JOB_STATUS_READY_TO_SEND)
-            notify_celery.send_task("aggregrate-dvla-files", ([str(job_id)], ), queue='aggregate-dvla-files')
         else:
             current_app.logger.info("All notifications for job {} are not persisted".format(job_id))
-            self.retry(queue="retry", exc="All notifications for job {} are not persisted".format(job_id))
+            self.retry(queue=QueueNames.RETRY, exc="All notifications for job {} are not persisted".format(job_id))
     except Exception as e:
         current_app.logger.exception("build_dvla_file threw exception")
         raise e
@@ -341,7 +335,7 @@ def handle_exception(task, notification, notification_id, exc):
         # send to the retry queue.
         current_app.logger.exception('Retry' + retry_msg)
         try:
-            task.retry(queue="retry", exc=exc)
+            task.retry(queue=QueueNames.RETRY, exc=exc)
         except task.MaxRetriesExceededError:
             current_app.logger.exception('Retry' + retry_msg)