diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py index e00591f09..4d8718489 100644 --- a/app/celery/provider_tasks.py +++ b/app/celery/provider_tasks.py @@ -3,6 +3,7 @@ from notifications_utils.recipients import InvalidEmailError from sqlalchemy.orm.exc import NoResultFound from app import notify_celery +from app.config import QueueNames from app.dao import notifications_dao from app.dao.notifications_dao import update_notification_status_by_id from app.statsd_decorators import statsd @@ -46,7 +47,7 @@ def deliver_sms(self, notification_id): current_app.logger.exception( "SMS notification delivery for id: {} failed".format(notification_id) ) - self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries)) + self.retry(queue=QueueNames.RETRY, countdown=retry_iteration_to_delay(self.request.retries)) except self.MaxRetriesExceededError: current_app.logger.exception( "RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification_id), @@ -70,7 +71,7 @@ def deliver_email(self, notification_id): current_app.logger.exception( "RETRY: Email notification {} failed".format(notification_id) ) - self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries)) + self.retry(queue=QueueNames.RETRY, countdown=retry_iteration_to_delay(self.request.retries)) except self.MaxRetriesExceededError: current_app.logger.error( "RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification_id) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 2b87150e0..460280424 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -23,6 +23,7 @@ from app.dao.provider_details_dao import ( from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago from app.statsd_decorators import statsd from app.celery.tasks import process_job +from app.config import QueueNames @notify_celery.task(name="remove_csv_files") @@ -39,7 +40,7 @@ def remove_csv_files(): def run_scheduled_jobs(): try: for job in dao_set_scheduled_jobs_to_pending(): - process_job.apply_async([str(job.id)], queue="process-job") + process_job.apply_async([str(job.id)], queue=QueueNames.JOBS) current_app.logger.info("Job ID {} added to process job queue".format(job.id)) except SQLAlchemyError as e: current_app.logger.exception("Failed to run scheduled jobs") diff --git a/app/celery/statistics_tasks.py b/app/celery/statistics_tasks.py index a82a3791f..150fa6aac 100644 --- a/app/celery/statistics_tasks.py +++ b/app/celery/statistics_tasks.py @@ -3,7 +3,6 @@ from sqlalchemy.exc import SQLAlchemyError from app import notify_celery from flask import current_app -from app.models import JobStatistics from app.statsd_decorators import statsd from app.dao.statistics_dao import ( create_or_update_job_sending_statistics, @@ -11,16 +10,17 @@ from app.dao.statistics_dao import ( ) from app.dao.notifications_dao import get_notification_by_id from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED +from app.config import QueueNames def create_initial_notification_statistic_tasks(notification): if notification.job_id and notification.status: - record_initial_job_statistics.apply_async((str(notification.id),), queue="statistics") + record_initial_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS) def create_outcome_notification_statistic_tasks(notification): if notification.job_id and notification.status in NOTIFICATION_STATUS_TYPES_COMPLETED: - record_outcome_job_statistics.apply_async((str(notification.id),), queue="statistics") + record_outcome_job_statistics.apply_async((str(notification.id),), queue=QueueNames.STATISTICS) @notify_celery.task(bind=True, name='record_initial_job_statistics', max_retries=20, default_retry_delay=10) @@ -35,7 +35,7 @@ def record_initial_job_statistics(self, notification_id): raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id)) except SQLAlchemyError as e: current_app.logger.exception(e) - self.retry(queue="retry") + self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: current_app.logger.error( "RETRY FAILED: task record_initial_job_statistics failed for notification {}".format( @@ -53,12 +53,12 @@ def record_outcome_job_statistics(self, notification_id): if notification: updated_count = update_job_stats_outcome_count(notification) if updated_count == 0: - self.retry(queue="retry") + self.retry(queue=QueueNames.RETRY) else: raise SQLAlchemyError("Failed to find notification with id {}".format(notification_id)) except SQLAlchemyError as e: current_app.logger.exception(e) - self.retry(queue="retry") + self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: current_app.logger.error( "RETRY FAILED: task update_job_stats_outcome_count failed for notification {}".format( diff --git a/app/celery/tasks.py b/app/celery/tasks.py index cce6b05f0..8fc6954f8 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -16,6 +16,7 @@ from app import ( ) from app.aws import s3 from app.celery import provider_tasks +from app.config import QueueNames from app.dao.jobs_dao import ( dao_update_job, dao_get_job_by_id, @@ -80,7 +81,7 @@ def process_job(job_id): process_row(row_number, recipient, personalisation, template, job, service) if template.template_type == LETTER_TYPE: - build_dvla_file.apply_async([str(job.id)], queue='process-job') + build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS) # temporary logging current_app.logger.info("send job {} to build-dvla-file in the process-job queue".format(job_id)) else: @@ -112,12 +113,6 @@ def process_row(row_number, recipient, personalisation, template, job, service): LETTER_TYPE: persist_letter } - queues = { - SMS_TYPE: 'db-sms', - EMAIL_TYPE: 'db-email', - LETTER_TYPE: 'db-letter', - } - send_fn = send_fns[template_type] send_fn.apply_async( @@ -127,7 +122,7 @@ def process_row(row_number, recipient, personalisation, template, job, service): encrypted, datetime.utcnow().strftime(DATETIME_FORMAT) ), - queue=queues[template_type] if not service.research_mode else 'research-mode' + queue=QueueNames.DATABASE if not service.research_mode else QueueNames.RESEARCH_MODE ) @@ -181,7 +176,7 @@ def send_sms(self, provider_tasks.deliver_sms.apply_async( [str(saved_notification.id)], - queue='send-sms' if not service.research_mode else 'research-mode' + queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE ) current_app.logger.info( @@ -226,7 +221,7 @@ def send_email(self, provider_tasks.deliver_email.apply_async( [str(saved_notification.id)], - queue='send-email' if not service.research_mode else 'research-mode' + queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE ) current_app.logger.info("Email {} created at {}".format(saved_notification.id, created_at)) @@ -284,10 +279,9 @@ def build_dvla_file(self, job_id): file_location="{}-dvla-job.text".format(job_id) ) dao_update_job_status(job_id, JOB_STATUS_READY_TO_SEND) - notify_celery.send_task("aggregrate-dvla-files", ([str(job_id)], ), queue='aggregate-dvla-files') else: current_app.logger.info("All notifications for job {} are not persisted".format(job_id)) - self.retry(queue="retry", exc="All notifications for job {} are not persisted".format(job_id)) + self.retry(queue=QueueNames.RETRY, exc="All notifications for job {} are not persisted".format(job_id)) except Exception as e: current_app.logger.exception("build_dvla_file threw exception") raise e @@ -341,7 +335,7 @@ def handle_exception(task, notification, notification_id, exc): # send to the retry queue. current_app.logger.exception('Retry' + retry_msg) try: - task.retry(queue="retry", exc=exc) + task.retry(queue=QueueNames.RETRY, exc=exc) except task.MaxRetriesExceededError: current_app.logger.exception('Retry' + retry_msg) diff --git a/app/delivery/rest.py b/app/delivery/rest.py index 0bacb43bb..489a5fcda 100644 --- a/app/delivery/rest.py +++ b/app/delivery/rest.py @@ -1,5 +1,6 @@ from flask import Blueprint, jsonify +from app.config import QueueNames from app.delivery import send_to_providers from app.models import EMAIL_TYPE from app.celery import provider_tasks @@ -23,18 +24,16 @@ def send_notification_to_provider(notification_id): send_response( send_to_providers.send_email_to_provider, provider_tasks.deliver_email, - notification, - 'send-email') + notification) else: send_response( send_to_providers.send_sms_to_provider, provider_tasks.deliver_sms, - notification, - 'send-sms') + notification) return jsonify({}), 204 -def send_response(send_call, task_call, notification, queue): +def send_response(send_call, task_call, notification): try: send_call(notification) except Exception as e: @@ -43,4 +42,4 @@ def send_response(send_call, task_call, notification, queue): notification.id, notification.notification_type), e) - task_call.apply_async((str(notification.id)), queue=queue) + task_call.apply_async((str(notification.id)), queue=QueueNames.SEND) diff --git a/app/invite/rest.py b/app/invite/rest.py index 2361629d1..8105b171f 100644 --- a/app/invite/rest.py +++ b/app/invite/rest.py @@ -4,6 +4,7 @@ from flask import ( jsonify, current_app) +from app.config import QueueNames from app.dao.invited_user_dao import ( save_invited_user, get_invited_user, @@ -44,7 +45,7 @@ def create_invited_user(service_id): key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue="notify") + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify(data=invited_user_schema.dump(invited_user).data), 201 diff --git a/app/job/rest.py b/app/job/rest.py index 8195ca87a..b4882945f 100644 --- a/app/job/rest.py +++ b/app/job/rest.py @@ -34,6 +34,8 @@ from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANC from app.utils import pagination_links +from app.config import QueueNames + job_blueprint = Blueprint('job', __name__, url_prefix='/service//job') from app.errors import ( @@ -143,7 +145,7 @@ def create_job(service_id): dao_create_job(job) if job.job_status == JOB_STATUS_PENDING: - process_job.apply_async([str(job.id)], queue="process-job") + process_job.apply_async([str(job.id)], queue=QueueNames.JOBS) job_json = job_schema.dump(job).data job_json['statistics'] = [] diff --git a/app/letters/send_letter_jobs.py b/app/letters/send_letter_jobs.py index 7030b9bc5..91c39615a 100644 --- a/app/letters/send_letter_jobs.py +++ b/app/letters/send_letter_jobs.py @@ -2,6 +2,7 @@ from flask import Blueprint, jsonify from flask import request from app import notify_celery +from app.config import QueueNames from app.dao.jobs_dao import dao_get_all_letter_jobs from app.schemas import job_schema from app.v2.errors import register_errors @@ -15,7 +16,7 @@ register_errors(letter_job) @letter_job.route('/send-letter-jobs', methods=['POST']) def send_letter_jobs(): job_ids = validate(request.get_json(), letter_job_ids) - notify_celery.send_task(name="send-files-to-dvla", args=(job_ids['job_ids'],), queue="process-ftp") + notify_celery.send_task(name="send-files-to-dvla", args=(job_ids['job_ids'],), queue=QueueNames.PROCESS_FTP) return jsonify(data={"response": "Task created to send files to DVLA"}), 201 diff --git a/app/notifications/notifications_letter_callback.py b/app/notifications/notifications_letter_callback.py index cfcaf28df..ac2de9e5c 100644 --- a/app/notifications/notifications_letter_callback.py +++ b/app/notifications/notifications_letter_callback.py @@ -13,7 +13,7 @@ from app.celery.tasks import update_letter_notifications_statuses from app.v2.errors import register_errors from app.notifications.utils import autoconfirm_subscription from app.schema_validation import validate - +from app.config import QueueNames letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__) register_errors(letter_callback_blueprint) @@ -54,7 +54,7 @@ def process_letter_response(): filename = message['Records'][0]['s3']['object']['key'] current_app.logger.info('Received file from DVLA: {}'.format(filename)) current_app.logger.info('DVLA callback: Calling task to update letter notifications') - update_letter_notifications_statuses.apply_async([filename], queue='notify') + update_letter_notifications_statuses.apply_async([filename], queue=QueueNames.NOTIFY) return jsonify( result="success", message="DVLA callback succeeded" diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index 1d03efe8b..d59477143 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -10,6 +10,8 @@ from notifications_utils.recipients import ( from app import redis_store from app.celery import provider_tasks from notifications_utils.clients import redis + +from app.config import QueueNames from app.dao.notifications_dao import dao_create_notification, dao_delete_notifications_and_history_by_id from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE from app.v2.errors import BadRequestError, SendNotificationToQueueError @@ -90,12 +92,9 @@ def persist_notification( def send_notification_to_queue(notification, research_mode, queue=None): if research_mode or notification.key_type == KEY_TYPE_TEST: - queue = 'research-mode' + queue = QueueNames.RESEARCH_MODE elif not queue: - if notification.notification_type == SMS_TYPE: - queue = 'send-sms' - if notification.notification_type == EMAIL_TYPE: - queue = 'send-email' + queue = QueueNames.SEND if notification.notification_type == SMS_TYPE: deliver_task = provider_tasks.deliver_sms diff --git a/app/notifications/rest.py b/app/notifications/rest.py index faa086e32..bb136f8c8 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -6,6 +6,7 @@ from flask import ( ) from app import api_user, authenticated_service +from app.config import QueueNames from app.dao import ( templates_dao, notifications_dao @@ -134,7 +135,7 @@ def send_notification(notification_type): key_type=api_user.key_type, simulated=simulated) if not simulated: - queue_name = 'priority' if template.process_type == PRIORITY else None + queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None send_notification_to_queue(notification=notification_model, research_mode=authenticated_service.research_mode, queue=queue_name) diff --git a/app/service/sender.py b/app/service/sender.py index 3c6a6a03e..4919a93bf 100644 --- a/app/service/sender.py +++ b/app/service/sender.py @@ -1,5 +1,6 @@ from flask import current_app +from app.config import QueueNames from app.dao.services_dao import dao_fetch_service_by_id, dao_fetch_active_users_for_service from app.dao.templates_dao import dao_get_template_by_id from app.models import EMAIL_TYPE, KEY_TYPE_NORMAL @@ -24,7 +25,7 @@ def send_notification_to_service_users(service_id, template_id, personalisation= api_key_id=None, key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(notification, False, queue='notify') + send_notification_to_queue(notification, False, queue=QueueNames.NOTIFY) def _add_user_fields(user, personalisation, fields): diff --git a/app/user/rest.py b/app/user/rest.py index b40c3a31c..cc0c6e41b 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -4,6 +4,7 @@ from datetime import datetime from flask import (jsonify, request, Blueprint, current_app) +from app.config import QueueNames from app.dao.users_dao import ( get_user_by_id, save_model_user, @@ -182,7 +183,7 @@ def send_user_sms_code(user_id): # Assume that we never want to observe the Notify service's research mode # setting for this notification - we still need to be able to log into the # admin even if we're doing user research using this service: - send_notification_to_queue(saved_notification, False, queue='notify') + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -212,7 +213,7 @@ def send_user_confirm_new_email(user_id): key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue='notify') + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -239,7 +240,7 @@ def send_user_email_verification(user_id): key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue="notify") + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -265,7 +266,7 @@ def send_already_registered_email(user_id): key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue="notify") + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 @@ -327,7 +328,7 @@ def send_user_reset_password(): key_type=KEY_TYPE_NORMAL ) - send_notification_to_queue(saved_notification, False, queue="notify") + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) return jsonify({}), 204 diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index a5a3a1ec4..cf9269318 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -2,6 +2,7 @@ from flask import request, jsonify, current_app from sqlalchemy.orm.exc import NoResultFound from app import api_user, authenticated_service +from app.config import QueueNames from app.dao import services_dao, templates_dao from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY from app.notifications.process_notifications import ( @@ -57,7 +58,7 @@ def post_notification(notification_type): simulated=simulated) if not simulated: - queue_name = 'priority' if template.process_type == PRIORITY else None + queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None send_notification_to_queue( notification=notification, research_mode=authenticated_service.research_mode, diff --git a/tests/app/celery/test_provider_tasks.py b/tests/app/celery/test_provider_tasks.py index d2f815d9d..9c9a44e5f 100644 --- a/tests/app/celery/test_provider_tasks.py +++ b/tests/app/celery/test_provider_tasks.py @@ -61,7 +61,7 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task deliver_sms(notification_id) app.delivery.send_to_providers.send_sms_to_provider.assert_not_called() - app.celery.provider_tasks.deliver_sms.retry.assert_called_with(queue="retry", countdown=10) + app.celery.provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=10) def test_should_call_send_email_to_provider_from_deliver_email_task( @@ -83,7 +83,7 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_ta deliver_email(notification_id) app.delivery.send_to_providers.send_email_to_provider.assert_not_called() - app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry", countdown=10) + app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks", countdown=10) # DO THESE FOR THE 4 TYPES OF TASK @@ -94,7 +94,7 @@ def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(s deliver_sms(sample_notification.id) - provider_tasks.deliver_sms.retry.assert_called_with(queue='retry', countdown=10) + provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=10) assert sample_notification.status == 'technical-failure' @@ -105,7 +105,7 @@ def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_email_task deliver_email(sample_notification.id) - provider_tasks.deliver_email.retry.assert_called_with(queue='retry', countdown=10) + provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks", countdown=10) assert sample_notification.status == 'technical-failure' diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 3cdc3796b..5566ebd70 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -165,7 +165,7 @@ def test_should_update_scheduled_jobs_and_put_on_queue(notify_db, notify_db_sess updated_job = dao_get_job_by_id(job.id) assert updated_job.job_status == 'pending' - mocked.assert_called_with([str(job.id)], queue='process-job') + mocked.assert_called_with([str(job.id)], queue="job-tasks") def test_should_update_all_scheduled_jobs_and_put_on_queue(notify_db, notify_db_session, mocker): @@ -200,9 +200,9 @@ def test_should_update_all_scheduled_jobs_and_put_on_queue(notify_db, notify_db_ assert dao_get_job_by_id(job_2.id).job_status == 'pending' mocked.assert_has_calls([ - call([str(job_3.id)], queue='process-job'), - call([str(job_2.id)], queue='process-job'), - call([str(job_1.id)], queue='process-job') + call([str(job_3.id)], queue="job-tasks"), + call([str(job_2.id)], queue="job-tasks"), + call([str(job_1.id)], queue="job-tasks") ]) diff --git a/tests/app/celery/test_statistics_tasks.py b/tests/app/celery/test_statistics_tasks.py index 24aaba97d..40d20117d 100644 --- a/tests/app/celery/test_statistics_tasks.py +++ b/tests/app/celery/test_statistics_tasks.py @@ -17,7 +17,7 @@ def test_should_create_initial_job_task_if_notification_is_related_to_a_job( mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async") notification = sample_notification(notify_db, notify_db_session, job=sample_job) create_initial_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics") + mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") @pytest.mark.parametrize('status', [ @@ -29,7 +29,7 @@ def test_should_create_intial_job_task_if_notification_is_not_in_completed_state mock = mocker.patch("app.celery.statistics_tasks.record_initial_job_statistics.apply_async") notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status) create_initial_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics") + mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") def test_should_not_create_initial_job_task_if_notification_is_not_related_to_a_job( @@ -47,7 +47,7 @@ def test_should_create_outcome_job_task_if_notification_is_related_to_a_job( mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async") notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=NOTIFICATION_DELIVERED) create_outcome_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue="statistics") + mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") @pytest.mark.parametrize('status', NOTIFICATION_STATUS_TYPES_COMPLETED) @@ -57,7 +57,7 @@ def test_should_create_outcome_job_task_if_notification_is_in_completed_state( mock = mocker.patch("app.celery.statistics_tasks.record_outcome_job_statistics.apply_async") notification = sample_notification(notify_db, notify_db_session, job=sample_job, status=status) create_outcome_notification_statistic_tasks(notification) - mock.assert_called_once_with((str(notification.id), ), queue='statistics') + mock.assert_called_once_with((str(notification.id), ), queue="statistics-tasks") @pytest.mark.parametrize('status', [ @@ -100,7 +100,7 @@ def test_should_retry_if_persisting_the_job_stats_has_a_sql_alchemy_exception( record_initial_job_statistics(str(sample_notification.id)) dao_mock.assert_called_once_with(sample_notification) - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") def test_should_call_update_job_stats_dao_outcome_methods(notify_db, notify_db_session, sample_notification, mocker): @@ -123,7 +123,7 @@ def test_should_retry_if_persisting_the_job_outcome_stats_has_a_sql_alchemy_exce record_outcome_job_statistics(str(sample_notification.id)) dao_mock.assert_called_once_with(sample_notification) - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") def test_should_retry_if_persisting_the_job_outcome_stats_updates_zero_rows( @@ -136,7 +136,7 @@ def test_should_retry_if_persisting_the_job_outcome_stats_updates_zero_rows( record_outcome_job_statistics(str(sample_notification.id)) dao_mock.assert_called_once_with(sample_notification) - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") def test_should_retry_if_persisting_the_job_stats_creation_cant_find_notification_by_id( @@ -148,7 +148,7 @@ def test_should_retry_if_persisting_the_job_stats_creation_cant_find_notificatio record_initial_job_statistics(str(create_uuid())) dao_mock.assert_not_called() - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") def test_should_retry_if_persisting_the_job_stats_outcome_cant_find_notification_by_id( @@ -161,4 +161,4 @@ def test_should_retry_if_persisting_the_job_stats_outcome_cant_find_notification record_outcome_job_statistics(str(create_uuid())) dao_mock.assert_not_called() - retry_mock.assert_called_with(queue="retry") + retry_mock.assert_called_with(queue="retry-tasks") diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 27c9833a4..8cb5a13d4 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -108,7 +108,7 @@ def test_should_process_sms_job(sample_job, mocker): "uuid", "something_encrypted", "2016-01-01T11:09:00.061258Z"), - queue="db-sms" + queue="database-tasks" ) job = jobs_dao.dao_get_job_by_id(sample_job.id) assert job.job_status == 'finished' @@ -237,7 +237,7 @@ def test_should_process_email_job_if_exactly_on_send_limits(notify_db, "something_encrypted", "2016-01-01T11:09:00.061258Z" ), - queue="db-email" + queue="database-tasks" ) @@ -283,7 +283,7 @@ def test_should_process_email_job(email_job_with_placeholders, mocker): "something_encrypted", "2016-01-01T11:09:00.061258Z" ), - queue="db-email" + queue="database-tasks" ) job = jobs_dao.dao_get_job_by_id(email_job_with_placeholders.id) assert job.job_status == 'finished' @@ -324,7 +324,7 @@ def test_should_process_letter_job(sample_letter_job, mocker): assert process_row_mock.call_count == 1 assert sample_letter_job.job_status == 'in progress' - tasks.build_dvla_file.apply_async.assert_called_once_with([str(sample_letter_job.id)], queue="process-job") + tasks.build_dvla_file.apply_async.assert_called_once_with([str(sample_letter_job.id)], queue="job-tasks") def test_should_process_all_sms_job(sample_job_with_placeholdered_template, @@ -355,12 +355,12 @@ def test_should_process_all_sms_job(sample_job_with_placeholdered_template, @freeze_time('2001-01-01T12:00:00') @pytest.mark.parametrize('template_type, research_mode, expected_function, expected_queue', [ - (SMS_TYPE, False, 'send_sms', 'db-sms'), - (SMS_TYPE, True, 'send_sms', 'research-mode'), - (EMAIL_TYPE, False, 'send_email', 'db-email'), - (EMAIL_TYPE, True, 'send_email', 'research-mode'), - (LETTER_TYPE, False, 'persist_letter', 'db-letter'), - (LETTER_TYPE, True, 'persist_letter', 'research-mode'), + (SMS_TYPE, False, 'send_sms', 'database-tasks'), + (SMS_TYPE, True, 'send_sms', 'research-mode-tasks'), + (EMAIL_TYPE, False, 'send_email', 'database-tasks'), + (EMAIL_TYPE, True, 'send_email', 'research-mode-tasks'), + (LETTER_TYPE, False, 'persist_letter', 'database-tasks'), + (LETTER_TYPE, True, 'persist_letter', 'research-mode-tasks'), ]) def test_process_row_sends_letter_task(template_type, research_mode, expected_function, expected_queue, mocker): mocker.patch('app.celery.tasks.create_uuid', return_value='noti_uuid') @@ -420,7 +420,7 @@ def test_should_send_template_to_correct_sms_task_and_persist(sample_template_wi assert persisted_notification.notification_type == 'sms' mocked_deliver_sms.assert_called_once_with( [str(persisted_notification.id)], - queue="send-sms" + queue="send-tasks" ) @@ -446,7 +446,7 @@ def test_should_put_send_sms_task_in_research_mode_queue_if_research_mode_servic persisted_notification = Notification.query.one() provider_tasks.deliver_sms.apply_async.assert_called_once_with( [str(persisted_notification.id)], - queue="research-mode" + queue="research-mode-tasks" ) assert mocked_deliver_sms.called @@ -481,7 +481,7 @@ def test_should_send_sms_if_restricted_service_and_valid_number(notify_db, notif assert persisted_notification.notification_type == 'sms' provider_tasks.deliver_sms.apply_async.assert_called_once_with( [str(persisted_notification.id)], - queue="send-sms" + queue="send-tasks" ) @@ -507,7 +507,7 @@ def test_should_send_sms_if_restricted_service_and_non_team_number_with_test_key persisted_notification = Notification.query.one() mocked_deliver_sms.assert_called_once_with( [str(persisted_notification.id)], - queue="send-sms" + queue="send-tasks" ) @@ -535,7 +535,7 @@ def test_should_send_email_if_restricted_service_and_non_team_email_address_with persisted_notification = Notification.query.one() mocked_deliver_email.assert_called_once_with( [str(persisted_notification.id)], - queue="send-email" + queue="send-tasks" ) @@ -602,7 +602,7 @@ def test_should_put_send_email_task_in_research_mode_queue_if_research_mode_serv persisted_notification = Notification.query.one() provider_tasks.deliver_email.apply_async.assert_called_once_with( [str(persisted_notification.id)], - queue="research-mode" + queue="research-mode-tasks" ) @@ -639,7 +639,7 @@ def test_should_send_sms_template_to_and_persist_with_job_id(sample_job, sample_ provider_tasks.deliver_sms.apply_async.assert_called_once_with( [str(persisted_notification.id)], - queue="send-sms" + queue="send-tasks" ) @@ -736,7 +736,7 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh assert persisted_notification.notification_type == 'email' provider_tasks.deliver_email.apply_async.assert_called_once_with( - [str(persisted_notification.id)], queue='send-email') + [str(persisted_notification.id)], queue='send-tasks') def test_send_email_should_use_template_version_from_job_not_latest(sample_email_template, mocker): @@ -767,7 +767,7 @@ def test_send_email_should_use_template_version_from_job_not_latest(sample_email assert not persisted_notification.sent_by assert persisted_notification.notification_type == 'email' provider_tasks.deliver_email.apply_async.assert_called_once_with([str(persisted_notification.id)], - queue='send-email') + queue='send-tasks') def test_should_use_email_template_subject_placeholders(sample_email_template_with_placeholders, mocker): @@ -793,7 +793,7 @@ def test_should_use_email_template_subject_placeholders(sample_email_template_wi assert not persisted_notification.reference assert persisted_notification.notification_type == 'email' provider_tasks.deliver_email.apply_async.assert_called_once_with( - [str(persisted_notification.id)], queue='send-email' + [str(persisted_notification.id)], queue='send-tasks' ) @@ -821,7 +821,7 @@ def test_should_use_email_template_and_persist_without_personalisation(sample_em assert not persisted_notification.reference assert persisted_notification.notification_type == 'email' provider_tasks.deliver_email.apply_async.assert_called_once_with([str(persisted_notification.id)], - queue='send-email') + queue='send-tasks') def test_send_sms_should_go_to_retry_queue_if_database_errors(sample_template, mocker): @@ -844,7 +844,7 @@ def test_send_sms_should_go_to_retry_queue_if_database_errors(sample_template, m now.strftime(DATETIME_FORMAT) ) assert provider_tasks.deliver_sms.apply_async.called is False - tasks.send_sms.retry.assert_called_with(exc=expected_exception, queue='retry') + tasks.send_sms.retry.assert_called_with(exc=expected_exception, queue="retry-tasks") assert Notification.query.count() == 0 @@ -869,7 +869,7 @@ def test_send_email_should_go_to_retry_queue_if_database_errors(sample_email_tem now.strftime(DATETIME_FORMAT) ) assert not provider_tasks.deliver_email.apply_async.called - tasks.send_email.retry.assert_called_with(exc=expected_exception, queue='retry') + tasks.send_email.retry.assert_called_with(exc=expected_exception, queue="retry-tasks") assert Notification.query.count() == 0 @@ -1002,7 +1002,6 @@ def test_build_dvla_file(sample_letter_template, mocker): file_location="{}-dvla-job.text".format(job.id) ) assert Job.query.get(job.id).job_status == 'ready to send' - mocked_send_task.assert_called_once_with("aggregrate-dvla-files", ([str(job.id)], ), queue='aggregate-dvla-files') def test_build_dvla_file_retries_if_all_notifications_are_not_created(sample_letter_template, mocker): @@ -1016,7 +1015,7 @@ def test_build_dvla_file_retries_if_all_notifications_are_not_created(sample_let build_dvla_file(job.id) mocked.assert_not_called() - tasks.build_dvla_file.retry.assert_called_with(queue='retry', + tasks.build_dvla_file.retry.assert_called_with(queue="retry-tasks", exc="All notifications for job {} are not persisted".format(job.id)) assert Job.query.get(job.id).job_status == 'in progress' mocked_send_task.assert_not_called() diff --git a/tests/app/delivery/test_rest.py b/tests/app/delivery/test_rest.py index fc51fb508..984bf90e1 100644 --- a/tests/app/delivery/test_rest.py +++ b/tests/app/delivery/test_rest.py @@ -78,7 +78,7 @@ def test_should_call_deliver_sms_task_if_send_sms_to_provider_fails(notify_api, ) app.delivery.send_to_providers.send_sms_to_provider.assert_called_with(sample_notification) app.celery.provider_tasks.deliver_sms.apply_async.assert_called_with( - (str(sample_notification.id)), queue='send-sms' + (str(sample_notification.id)), queue='send-tasks' ) assert response.status_code == 204 @@ -100,6 +100,6 @@ def test_should_call_deliver_email_task_if_send_email_to_provider_fails( ) app.delivery.send_to_providers.send_email_to_provider.assert_called_with(sample_email_notification) app.celery.provider_tasks.deliver_email.apply_async.assert_called_with( - (str(sample_email_notification.id)), queue='send-email' + (str(sample_email_notification.id)), queue='send-tasks' ) assert response.status_code == 204 diff --git a/tests/app/invite/test_invite_rest.py b/tests/app/invite/test_invite_rest.py index 182f4b05b..be1f46d82 100644 --- a/tests/app/invite/test_invite_rest.py +++ b/tests/app/invite/test_invite_rest.py @@ -33,7 +33,7 @@ def test_create_invited_user(client, sample_service, mocker, invitation_email_te assert json_resp['data']['id'] notification = Notification.query.first() - mocked.assert_called_once_with([(str(notification.id))], queue="notify") + mocked.assert_called_once_with([(str(notification.id))], queue="notify-internal-tasks") def test_create_invited_user_invalid_email(client, sample_service, mocker): diff --git a/tests/app/job/test_rest.py b/tests/app/job/test_rest.py index d583d6b31..6ebcb2e89 100644 --- a/tests/app/job/test_rest.py +++ b/tests/app/job/test_rest.py @@ -119,7 +119,7 @@ def test_create_unscheduled_job(notify_api, sample_template, mocker, fake_uuid): app.celery.tasks.process_job.apply_async.assert_called_once_with( ([str(fake_uuid)]), - queue="process-job" + queue="job-tasks" ) resp_json = json.loads(response.get_data(as_text=True)) diff --git a/tests/app/letters/test_send_letter_jobs.py b/tests/app/letters/test_send_letter_jobs.py index 7f1d88c85..1d32baf12 100644 --- a/tests/app/letters/test_send_letter_jobs.py +++ b/tests/app/letters/test_send_letter_jobs.py @@ -21,7 +21,7 @@ def test_send_letter_jobs(client, mocker): mock_celery.assert_called_once_with(name="send-files-to-dvla", args=(job_ids['job_ids'],), - queue="process-ftp") + queue="process-ftp-tasks") def test_send_letter_jobs_throws_validation_error(client, mocker): diff --git a/tests/app/notifications/rest/test_callbacks.py b/tests/app/notifications/rest/test_callbacks.py index 2c2ffd9cf..58bbc8b1f 100644 --- a/tests/app/notifications/rest/test_callbacks.py +++ b/tests/app/notifications/rest/test_callbacks.py @@ -68,7 +68,7 @@ def test_dvla_callback_calls_update_letter_notifications_task(client, mocker): assert response.status_code == 200 assert update_task.called - update_task.assert_called_with(['bar.txt'], queue='notify') + update_task.assert_called_with(['bar.txt'], queue='notify-internal-tasks') def test_dvla_callback_does_not_raise_error_parsing_json_for_plaintext_header(client, mocker): diff --git a/tests/app/notifications/rest/test_send_notification.py b/tests/app/notifications/rest/test_send_notification.py index ab527e06e..b491e3d7f 100644 --- a/tests/app/notifications/rest/test_send_notification.py +++ b/tests/app/notifications/rest/test_send_notification.py @@ -127,7 +127,7 @@ def test_send_notification_with_placeholders_replaced(notify_api, sample_email_t mocked.assert_called_once_with( [notification_id], - queue="send-email" + queue="send-tasks" ) assert response.status_code == 201 assert response_data['body'] == u'Hello Jo\nThis is an email from GOV.\u200BUK' @@ -338,7 +338,7 @@ def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker response_data = json.loads(response.data)['data'] notification_id = response_data['notification']['id'] - mocked.assert_called_once_with([notification_id], queue='send-sms') + mocked.assert_called_once_with([notification_id], queue='send-tasks') assert response.status_code == 201 assert notification_id assert 'subject' not in response_data @@ -392,7 +392,7 @@ def test_should_allow_valid_email_notification(notify_api, sample_email_template notification_id = response_data['notification']['id'] app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with( [notification_id], - queue="send-email" + queue="send-tasks" ) assert response.status_code == 201 @@ -593,7 +593,7 @@ def test_should_send_email_if_team_api_key_and_a_service_user(notify_api, sample data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) - app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue='send-email') + app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue='send-tasks') assert response.status_code == 201 @@ -626,7 +626,9 @@ def test_should_send_sms_to_anyone_with_test_key( data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))] ) - app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='research-mode') + app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with( + [fake_uuid], queue='research-mode-tasks' + ) assert response.status_code == 201 @@ -660,7 +662,9 @@ def test_should_send_email_to_anyone_with_test_key( headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))] ) - app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with([fake_uuid], queue='research-mode') + app.celery.provider_tasks.deliver_email.apply_async.assert_called_once_with( + [fake_uuid], queue='research-mode-tasks' + ) assert response.status_code == 201 @@ -685,7 +689,7 @@ def test_should_send_sms_if_team_api_key_and_a_service_user(notify_api, sample_t data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) - app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='send-sms') + app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with([fake_uuid], queue='send-tasks') assert response.status_code == 201 @@ -718,7 +722,7 @@ def test_should_persist_notification(notify_api, sample_template, data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) - mocked.assert_called_once_with([fake_uuid], queue='send-{}'.format(template_type)) + mocked.assert_called_once_with([fake_uuid], queue='send-tasks') assert response.status_code == 201 notification = notifications_dao.get_notification_by_id(fake_uuid) @@ -761,7 +765,7 @@ def test_should_delete_notification_and_return_error_if_sqs_fails( data=json.dumps(data), headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))]) - mocked.assert_called_once_with([fake_uuid], queue='send-{}'.format(template_type)) + mocked.assert_called_once_with([fake_uuid], queue='send-tasks') assert response.status_code == 500 assert not notifications_dao.get_notification_by_id(fake_uuid) assert not NotificationHistory.query.get(fake_uuid) @@ -1046,7 +1050,7 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori notification_id = response_data['notification']['id'] assert response.status_code == 201 - mocked.assert_called_once_with([notification_id], queue='priority') + mocked.assert_called_once_with([notification_id], queue='priority-tasks') @pytest.mark.parametrize( @@ -1114,7 +1118,7 @@ def test_should_allow_store_original_number_on_sms_notification(client, sample_t response_data = json.loads(response.data)['data'] notification_id = response_data['notification']['id'] - mocked.assert_called_once_with([notification_id], queue='send-sms') + mocked.assert_called_once_with([notification_id], queue='send-tasks') assert response.status_code == 201 assert notification_id notifications = Notification.query.all() diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py index be2356c69..8d707c7dc 100644 --- a/tests/app/notifications/test_process_notification.py +++ b/tests/app/notifications/test_process_notification.py @@ -207,17 +207,17 @@ def test_persist_notification_increments_cache_if_key_exists(sample_template, sa @pytest.mark.parametrize('research_mode, requested_queue, expected_queue, notification_type, key_type', - [(True, None, 'research-mode', 'sms', 'normal'), - (True, None, 'research-mode', 'email', 'normal'), - (True, None, 'research-mode', 'email', 'team'), - (False, None, 'send-sms', 'sms', 'normal'), - (False, None, 'send-email', 'email', 'normal'), - (False, None, 'send-sms', 'sms', 'team'), - (False, None, 'research-mode', 'sms', 'test'), - (True, 'notify', 'research-mode', 'email', 'normal'), - (False, 'notify', 'notify', 'sms', 'normal'), - (False, 'notify', 'notify', 'email', 'normal'), - (False, 'notify', 'research-mode', 'sms', 'test')]) + [(True, None, 'research-mode-tasks', 'sms', 'normal'), + (True, None, 'research-mode-tasks', 'email', 'normal'), + (True, None, 'research-mode-tasks', 'email', 'team'), + (False, None, 'send-tasks', 'sms', 'normal'), + (False, None, 'send-tasks', 'email', 'normal'), + (False, None, 'send-tasks', 'sms', 'team'), + (False, None, 'research-mode-tasks', 'sms', 'test'), + (True, 'notify-internal-tasks', 'research-mode-tasks', 'email', 'normal'), + (False, 'notify-internal-tasks', 'notify-internal-tasks', 'sms', 'normal'), + (False, 'notify-internal-tasks', 'notify-internal-tasks', 'email', 'normal'), + (False, 'notify-internal-tasks', 'research-mode-tasks', 'sms', 'test')]) def test_send_notification_to_queue(notify_db, notify_db_session, research_mode, requested_queue, expected_queue, notification_type, key_type, mocker): diff --git a/tests/app/user/test_rest.py b/tests/app/user/test_rest.py index ddbb3eaae..73f60f05b 100644 --- a/tests/app/user/test_rest.py +++ b/tests/app/user/test_rest.py @@ -374,7 +374,7 @@ def test_send_user_reset_password_should_send_reset_password_link(client, assert resp.status_code == 204 notification = Notification.query.first() - mocked.assert_called_once_with([str(notification.id)], queue="notify") + mocked.assert_called_once_with([str(notification.id)], queue="notify-internal-tasks") def test_send_user_reset_password_should_return_400_when_email_is_missing(client, mocker): @@ -436,7 +436,7 @@ def test_send_already_registered_email(client, sample_user, already_registered_t assert resp.status_code == 204 notification = Notification.query.first() - mocked.assert_called_once_with(([str(notification.id)]), queue="notify") + mocked.assert_called_once_with(([str(notification.id)]), queue="notify-internal-tasks") def test_send_already_registered_email_returns_400_when_data_is_missing(client, sample_user): @@ -464,7 +464,7 @@ def test_send_user_confirm_new_email_returns_204(client, sample_user, change_ema notification = Notification.query.first() mocked.assert_called_once_with( ([str(notification.id)]), - queue="notify") + queue="notify-internal-tasks") def test_send_user_confirm_new_email_returns_400_when_email_missing(client, sample_user, mocker): diff --git a/tests/app/user/test_rest_verify.py b/tests/app/user/test_rest_verify.py index 08f510bbd..84a88f6a0 100644 --- a/tests/app/user/test_rest_verify.py +++ b/tests/app/user/test_rest_verify.py @@ -218,7 +218,7 @@ def test_send_user_sms_code(client, app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with( ([str(notification.id)]), - queue="notify" + queue="notify-internal-tasks" ) @@ -246,7 +246,7 @@ def test_send_user_code_for_sms_with_optional_to_field(client, assert notification.to == to_number app.celery.provider_tasks.deliver_sms.apply_async.assert_called_once_with( ([str(notification.id)]), - queue="notify" + queue="notify-internal-tasks" ) @@ -294,7 +294,7 @@ def test_send_user_email_verification(client, headers=[('Content-Type', 'application/json'), auth_header]) assert resp.status_code == 204 notification = Notification.query.first() - mocked.assert_called_once_with(([str(notification.id)]), queue="notify") + mocked.assert_called_once_with(([str(notification.id)]), queue="notify-internal-tasks") def test_send_email_verification_returns_404_for_bad_input_data(client, notify_db_session, mocker): diff --git a/tests/app/v2/notifications/test_post_notifications.py b/tests/app/v2/notifications/test_post_notifications.py index 4e721f15f..0bd1156ba 100644 --- a/tests/app/v2/notifications/test_post_notifications.py +++ b/tests/app/v2/notifications/test_post_notifications.py @@ -228,7 +228,7 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori notification_id = json.loads(response.data)['id'] assert response.status_code == 201 - mocked.assert_called_once_with([notification_id], queue='priority') + mocked.assert_called_once_with([notification_id], queue='priority-tasks') @pytest.mark.parametrize(