From 8c4a084a0dc55f30db424871ab9ccf426b315945 Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Mon, 21 Aug 2017 14:33:08 +0100 Subject: [PATCH 1/9] Refactor letters filename --- app/__init__.py | 2 +- app/letters/{send_letter_jobs.py => rest.py} | 0 tests/app/letters/test_send_letter_jobs.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename app/letters/{send_letter_jobs.py => rest.py} (100%) diff --git a/app/__init__.py b/app/__init__.py index cb9e346a1..d2c513b57 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -100,7 +100,7 @@ def register_blueprint(application): from app.notifications.notifications_sms_callback import sms_callback_blueprint from app.notifications.notifications_letter_callback import letter_callback_blueprint from app.authentication.auth import requires_admin_auth, requires_auth, requires_no_auth, restrict_ip_sms - from app.letters.send_letter_jobs import letter_job + from app.letters.rest import letter_job from app.billing.rest import billing_blueprint service_blueprint.before_request(requires_admin_auth) diff --git a/app/letters/send_letter_jobs.py b/app/letters/rest.py similarity index 100% rename from app/letters/send_letter_jobs.py rename to app/letters/rest.py diff --git a/tests/app/letters/test_send_letter_jobs.py b/tests/app/letters/test_send_letter_jobs.py index b0d34b987..d38e24c8f 100644 --- a/tests/app/letters/test_send_letter_jobs.py +++ b/tests/app/letters/test_send_letter_jobs.py @@ -8,7 +8,7 @@ from tests import create_authorization_header def test_send_letter_jobs(client, mocker): - mock_celery = mocker.patch("app.letters.send_letter_jobs.notify_celery.send_task") + mock_celery = mocker.patch("app.letters.rest.notify_celery.send_task") job_ids = {"job_ids": [str(uuid.uuid4()), str(uuid.uuid4()), str(uuid.uuid4())]} auth_header = create_authorization_header() @@ -27,7 +27,7 @@ def test_send_letter_jobs(client, mocker): def test_send_letter_jobs_throws_validation_error(client, mocker): - mock_celery = mocker.patch("app.letters.send_letter_jobs.notify_celery.send_task") + mock_celery = mocker.patch("app.letters.rest.notify_celery.send_task") job_ids = {"job_ids": ["1", "2"]} From 18881cd580d97e8b02757ec9468950271383a07f Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Tue, 22 Aug 2017 09:55:47 +0100 Subject: [PATCH 2/9] Add scheduled letter jobs --- app/celery/scheduled_tasks.py | 15 +++++++++++++- app/config.py | 9 +++++++- app/dao/jobs_dao.py | 16 +++++++++++++++ app/letters/rest.py | 2 +- tests/app/celery/test_scheduled_tasks.py | 19 +++++++++++++++++ tests/app/dao/test_jobs_dao.py | 26 +++++++++++++++++++++--- 6 files changed, 81 insertions(+), 6 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 8e0beef4b..f95420f33 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -13,6 +13,7 @@ from app.dao.date_util import get_month_start_and_end_date_in_utc from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago from app.dao.jobs_dao import ( + dao_get_letter_jobs_by_status, dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by ) @@ -32,7 +33,7 @@ from app.dao.provider_details_dao import ( dao_toggle_sms_provider ) from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago -from app.models import LETTER_TYPE +from app.models import LETTER_TYPE, JOB_STATUS_READY_TO_SEND from app.notifications.process_notifications import send_notification_to_queue from app.statsd_decorators import statsd from app.celery.tasks import process_job @@ -302,3 +303,15 @@ def populate_monthly_billing(): start_date, end_date = get_month_start_and_end_date_in_utc(yesterday_in_bst) services = get_service_ids_that_need_billing_populated(start_date=start_date, end_date=end_date) [create_or_update_monthly_billing(service_id=s.service_id, billing_month=end_date) for s in services] + + +@notify_celery.task(name="run-letter-jobs") +@statsd(namespace="tasks") +def run_letter_jobs(): + ids = dao_get_letter_jobs_by_status(JOB_STATUS_READY_TO_SEND) + notify_celery.send_task( + name=QueueNames.DVLA_FILES, + args=(ids), + queue=QueueNames.PROCESS_FTP + ) + current_app.logger.info("Queued {} ready letter job ids onto {}".format(len(ids), QueueNames.PROCESS_FTP)) diff --git a/app/config.py b/app/config.py index 678193dc4..51285050a 100644 --- a/app/config.py +++ b/app/config.py @@ -30,6 +30,7 @@ class QueueNames(object): RETRY = 'retry-tasks' NOTIFY = 'notify-internal-tasks' PROCESS_FTP = 'process-ftp-tasks' + DVLA_FILES = 'send-files-to-dvla' @staticmethod def all_queues(): @@ -44,7 +45,8 @@ class QueueNames(object): QueueNames.JOBS, QueueNames.RETRY, QueueNames.NOTIFY, - QueueNames.PROCESS_FTP + QueueNames.PROCESS_FTP, + QueueNames.DVLA_FILES ] @@ -220,6 +222,11 @@ class Config(object): 'task': 'populate_monthly_billing', 'schedule': crontab(minute=10, hour=5), 'options': {'queue': QueueNames.PERIODIC} + }, + 'run-letter-jobs': { + 'task': 'run-letter-jobs', + 'schedule': crontab(minute=0, hour=16), + 'options': {'queue': QueueNames.PERIODIC} } } CELERY_QUEUES = [] diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index e6a80832e..ee90ffbe8 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -157,6 +157,22 @@ def dao_get_all_letter_jobs(): ).all() +def dao_get_letter_jobs_by_status(status): + return db.session.query( + Job + ).join( + Job.template + ).filter( + Job.job_status == status, + Template.template_type == LETTER_TYPE, + # test letter jobs (or from research mode services) are created with a different filename, + # exclude them so we don't see them on the send to CSV + Job.original_file_name != LETTER_TEST_API_FILENAME + ).order_by( + desc(Job.created_at) + ).all() + + @statsd(namespace="dao") def dao_get_job_statistics_for_job(service_id, job_id): query = Job.query.join( diff --git a/app/letters/rest.py b/app/letters/rest.py index 91c39615a..ce43ea169 100644 --- a/app/letters/rest.py +++ b/app/letters/rest.py @@ -16,7 +16,7 @@ register_errors(letter_job) @letter_job.route('/send-letter-jobs', methods=['POST']) def send_letter_jobs(): job_ids = validate(request.get_json(), letter_job_ids) - notify_celery.send_task(name="send-files-to-dvla", args=(job_ids['job_ids'],), queue=QueueNames.PROCESS_FTP) + notify_celery.send_task(name=QueueNames.DVLA_FILES, args=(job_ids['job_ids'],), queue=QueueNames.PROCESS_FTP) return jsonify(data={"response": "Task created to send files to DVLA"}), 201 diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 2e20ee138..f2720a2cb 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -1,3 +1,5 @@ +import uuid + from datetime import datetime, timedelta from functools import partial from unittest.mock import call, patch, PropertyMock @@ -20,6 +22,7 @@ from app.celery.scheduled_tasks import ( remove_csv_files, remove_transformed_dvla_files, run_scheduled_jobs, + run_letter_jobs, s3, send_daily_performance_platform_stats, send_scheduled_notifications, @@ -28,6 +31,7 @@ from app.celery.scheduled_tasks import ( timeout_notifications, populate_monthly_billing) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient +from app.config import QueueNames from app.dao.jobs_dao import dao_get_job_by_id from app.dao.notifications_dao import dao_get_scheduled_notifications from app.dao.provider_details_dao import ( @@ -677,3 +681,18 @@ def test_populate_monthly_billing_updates_correct_month_in_bst(sample_template): assert monthly_billing[1].notification_type == 'sms' assert monthly_billing[1].monthly_totals[0]['billing_units'] == 1 assert monthly_billing[1].monthly_totals[0]['total_cost'] == 0.0123 + + +def test_run_letter_jobs(client, mocker): + job_ids = [str(uuid.uuid4()), str(uuid.uuid4())] + mocker.patch( + "app.celery.scheduled_tasks.dao_get_letter_jobs_by_status", + return_value=job_ids + ) + mock_celery = mocker.patch("app.celery.tasks.notify_celery.send_task") + + run_letter_jobs() + + mock_celery.assert_called_once_with(name=QueueNames.DVLA_FILES, + args=(job_ids), + queue=QueueNames.PROCESS_FTP) diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index d2c00981b..4e2f3062f 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -18,18 +18,24 @@ from app.dao.jobs_dao import ( dao_get_all_notifications_for_job, dao_get_jobs_older_than_limited_by, dao_get_job_statistics_for_job, - dao_get_job_stats_for_service) + dao_get_job_stats_for_service, + dao_get_letter_jobs_by_status) from app.dao.statistics_dao import create_or_update_job_sending_statistics, update_job_stats_outcome_count from app.models import ( Job, JobStatistics, - EMAIL_TYPE, SMS_TYPE, LETTER_TYPE + EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, + JOB_STATUS_READY_TO_SEND ) from tests.app.conftest import sample_notification as create_notification from tests.app.conftest import sample_job as create_job from tests.app.conftest import sample_service as create_service from tests.app.conftest import sample_template as create_template -from tests.app.db import create_user +from tests.app.db import ( + create_user, + create_job as create_db_job, + create_template as create_db_template +) def test_should_have_decorated_notifications_dao_functions(): @@ -542,3 +548,17 @@ def stats_set_up(notify_db, notify_db_session, service): update_job_stats_outcome_count(notification_delivered) update_job_stats_outcome_count(notification_failed) return job_1, job_2 + + +def test_dao_get_letter_jobs_by_status(sample_service): + create_db_template(service=sample_service, template_type=SMS_TYPE) + create_db_template(service=sample_service, template_type=EMAIL_TYPE) + letter_template = create_db_template(service=sample_service, template_type=LETTER_TYPE) + jobs = [] + jobs.append(create_db_job(letter_template, job_status=JOB_STATUS_READY_TO_SEND, original_file_name='1.csv')) + jobs.append(create_db_job(letter_template, job_status=JOB_STATUS_READY_TO_SEND, original_file_name='2.csv')) + + result = dao_get_letter_jobs_by_status(JOB_STATUS_READY_TO_SEND) + + assert len(result) == 2 + assert set(result) == set(jobs) From bd79a6f770834226178f4f4d14ab19c33472d226 Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Tue, 22 Aug 2017 11:05:58 +0100 Subject: [PATCH 3/9] Improved tests on letter jobs by status --- tests/app/dao/test_jobs_dao.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index 4e2f3062f..72f3ece04 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -24,7 +24,7 @@ from app.dao.statistics_dao import create_or_update_job_sending_statistics, upda from app.models import ( Job, JobStatistics, EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, - JOB_STATUS_READY_TO_SEND + JOB_STATUS_READY_TO_SEND, JOB_STATUS_SENT_TO_DVLA, JOB_STATUS_FINISHED, JOB_STATUS_PENDING ) from tests.app.conftest import sample_notification as create_notification @@ -557,7 +557,9 @@ def test_dao_get_letter_jobs_by_status(sample_service): jobs = [] jobs.append(create_db_job(letter_template, job_status=JOB_STATUS_READY_TO_SEND, original_file_name='1.csv')) jobs.append(create_db_job(letter_template, job_status=JOB_STATUS_READY_TO_SEND, original_file_name='2.csv')) - + create_db_job(letter_template, job_status=JOB_STATUS_SENT_TO_DVLA, original_file_name='3.csv') + create_db_job(letter_template, job_status=JOB_STATUS_FINISHED, original_file_name='4.csv') + create_db_job(letter_template, job_status=JOB_STATUS_PENDING, original_file_name='5.csv') result = dao_get_letter_jobs_by_status(JOB_STATUS_READY_TO_SEND) assert len(result) == 2 From 62172ce39b56d351af197fc08bb99c546ad9f51c Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Tue, 22 Aug 2017 12:50:14 +0100 Subject: [PATCH 4/9] Updated schedule to run at 5:15pm --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 51285050a..75009a9e2 100644 --- a/app/config.py +++ b/app/config.py @@ -225,7 +225,7 @@ class Config(object): }, 'run-letter-jobs': { 'task': 'run-letter-jobs', - 'schedule': crontab(minute=0, hour=16), + 'schedule': crontab(minute=15, hour=17), 'options': {'queue': QueueNames.PERIODIC} } } From b2b74e2b5dd30eab51e97718df40710f7efbcb24 Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Tue, 22 Aug 2017 14:23:35 +0100 Subject: [PATCH 5/9] Fixed bug in letter job schedule --- app/celery/scheduled_tasks.py | 6 ++-- tests/app/celery/test_scheduled_tasks.py | 10 ++++--- tests/app/dao/test_jobs_dao.py | 37 ++++++++++++++++++------ 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index f95420f33..e40021d38 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -308,10 +308,10 @@ def populate_monthly_billing(): @notify_celery.task(name="run-letter-jobs") @statsd(namespace="tasks") def run_letter_jobs(): - ids = dao_get_letter_jobs_by_status(JOB_STATUS_READY_TO_SEND) + job_ids = [job.id for job in dao_get_letter_jobs_by_status(JOB_STATUS_READY_TO_SEND)] notify_celery.send_task( name=QueueNames.DVLA_FILES, - args=(ids), + args=(job_ids), queue=QueueNames.PROCESS_FTP ) - current_app.logger.info("Queued {} ready letter job ids onto {}".format(len(ids), QueueNames.PROCESS_FTP)) + current_app.logger.info("Queued {} ready letter job ids onto {}".format(len(job_ids), QueueNames.PROCESS_FTP)) diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index f2720a2cb..4d1be73ae 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -41,6 +41,7 @@ from app.dao.provider_details_dao import ( from app.models import ( Service, Template, SMS_TYPE, LETTER_TYPE, + JOB_STATUS_READY_TO_SEND, MonthlyBilling) from app.utils import get_london_midnight_in_utc, convert_utc_to_bst from tests.app.db import create_notification, create_service, create_template, create_job, create_rate @@ -683,16 +684,17 @@ def test_populate_monthly_billing_updates_correct_month_in_bst(sample_template): assert monthly_billing[1].monthly_totals[0]['total_cost'] == 0.0123 -def test_run_letter_jobs(client, mocker): - job_ids = [str(uuid.uuid4()), str(uuid.uuid4())] +def test_run_letter_jobs(client, mocker, sample_letter_template): + jobs = [create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND), + create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND)] mocker.patch( "app.celery.scheduled_tasks.dao_get_letter_jobs_by_status", - return_value=job_ids + return_value=jobs ) mock_celery = mocker.patch("app.celery.tasks.notify_celery.send_task") run_letter_jobs() mock_celery.assert_called_once_with(name=QueueNames.DVLA_FILES, - args=(job_ids), + args=([job.id for job in jobs]), queue=QueueNames.PROCESS_FTP) diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index 72f3ece04..ea09d5c18 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -34,6 +34,7 @@ from tests.app.conftest import sample_template as create_template from tests.app.db import ( create_user, create_job as create_db_job, + create_service as create_db_service, create_template as create_db_template ) @@ -551,16 +552,34 @@ def stats_set_up(notify_db, notify_db_session, service): def test_dao_get_letter_jobs_by_status(sample_service): - create_db_template(service=sample_service, template_type=SMS_TYPE) - create_db_template(service=sample_service, template_type=EMAIL_TYPE) + another_service = create_db_service(service_name="another service") + + sms_template = create_db_template(service=sample_service, template_type=SMS_TYPE) + email_template = create_db_template(service=sample_service, template_type=EMAIL_TYPE) letter_template = create_db_template(service=sample_service, template_type=LETTER_TYPE) - jobs = [] - jobs.append(create_db_job(letter_template, job_status=JOB_STATUS_READY_TO_SEND, original_file_name='1.csv')) - jobs.append(create_db_job(letter_template, job_status=JOB_STATUS_READY_TO_SEND, original_file_name='2.csv')) - create_db_job(letter_template, job_status=JOB_STATUS_SENT_TO_DVLA, original_file_name='3.csv') - create_db_job(letter_template, job_status=JOB_STATUS_FINISHED, original_file_name='4.csv') - create_db_job(letter_template, job_status=JOB_STATUS_PENDING, original_file_name='5.csv') + another_letter_template = create_db_template(service=another_service, template_type=LETTER_TYPE) + ready_letter_jobs = [] + ready_letter_jobs.append( + create_db_job( + letter_template, + job_status=JOB_STATUS_READY_TO_SEND, + original_file_name='1.csv' + ) + ) + ready_letter_jobs.append( + create_db_job( + another_letter_template, + job_status=JOB_STATUS_READY_TO_SEND, + original_file_name='2.csv' + ) + ) + create_db_job(sms_template, job_status=JOB_STATUS_FINISHED) + create_db_job(email_template, job_status=JOB_STATUS_FINISHED) + create_db_job(letter_template, job_status=JOB_STATUS_SENT_TO_DVLA) + create_db_job(letter_template, job_status=JOB_STATUS_FINISHED) + create_db_job(letter_template, job_status=JOB_STATUS_PENDING) + result = dao_get_letter_jobs_by_status(JOB_STATUS_READY_TO_SEND) assert len(result) == 2 - assert set(result) == set(jobs) + assert set(result) == set(ready_letter_jobs) From 41fce5793276f44e55b72c62dbd895c6b4125629 Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Tue, 22 Aug 2017 14:29:36 +0100 Subject: [PATCH 6/9] Removed PROCESS_FTP from all_queues --- app/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/config.py b/app/config.py index 75009a9e2..2451c952d 100644 --- a/app/config.py +++ b/app/config.py @@ -45,8 +45,6 @@ class QueueNames(object): QueueNames.JOBS, QueueNames.RETRY, QueueNames.NOTIFY, - QueueNames.PROCESS_FTP, - QueueNames.DVLA_FILES ] From 4fb5e68ce7f832658e7054b8075963cece0d4516 Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Tue, 22 Aug 2017 15:49:56 +0100 Subject: [PATCH 7/9] Refactor code - Created TaskNames for DVLA_FILES rather than have DVLA_FILES in QueueNames - Removed PROCESS_FTP from all_queues() as this was causing problems in picking up letter job tasks - Created test to ensure that we don't arbitrarily add queue names to all_queues --- app/celery/scheduled_tasks.py | 4 ++-- app/config.py | 5 ++++- app/letters/rest.py | 4 ++-- tests/app/celery/test_scheduled_tasks.py | 4 ++-- tests/app/test_config.py | 18 ++++++++++++++++++ 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index e40021d38..5cc35f257 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -37,7 +37,7 @@ from app.models import LETTER_TYPE, JOB_STATUS_READY_TO_SEND from app.notifications.process_notifications import send_notification_to_queue from app.statsd_decorators import statsd from app.celery.tasks import process_job -from app.config import QueueNames +from app.config import QueueNames, TaskNames from app.utils import convert_utc_to_bst @@ -310,7 +310,7 @@ def populate_monthly_billing(): def run_letter_jobs(): job_ids = [job.id for job in dao_get_letter_jobs_by_status(JOB_STATUS_READY_TO_SEND)] notify_celery.send_task( - name=QueueNames.DVLA_FILES, + name=TaskNames.DVLA_FILES, args=(job_ids), queue=QueueNames.PROCESS_FTP ) diff --git a/app/config.py b/app/config.py index 2451c952d..1cb389eb3 100644 --- a/app/config.py +++ b/app/config.py @@ -30,7 +30,6 @@ class QueueNames(object): RETRY = 'retry-tasks' NOTIFY = 'notify-internal-tasks' PROCESS_FTP = 'process-ftp-tasks' - DVLA_FILES = 'send-files-to-dvla' @staticmethod def all_queues(): @@ -48,6 +47,10 @@ class QueueNames(object): ] +class TaskNames(object): + DVLA_FILES = 'send-files-to-dvla' + + class Config(object): # URL of admin app ADMIN_BASE_URL = os.environ['ADMIN_BASE_URL'] diff --git a/app/letters/rest.py b/app/letters/rest.py index ce43ea169..b9ce7bf35 100644 --- a/app/letters/rest.py +++ b/app/letters/rest.py @@ -2,7 +2,7 @@ from flask import Blueprint, jsonify from flask import request from app import notify_celery -from app.config import QueueNames +from app.config import QueueNames, TaskNames from app.dao.jobs_dao import dao_get_all_letter_jobs from app.schemas import job_schema from app.v2.errors import register_errors @@ -16,7 +16,7 @@ register_errors(letter_job) @letter_job.route('/send-letter-jobs', methods=['POST']) def send_letter_jobs(): job_ids = validate(request.get_json(), letter_job_ids) - notify_celery.send_task(name=QueueNames.DVLA_FILES, args=(job_ids['job_ids'],), queue=QueueNames.PROCESS_FTP) + notify_celery.send_task(name=TaskNames.DVLA_FILES, args=(job_ids['job_ids'],), queue=QueueNames.PROCESS_FTP) return jsonify(data={"response": "Task created to send files to DVLA"}), 201 diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 4d1be73ae..78d46a54b 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -31,7 +31,7 @@ from app.celery.scheduled_tasks import ( timeout_notifications, populate_monthly_billing) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient -from app.config import QueueNames +from app.config import QueueNames, TaskNames from app.dao.jobs_dao import dao_get_job_by_id from app.dao.notifications_dao import dao_get_scheduled_notifications from app.dao.provider_details_dao import ( @@ -695,6 +695,6 @@ def test_run_letter_jobs(client, mocker, sample_letter_template): run_letter_jobs() - mock_celery.assert_called_once_with(name=QueueNames.DVLA_FILES, + mock_celery.assert_called_once_with(name=TaskNames.DVLA_FILES, args=([job.id for job in jobs]), queue=QueueNames.PROCESS_FTP) diff --git a/tests/app/test_config.py b/tests/app/test_config.py index c1e1cbeee..fb251307b 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -5,6 +5,7 @@ from unittest import mock import pytest from app import config +from app.config import QueueNames def cf_conf(): @@ -57,3 +58,20 @@ def test_load_config_if_cloudfoundry_not_available(monkeypatch, reload_config): def test_cloudfoundry_config_has_different_defaults(): # these should always be set on Sandbox assert config.Sandbox.REDIS_ENABLED is False + + +def test_queue_names_all_queues_correct(): + # Need to ensure that all_queues() only returns queue names used in API + queues = QueueNames.all_queues() + assert len(queues) == 10 + assert set([QueueNames.PRIORITY, + QueueNames.PERIODIC, + QueueNames.DATABASE, + QueueNames.SEND_SMS, + QueueNames.SEND_EMAIL, + QueueNames.RESEARCH_MODE, + QueueNames.STATISTICS, + QueueNames.JOBS, + QueueNames.RETRY, + QueueNames.NOTIFY + ]) == set(queues) From 665420402ee41b2aad87d72784fb092b656a857f Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Tue, 22 Aug 2017 16:26:07 +0100 Subject: [PATCH 8/9] Improved code style --- tests/app/test_config.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/app/test_config.py b/tests/app/test_config.py index fb251307b..3a40b4db4 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -64,14 +64,15 @@ def test_queue_names_all_queues_correct(): # Need to ensure that all_queues() only returns queue names used in API queues = QueueNames.all_queues() assert len(queues) == 10 - assert set([QueueNames.PRIORITY, - QueueNames.PERIODIC, - QueueNames.DATABASE, - QueueNames.SEND_SMS, - QueueNames.SEND_EMAIL, - QueueNames.RESEARCH_MODE, - QueueNames.STATISTICS, - QueueNames.JOBS, - QueueNames.RETRY, - QueueNames.NOTIFY - ]) == set(queues) + assert set([ + QueueNames.PRIORITY, + QueueNames.PERIODIC, + QueueNames.DATABASE, + QueueNames.SEND_SMS, + QueueNames.SEND_EMAIL, + QueueNames.RESEARCH_MODE, + QueueNames.STATISTICS, + QueueNames.JOBS, + QueueNames.RETRY, + QueueNames.NOTIFY + ]) == set(queues) From c3ce8286359c975438d233fb591cd6be07bf5f6b Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Wed, 23 Aug 2017 12:13:07 +0100 Subject: [PATCH 9/9] Updated letter jobs schedule to 5:30pm --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 1cb389eb3..958a0a532 100644 --- a/app/config.py +++ b/app/config.py @@ -226,7 +226,7 @@ class Config(object): }, 'run-letter-jobs': { 'task': 'run-letter-jobs', - 'schedule': crontab(minute=15, hour=17), + 'schedule': crontab(minute=30, hour=17), 'options': {'queue': QueueNames.PERIODIC} } }