From c518f6ca76f3f7c47d8ec1f6e4cac13d340e116c Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Tue, 11 Jun 2019 15:13:06 +0100 Subject: [PATCH] Add scheduled task to find old letters which still have 'created' status Added a scheduled task to run once a day and check if there were any letters from before 17.30 that still have a status of 'created'. This logs an exception instead of trying to fix the error because the fix will be different depending on which bucket the letter is in. --- app/celery/scheduled_tasks.py | 15 +++++++++ app/config.py | 5 +++ app/dao/notifications_dao.py | 16 +++++++++- tests/app/celery/test_scheduled_tasks.py | 39 ++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index ad955f12c..ebef57b40 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -21,6 +21,7 @@ from app.dao.notifications_dao import ( set_scheduled_notification_to_processed, notifications_not_yet_sent, dao_precompiled_letters_still_pending_virus_check, + dao_old_letters_with_created_status, ) from app.dao.provider_details_dao import ( get_current_provider, @@ -193,3 +194,17 @@ def check_precompiled_letter_state(): len(letters), letter_ids) ) + + +@notify_celery.task(name='check-templated-letter-state') +@statsd(namespace="tasks") +def check_templated_letter_state(): + letters = dao_old_letters_with_created_status() + letter_ids = [str(letter.id) for letter in letters] + + if len(letters) > 0: + current_app.logger.exception( + "{} letters were created before 17.30 yesterday and still have 'created' state. Notifications: {}".format( + len(letters), + letter_ids) + ) diff --git a/app/config.py b/app/config.py index 3690944a9..2c5ee54ae 100644 --- a/app/config.py +++ b/app/config.py @@ -259,6 +259,11 @@ class Config(object): # since we mark jobs as archived 'options': {'queue': QueueNames.PERIODIC}, }, + 'check-templated-letter-state': { + 'task': 'check-templated-letter-state', + 'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0), + 'options': {'queue', QueueNames.PERIODIC} + }, 'raise-alert-if-letter-notifications-still-sending': { 'task': 'raise-alert-if-letter-notifications-still-sending', 'schedule': crontab(hour=16, minute=30), diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 069a8ea2a..882189ae3 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -14,7 +14,7 @@ from notifications_utils.recipients import ( try_validate_and_format_phone_number ) from notifications_utils.statsd_decorators import statsd -from notifications_utils.timezones import convert_utc_to_bst +from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst from sqlalchemy import (desc, func, asc) from sqlalchemy.orm import joinedload from sqlalchemy.orm.exc import NoResultFound @@ -692,6 +692,20 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ return notifications +def dao_old_letters_with_created_status(): + yesterday_bst = convert_utc_to_bst(datetime.utcnow()) - timedelta(days=1) + last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0) + + notifications = Notification.query.filter( + Notification.created_at < convert_bst_to_utc(last_processing_deadline), + Notification.notification_type == LETTER_TYPE, + Notification.status == NOTIFICATION_CREATED + ).order_by( + Notification.created_at + ).all() + return notifications + + def dao_precompiled_letters_still_pending_virus_check(): ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400) diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 95eb7498f..0b57b489f 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -15,6 +15,7 @@ from app.celery.scheduled_tasks import ( switch_current_sms_provider_on_slow_delivery, replay_created_notifications, check_precompiled_letter_state, + check_templated_letter_state, ) from app.config import QueueNames, TaskNames from app.dao.jobs_dao import dao_get_job_by_id @@ -363,3 +364,41 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template): noti_2.id, noti_1.id) ) + + +@freeze_time("2019-05-30 14:00:00") +def test_check_templated_letter_state_during_bst(mocker, sample_letter_template): + mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception') + + noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0)) + noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 30)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 17, 29)) + create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0)) + + check_templated_letter_state() + + mock_logger.assert_called_once_with( + "2 letters were created before 17.30 yesterday and still have 'created' state. " + "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id) + ) + + +@freeze_time("2019-01-30 14:00:00") +def test_check_templated_letter_state_during_utc(mocker, sample_letter_template): + mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception') + + noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0)) + noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 30)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 18, 29)) + create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0)) + + check_templated_letter_state() + + mock_logger.assert_called_once_with( + "2 letters were created before 17.30 yesterday and still have 'created' state. " + "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id) + )