From 29777c31233b87114a7efcc41c257ee3f91f9494 Mon Sep 17 00:00:00 2001 From: Alexey Bezhan Date: Wed, 17 Jan 2018 14:43:28 +0000 Subject: [PATCH] Add a celery scheduled task to alert if letters are still sending We should receive a response file from DVLA by 4pm the next working day (next Monday for letters created on Friday, Saturday or Sunday). Response file triggers a task to update the letters status from 'sending' to either 'failed' or 'delivered', at which point there should be no letter notifications in the 'sending' state for that day. To catch any errors in the process (eg a missing response file from DVLA) we add a scheduled task that checks letter notifications for previous day (or Friday when run on Monday) and raises a Deskpro ticket if it finds any in a 'sending' state. We're checking letter notifications based on the `sent_at` date, which is set when the letter PDF is sent to DVLA (so for letters created after 5:30pm it will be the next day). The task runs at 4:30pm, which should give the response file processing task enough time to finish if the file was uploaded at 4pm. --- app/celery/scheduled_tasks.py | 36 ++++++++++- tests/app/celery/test_scheduled_tasks.py | 78 ++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index e5b0fbf55..6ebaec87a 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -17,7 +17,7 @@ from app.dao.services_dao import ( ) from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template from app.performance_platform import total_sent_notifications, processing_time -from app import performance_platform_client +from app import performance_platform_client, deskpro_client from app.dao.date_util import get_month_start_and_end_date_in_utc from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago @@ -47,6 +47,8 @@ from app.dao.provider_details_dao import ( from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago from app.models import ( Job, + Notification, + NOTIFICATION_SENDING, LETTER_TYPE, JOB_STATUS_IN_PROGRESS, JOB_STATUS_READY_TO_SEND @@ -332,6 +334,38 @@ def delete_dvla_response_files_older_than_seven_days(): raise +@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending") +@statsd(namespace="tasks") +def raise_alert_if_letter_notifications_still_sending(): + today = datetime.utcnow().date() + + # Do nothing on the weekend + if today.isoweekday() in [6, 7]: + return + + if today.isoweekday() == 1: + offset_days = 3 + else: + offset_days = 1 + + still_sending = Notification.query.filter( + Notification.notification_type == LETTER_TYPE, + Notification.status == NOTIFICATION_SENDING, + Notification.sent_at >= today - timedelta(days=offset_days), + Notification.sent_at < today + ).count() + + if still_sending: + deskpro_client.create_ticket( + subject="Letters still sending", + message="There are {} letters in the 'sending' state from {}".format( + still_sending, + (today - timedelta(days=offset_days)).strftime('%A %d %B') + ), + ticket_type="alert" + ) + + @notify_celery.task(name="populate_monthly_billing") @statsd(namespace="tasks") def populate_monthly_billing(): diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 7178d8681..0eb25f2a9 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -20,6 +20,7 @@ from app.celery.scheduled_tasks import ( delete_letter_notifications_older_than_seven_days, delete_sms_notifications_older_than_seven_days, delete_verify_codes, + raise_alert_if_letter_notifications_still_sending, remove_csv_files, remove_transformed_dvla_files, run_scheduled_jobs, @@ -631,6 +632,83 @@ def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files( remove_s3_mock.assert_not_called() +@freeze_time("2018-01-17 17:00:00") +def test_alert_if_letter_notifications_still_sending(sample_letter_template, mocker): + yesterday = datetime(2018, 1, 16, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_celery.assert_called_once_with( + subject="Letters still sending", + message="There are 1 letters in the 'sending' state from Tuesday 16 January", + ticket_type='alert' + ) + + +@freeze_time("2018-01-17 17:00:00") +def test_alert_if_letter_notifications_still_sending_only_alerts_sending(sample_letter_template, mocker): + yesterday = datetime(2018, 1, 16, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + create_notification(template=sample_letter_template, status='delivered', sent_at=yesterday) + create_notification(template=sample_letter_template, status='failed', sent_at=yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_celery.assert_called_once_with( + subject="Letters still sending", + message="There are 1 letters in the 'sending' state from Tuesday 16 January", + ticket_type='alert' + ) + + +@freeze_time("2018-01-17 17:00:00") +def test_alert_if_letter_notifications_still_sending_only_alerts_previous_day(sample_letter_template, mocker): + day_before_yesterday = datetime(2018, 1, 15, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=day_before_yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + assert not mock_celery.called + + +@freeze_time("2018-01-14 17:00:00") +def test_alert_if_letter_notifications_still_sending_does_nothing_on_the_weekend(sample_letter_template, mocker): + yesterday = datetime(2018, 1, 13, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + assert not mock_celery.called + + +@freeze_time("2018-01-15 17:00:00") +def test_monday_alert_if_letter_notifications_still_sending_reports_friday_letters(sample_letter_template, mocker): + friday = datetime(2018, 1, 12, 13, 30) + yesterday = datetime(2018, 1, 14, 13, 30) + + create_notification(template=sample_letter_template, status='sending', sent_at=friday) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_celery.assert_called_once_with( + subject="Letters still sending", + message="There are 2 letters in the 'sending' state from Friday 12 January", + ticket_type='alert' + ) + + @freeze_time("2017-07-12 02:00:00") def test_populate_monthly_billing_populates_correctly(sample_template): yesterday = datetime(2017, 7, 11, 13, 30)