diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index e5b0fbf55..6ebaec87a 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -17,7 +17,7 @@ from app.dao.services_dao import ( ) from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template from app.performance_platform import total_sent_notifications, processing_time -from app import performance_platform_client +from app import performance_platform_client, deskpro_client from app.dao.date_util import get_month_start_and_end_date_in_utc from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago @@ -47,6 +47,8 @@ from app.dao.provider_details_dao import ( from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago from app.models import ( Job, + Notification, + NOTIFICATION_SENDING, LETTER_TYPE, JOB_STATUS_IN_PROGRESS, JOB_STATUS_READY_TO_SEND @@ -332,6 +334,38 @@ def delete_dvla_response_files_older_than_seven_days(): raise +@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending") +@statsd(namespace="tasks") +def raise_alert_if_letter_notifications_still_sending(): + today = datetime.utcnow().date() + + # Do nothing on the weekend + if today.isoweekday() in [6, 7]: + return + + if today.isoweekday() == 1: + offset_days = 3 + else: + offset_days = 1 + + still_sending = Notification.query.filter( + Notification.notification_type == LETTER_TYPE, + Notification.status == NOTIFICATION_SENDING, + Notification.sent_at >= today - timedelta(days=offset_days), + Notification.sent_at < today + ).count() + + if still_sending: + deskpro_client.create_ticket( + subject="Letters still sending", + message="There are {} letters in the 'sending' state from {}".format( + still_sending, + (today - timedelta(days=offset_days)).strftime('%A %d %B') + ), + ticket_type="alert" + ) + + @notify_celery.task(name="populate_monthly_billing") @statsd(namespace="tasks") def populate_monthly_billing(): diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 7178d8681..0eb25f2a9 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -20,6 +20,7 @@ from app.celery.scheduled_tasks import ( delete_letter_notifications_older_than_seven_days, delete_sms_notifications_older_than_seven_days, delete_verify_codes, + raise_alert_if_letter_notifications_still_sending, remove_csv_files, remove_transformed_dvla_files, run_scheduled_jobs, @@ -631,6 +632,83 @@ def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files( remove_s3_mock.assert_not_called() +@freeze_time("2018-01-17 17:00:00") +def test_alert_if_letter_notifications_still_sending(sample_letter_template, mocker): + yesterday = datetime(2018, 1, 16, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_celery.assert_called_once_with( + subject="Letters still sending", + message="There are 1 letters in the 'sending' state from Tuesday 16 January", + ticket_type='alert' + ) + + +@freeze_time("2018-01-17 17:00:00") +def test_alert_if_letter_notifications_still_sending_only_alerts_sending(sample_letter_template, mocker): + yesterday = datetime(2018, 1, 16, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + create_notification(template=sample_letter_template, status='delivered', sent_at=yesterday) + create_notification(template=sample_letter_template, status='failed', sent_at=yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_celery.assert_called_once_with( + subject="Letters still sending", + message="There are 1 letters in the 'sending' state from Tuesday 16 January", + ticket_type='alert' + ) + + +@freeze_time("2018-01-17 17:00:00") +def test_alert_if_letter_notifications_still_sending_only_alerts_previous_day(sample_letter_template, mocker): + day_before_yesterday = datetime(2018, 1, 15, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=day_before_yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + assert not mock_celery.called + + +@freeze_time("2018-01-14 17:00:00") +def test_alert_if_letter_notifications_still_sending_does_nothing_on_the_weekend(sample_letter_template, mocker): + yesterday = datetime(2018, 1, 13, 13, 30) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + assert not mock_celery.called + + +@freeze_time("2018-01-15 17:00:00") +def test_monday_alert_if_letter_notifications_still_sending_reports_friday_letters(sample_letter_template, mocker): + friday = datetime(2018, 1, 12, 13, 30) + yesterday = datetime(2018, 1, 14, 13, 30) + + create_notification(template=sample_letter_template, status='sending', sent_at=friday) + create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) + + mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket") + + raise_alert_if_letter_notifications_still_sending() + + mock_celery.assert_called_once_with( + subject="Letters still sending", + message="There are 2 letters in the 'sending' state from Friday 12 January", + ticket_type='alert' + ) + + @freeze_time("2017-07-12 02:00:00") def test_populate_monthly_billing_populates_correctly(sample_template): yesterday = datetime(2017, 7, 11, 13, 30)