Add a celery scheduled task to alert if letters are still sending

We should receive a response file from DVLA by 4pm the next working
day (next Monday for letters created on Friday, Saturday or Sunday).

Response file triggers a task to update the letters status from
'sending' to either 'failed' or 'delivered', at which point there
should be no letter notifications in the 'sending' state for that day.

To catch any errors in the process (eg a missing response file from
DVLA) we add a scheduled task that checks letter notifications for
previous day (or Friday when run on Monday) and raises a Deskpro
ticket if it finds any in a 'sending' state. We're checking letter
notifications based on the `sent_at` date, which is set when the
letter PDF is sent to DVLA (so for letters created after 5:30pm it
will be the next day).

The task runs at 4:30pm, which should give the response file processing
task enough time to finish if the file was uploaded at 4pm.

app/celery/scheduled_tasks.py

@@ -17,7 +17,7 @@ from app.dao.services_dao import (
 )
 from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template
 from app.performance_platform import total_sent_notifications, processing_time
-from app import performance_platform_client
+from app import performance_platform_client, deskpro_client
 from app.dao.date_util import get_month_start_and_end_date_in_utc
 from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
@@ -47,6 +47,8 @@ from app.dao.provider_details_dao import (
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
 from app.models import (
     Job,
+    Notification,
+    NOTIFICATION_SENDING,
     LETTER_TYPE,
     JOB_STATUS_IN_PROGRESS,
     JOB_STATUS_READY_TO_SEND
@@ -332,6 +334,38 @@ def delete_dvla_response_files_older_than_seven_days():
         raise
 
 
+@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
+@statsd(namespace="tasks")
+def raise_alert_if_letter_notifications_still_sending():
+    today = datetime.utcnow().date()
+
+    # Do nothing on the weekend
+    if today.isoweekday() in [6, 7]:
+        return
+
+    if today.isoweekday() == 1:
+        offset_days = 3
+    else:
+        offset_days = 1
+
+    still_sending = Notification.query.filter(
+        Notification.notification_type == LETTER_TYPE,
+        Notification.status == NOTIFICATION_SENDING,
+        Notification.sent_at >= today - timedelta(days=offset_days),
+        Notification.sent_at < today
+    ).count()
+
+    if still_sending:
+        deskpro_client.create_ticket(
+            subject="Letters still sending",
+            message="There are {} letters in the 'sending' state from {}".format(
+                still_sending,
+                (today - timedelta(days=offset_days)).strftime('%A %d %B')
+            ),
+            ticket_type="alert"
+        )
+
+
 @notify_celery.task(name="populate_monthly_billing")
 @statsd(namespace="tasks")
 def populate_monthly_billing():

tests/app/celery/test_scheduled_tasks.py

@@ -20,6 +20,7 @@ from app.celery.scheduled_tasks import (
     delete_letter_notifications_older_than_seven_days,
     delete_sms_notifications_older_than_seven_days,
     delete_verify_codes,
+    raise_alert_if_letter_notifications_still_sending,
     remove_csv_files,
     remove_transformed_dvla_files,
     run_scheduled_jobs,
@@ -631,6 +632,83 @@ def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files(
     remove_s3_mock.assert_not_called()
 
 
+@freeze_time("2018-01-17 17:00:00")
+def test_alert_if_letter_notifications_still_sending(sample_letter_template, mocker):
+    yesterday = datetime(2018, 1, 16, 13, 30)
+    create_notification(template=sample_letter_template, status='sending', sent_at=yesterday)
+
+    mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket")
+
+    raise_alert_if_letter_notifications_still_sending()
+
+    mock_celery.assert_called_once_with(
+        subject="Letters still sending",
+        message="There are 1 letters in the 'sending' state from Tuesday 16 January",
+        ticket_type='alert'
+    )
+
+
+@freeze_time("2018-01-17 17:00:00")
+def test_alert_if_letter_notifications_still_sending_only_alerts_sending(sample_letter_template, mocker):
+    yesterday = datetime(2018, 1, 16, 13, 30)
+    create_notification(template=sample_letter_template, status='sending', sent_at=yesterday)
+    create_notification(template=sample_letter_template, status='delivered', sent_at=yesterday)
+    create_notification(template=sample_letter_template, status='failed', sent_at=yesterday)
+
+    mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket")
+
+    raise_alert_if_letter_notifications_still_sending()
+
+    mock_celery.assert_called_once_with(
+        subject="Letters still sending",
+        message="There are 1 letters in the 'sending' state from Tuesday 16 January",
+        ticket_type='alert'
+    )
+
+
+@freeze_time("2018-01-17 17:00:00")
+def test_alert_if_letter_notifications_still_sending_only_alerts_previous_day(sample_letter_template, mocker):
+    day_before_yesterday = datetime(2018, 1, 15, 13, 30)
+    create_notification(template=sample_letter_template, status='sending', sent_at=day_before_yesterday)
+
+    mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket")
+
+    raise_alert_if_letter_notifications_still_sending()
+
+    assert not mock_celery.called
+
+
+@freeze_time("2018-01-14 17:00:00")
+def test_alert_if_letter_notifications_still_sending_does_nothing_on_the_weekend(sample_letter_template, mocker):
+    yesterday = datetime(2018, 1, 13, 13, 30)
+    create_notification(template=sample_letter_template, status='sending', sent_at=yesterday)
+
+    mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket")
+
+    raise_alert_if_letter_notifications_still_sending()
+
+    assert not mock_celery.called
+
+
+@freeze_time("2018-01-15 17:00:00")
+def test_monday_alert_if_letter_notifications_still_sending_reports_friday_letters(sample_letter_template, mocker):
+    friday = datetime(2018, 1, 12, 13, 30)
+    yesterday = datetime(2018, 1, 14, 13, 30)
+
+    create_notification(template=sample_letter_template, status='sending', sent_at=friday)
+    create_notification(template=sample_letter_template, status='sending', sent_at=yesterday)
+
+    mock_celery = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket")
+
+    raise_alert_if_letter_notifications_still_sending()
+
+    mock_celery.assert_called_once_with(
+        subject="Letters still sending",
+        message="There are 2 letters in the 'sending' state from Friday 12 January",
+        ticket_type='alert'
+    )
+
+
 @freeze_time("2017-07-12 02:00:00")
 def test_populate_monthly_billing_populates_correctly(sample_template):
     yesterday = datetime(2017, 7, 11, 13, 30)