Add scheduled task to find old letters which still have 'created' status

Added a scheduled task to run once a day and check if there were any
letters from before 17.30 that still have a status of 'created'. This
logs an exception instead of trying to fix the error because the fix
will be different depending on which bucket the letter is in.

app/celery/scheduled_tasks.py

@@ -21,6 +21,7 @@ from app.dao.notifications_dao import (
     set_scheduled_notification_to_processed,
     notifications_not_yet_sent,
     dao_precompiled_letters_still_pending_virus_check,
+    dao_old_letters_with_created_status,
 )
 from app.dao.provider_details_dao import (
     get_current_provider,
@@ -193,3 +194,17 @@ def check_precompiled_letter_state():
                 len(letters),
                 letter_ids)
         )
+
+
+@notify_celery.task(name='check-templated-letter-state')
+@statsd(namespace="tasks")
+def check_templated_letter_state():
+    letters = dao_old_letters_with_created_status()
+    letter_ids = [str(letter.id) for letter in letters]
+
+    if len(letters) > 0:
+        current_app.logger.exception(
+            "{} letters were created before 17.30 yesterday and still have 'created' state. Notifications: {}".format(
+                len(letters),
+                letter_ids)
+        )

app/config.py

@@ -259,6 +259,11 @@ class Config(object):
             # since we mark jobs as archived
             'options': {'queue': QueueNames.PERIODIC},
         },
+        'check-templated-letter-state': {
+            'task': 'check-templated-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0),
+            'options': {'queue', QueueNames.PERIODIC}
+        },
         'raise-alert-if-letter-notifications-still-sending': {
             'task': 'raise-alert-if-letter-notifications-still-sending',
             'schedule': crontab(hour=16, minute=30),

app/dao/notifications_dao.py

@@ -14,7 +14,7 @@ from notifications_utils.recipients import (
     try_validate_and_format_phone_number
 )
 from notifications_utils.statsd_decorators import statsd
-from notifications_utils.timezones import convert_utc_to_bst
+from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
 from sqlalchemy import (desc, func, asc)
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.exc import NoResultFound
@@ -692,6 +692,20 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
+def dao_old_letters_with_created_status():
+    yesterday_bst = convert_utc_to_bst(datetime.utcnow()) - timedelta(days=1)
+    last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0)
+
+    notifications = Notification.query.filter(
+        Notification.created_at < convert_bst_to_utc(last_processing_deadline),
+        Notification.notification_type == LETTER_TYPE,
+        Notification.status == NOTIFICATION_CREATED
+    ).order_by(
+        Notification.created_at
+    ).all()
+    return notifications
+
+
 def dao_precompiled_letters_still_pending_virus_check():
     ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400)
 

tests/app/celery/test_scheduled_tasks.py

@@ -15,6 +15,7 @@ from app.celery.scheduled_tasks import (
     switch_current_sms_provider_on_slow_delivery,
     replay_created_notifications,
     check_precompiled_letter_state,
+    check_templated_letter_state,
 )
 from app.config import QueueNames, TaskNames
 from app.dao.jobs_dao import dao_get_job_by_id
@@ -363,3 +364,41 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template):
             noti_2.id,
             noti_1.id)
     )
+
+
+@freeze_time("2019-05-30 14:00:00")
+def test_check_templated_letter_state_during_bst(mocker, sample_letter_template):
+    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
+
+    noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0))
+    noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 30))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 17, 29))
+    create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0))
+
+    check_templated_letter_state()
+
+    mock_logger.assert_called_once_with(
+        "2 letters were created before 17.30 yesterday and still have 'created' state. "
+        "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
+    )
+
+
+@freeze_time("2019-01-30 14:00:00")
+def test_check_templated_letter_state_during_utc(mocker, sample_letter_template):
+    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
+
+    noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0))
+    noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 30))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 18, 29))
+    create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0))
+
+    check_templated_letter_state()
+
+    mock_logger.assert_called_once_with(
+        "2 letters were created before 17.30 yesterday and still have 'created' state. "
+        "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
+    )