Add scheduled task to find precompiled letters in wrong state

Added a task which runs twice a day on weekdays and checks for letters that have
been in the state of `pending-virus-check` for over 90 minutes. This is
just logging an exception for now, not trying to fix things, since we
will need to manually check where the issue was.

app/celery/scheduled_tasks.py

@@ -19,7 +19,8 @@ from app.dao.notifications_dao import (
     is_delivery_slow_for_provider,
     dao_get_scheduled_notifications,
     set_scheduled_notification_to_processed,
-    notifications_not_yet_sent
+    notifications_not_yet_sent,
+    dao_precompiled_letters_still_pending_virus_check,
 )
 from app.dao.provider_details_dao import (
     get_current_provider,
@@ -178,3 +179,17 @@ def replay_created_notifications():
 
         for n in notifications_to_resend:
             send_notification_to_queue(notification=n, research_mode=n.service.research_mode)
+
+
+@notify_celery.task(name='check-precompiled-letter-state')
+@statsd(namespace="tasks")
+def check_precompiled_letter_state():
+    letters = dao_precompiled_letters_still_pending_virus_check()
+    letter_ids = [str(letter.id) for letter in letters]
+
+    if len(letters) > 0:
+        current_app.logger.exception(
+            '{} precompiled letters have been pending-virus-check for over 90 minutes. Notifications: {}'.format(
+                len(letters),
+                letter_ids)
+        )

app/config.py

@@ -276,6 +276,11 @@ class Config(object):
             'schedule': crontab(hour=23, minute=00),
             'options': {'queue': QueueNames.PERIODIC}
         },
+        'check-precompiled-letter-state': {
+            'task': 'check-precompiled-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
+            'options': {'queue', QueueNames.PERIODIC}
+        },
     }
     CELERY_QUEUES = []
 

app/dao/notifications_dao.py

@@ -692,6 +692,18 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
+def dao_precompiled_letters_still_pending_virus_check():
+    ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400)
+
+    notifications = Notification.query.filter(
+        Notification.created_at < ninety_minutes_ago,
+        Notification.status == NOTIFICATION_PENDING_VIRUS_CHECK
+    ).order_by(
+        Notification.created_at
+    ).all()
+    return notifications
+
+
 def guess_notification_type(search_term):
     if set(search_term) & set(string.ascii_letters + '@'):
         return EMAIL_TYPE

tests/app/celery/test_scheduled_tasks.py

@@ -13,7 +13,8 @@ from app.celery.scheduled_tasks import (
     run_scheduled_jobs,
     send_scheduled_notifications,
     switch_current_sms_provider_on_slow_delivery,
-    replay_created_notifications
+    replay_created_notifications,
+    check_precompiled_letter_state,
 )
 from app.config import QueueNames, TaskNames
 from app.dao.jobs_dao import dao_get_job_by_id
@@ -26,6 +27,8 @@ from app.models import (
     JOB_STATUS_IN_PROGRESS,
     JOB_STATUS_ERROR,
     JOB_STATUS_FINISHED,
+    NOTIFICATION_DELIVERED,
+    NOTIFICATION_PENDING_VIRUS_CHECK,
 )
 from app.v2.errors import JobIncompleteError
 
@@ -334,3 +337,29 @@ def test_check_job_status_task_does_not_raise_error(sample_template):
         job_status=JOB_STATUS_FINISHED)
 
     check_job_status()
+
+
+@freeze_time("2019-05-30 14:00:00")
+def test_check_precompiled_letter_state(mocker, sample_letter_template):
+    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
+
+    create_notification(template=sample_letter_template,
+                        status=NOTIFICATION_PENDING_VIRUS_CHECK,
+                        created_at=datetime.utcnow() - timedelta(seconds=5400))
+    create_notification(template=sample_letter_template,
+                        status=NOTIFICATION_DELIVERED,
+                        created_at=datetime.utcnow() - timedelta(seconds=6000))
+    noti_1 = create_notification(template=sample_letter_template,
+                                 status=NOTIFICATION_PENDING_VIRUS_CHECK,
+                                 created_at=datetime.utcnow() - timedelta(seconds=5401))
+    noti_2 = create_notification(template=sample_letter_template,
+                                 status=NOTIFICATION_PENDING_VIRUS_CHECK,
+                                 created_at=datetime.utcnow() - timedelta(seconds=70000))
+
+    check_precompiled_letter_state()
+
+    mock_logger.assert_called_once_with(
+        "2 precompiled letters have been pending-virus-check for over 90 minutes. Notifications: ['{}', '{}']".format(
+            noti_2.id,
+            noti_1.id)
+    )