Add scheduled task to find precompiled letters in wrong state

Added a task which runs twice a day on weekdays and checks for letters that have
been in the state of `pending-virus-check` for over 90 minutes. This is
just logging an exception for now, not trying to fix things, since we
will need to manually check where the issue was.

app/celery/scheduled_tasks.py

@@ -19,7 +19,8 @@ from app.dao.notifications_dao import (
     is_delivery_slow_for_provider,
     dao_get_scheduled_notifications,
     set_scheduled_notification_to_processed,
-    notifications_not_yet_sent
+    notifications_not_yet_sent,
+    dao_precompiled_letters_still_pending_virus_check,
 )
 from app.dao.provider_details_dao import (
     get_current_provider,
@@ -178,3 +179,17 @@ def replay_created_notifications():
 
         for n in notifications_to_resend:
             send_notification_to_queue(notification=n, research_mode=n.service.research_mode)
+
+
+@notify_celery.task(name='check-precompiled-letter-state')
+@statsd(namespace="tasks")
+def check_precompiled_letter_state():
+    letters = dao_precompiled_letters_still_pending_virus_check()
+    letter_ids = [str(letter.id) for letter in letters]
+
+    if len(letters) > 0:
+        current_app.logger.exception(
+            '{} precompiled letters have been pending-virus-check for over 90 minutes. Notifications: {}'.format(
+                len(letters),
+                letter_ids)
+        )

app/config.py

@@ -276,6 +276,11 @@ class Config(object):
             'schedule': crontab(hour=23, minute=00),
             'options': {'queue': QueueNames.PERIODIC}
         },
+        'check-precompiled-letter-state': {
+            'task': 'check-precompiled-letter-state',
+            'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
+            'options': {'queue', QueueNames.PERIODIC}
+        },
     }
     CELERY_QUEUES = []
 

app/dao/notifications_dao.py

@@ -692,6 +692,18 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
+def dao_precompiled_letters_still_pending_virus_check():
+    ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400)
+
+    notifications = Notification.query.filter(
+        Notification.created_at < ninety_minutes_ago,
+        Notification.status == NOTIFICATION_PENDING_VIRUS_CHECK
+    ).order_by(
+        Notification.created_at
+    ).all()
+    return notifications
+
+
 def guess_notification_type(search_term):
     if set(search_term) & set(string.ascii_letters + '@'):
         return EMAIL_TYPE