Merge pull request #2658 from alphagov/fix-letters-in-created-status

Alert if a letter doesn't make it past created status

app/celery/scheduled_tasks.py

@@ -14,6 +14,7 @@ from app.celery.tasks import (
     get_recipient_csv_and_template_and_sender_id,
     process_row
 )
+from app.celery.letters_pdf_tasks import create_letters_pdf
 from app.config import QueueNames, TaskNames
 from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
@@ -30,6 +31,7 @@ from app.dao.notifications_dao import (
     notifications_not_yet_sent,
     dao_precompiled_letters_still_pending_virus_check,
     dao_old_letters_with_created_status,
+    letters_missing_from_sending_bucket
 )
 from app.dao.provider_details_dao import (
     get_current_provider,
@@ -173,8 +175,8 @@ def check_job_status():
 @notify_celery.task(name='replay-created-notifications')
 @statsd(namespace="tasks")
 def replay_created_notifications():
-    # if the notification has not be send after 4 hours + 15 minutes, then try to resend.
-    resend_created_notifications_older_than = (60 * 60 * 4) + (60 * 15)
+    # if the notification has not be send after 1 hour, then try to resend.
+    resend_created_notifications_older_than = (60 * 60)
     for notification_type in (EMAIL_TYPE, SMS_TYPE):
         notifications_to_resend = notifications_not_yet_sent(
             resend_created_notifications_older_than,
@@ -189,6 +191,20 @@ def replay_created_notifications():
         for n in notifications_to_resend:
             send_notification_to_queue(notification=n, research_mode=n.service.research_mode)
 
+    # if the letter has not be send after an hour, then create a zendesk ticket
+    letters = letters_missing_from_sending_bucket(resend_created_notifications_older_than)
+
+    if len(letters) > 0:
+        msg = "{} letters were created over an hour ago, " \
+              "but do not have an updated_at timestamp or billable units. " \
+              "\n Creating app.celery.letters_pdf_tasks.create_letters tasks to upload letter to S3 " \
+              "and update notifications for the following notification ids: " \
+              "\n {}".format(len(letters), [x.id for x in letters])
+
+        current_app.logger.info(msg)
+        for letter in letters:
+            create_letters_pdf.apply_async([letter.id], queue=QueueNames.LETTERS)
+
 
 @notify_celery.task(name='check-precompiled-letter-state')
 @statsd(namespace="tasks")

app/dao/notifications_dao.py

@@ -32,6 +32,7 @@ from app.models import (
     Notification,
     NotificationHistory,
     ScheduledNotification,
+    KEY_TYPE_NORMAL,
     KEY_TYPE_TEST,
     LETTER_TYPE,
     NOTIFICATION_CREATED,
@@ -46,7 +47,7 @@ from app.models import (
     SMS_TYPE,
     EMAIL_TYPE,
     ServiceDataRetention,
-    Service
+    Service,
 )
 from app.utils import get_london_midnight_in_utc
 from app.utils import midnight_n_days_ago, escape_special_characters
@@ -698,15 +699,32 @@ def dao_old_letters_with_created_status():
     last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0)
 
     notifications = Notification.query.filter(
-        Notification.updated_at < convert_bst_to_utc(last_processing_deadline),
+        Notification.created_at < convert_bst_to_utc(last_processing_deadline),
         Notification.notification_type == LETTER_TYPE,
         Notification.status == NOTIFICATION_CREATED
     ).order_by(
-        Notification.updated_at
+        Notification.created_at
     ).all()
     return notifications
 
 
+def letters_missing_from_sending_bucket(seconds_to_subtract):
+    older_than_date = datetime.utcnow() - timedelta(seconds=seconds_to_subtract)
+    # We expect letters to have a `created` status, updated_at timestamp and billable units greater than zero.
+    notifications = Notification.query.filter(
+        Notification.billable_units == 0,
+        Notification.updated_at == None,  # noqa
+        Notification.status == NOTIFICATION_CREATED,
+        Notification.created_at <= older_than_date,
+        Notification.notification_type == LETTER_TYPE,
+        Notification.key_type == KEY_TYPE_NORMAL
+    ).order_by(
+        Notification.created_at
+    ).all()
+
+    return notifications
+
+
 def dao_precompiled_letters_still_pending_virus_check():
     ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400)