Merge pull request #2658 from alphagov/fix-letters-in-created-status

Alert if a letter doesn't make it past created status

app/celery/scheduled_tasks.py

@@ -14,6 +14,7 @@ from app.celery.tasks import (
     get_recipient_csv_and_template_and_sender_id,
     process_row
 )
+from app.celery.letters_pdf_tasks import create_letters_pdf
 from app.config import QueueNames, TaskNames
 from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
@@ -30,6 +31,7 @@ from app.dao.notifications_dao import (
     notifications_not_yet_sent,
     dao_precompiled_letters_still_pending_virus_check,
     dao_old_letters_with_created_status,
+    letters_missing_from_sending_bucket
 )
 from app.dao.provider_details_dao import (
     get_current_provider,
@@ -173,8 +175,8 @@ def check_job_status():
 @notify_celery.task(name='replay-created-notifications')
 @statsd(namespace="tasks")
 def replay_created_notifications():
-    # if the notification has not be send after 4 hours + 15 minutes, then try to resend.
+    # if the notification has not be send after 1 hour, then try to resend.
-    resend_created_notifications_older_than = (60 * 60 * 4) + (60 * 15)
+    resend_created_notifications_older_than = (60 * 60)
     for notification_type in (EMAIL_TYPE, SMS_TYPE):
         notifications_to_resend = notifications_not_yet_sent(
             resend_created_notifications_older_than,
@@ -189,6 +191,20 @@ def replay_created_notifications():
         for n in notifications_to_resend:
             send_notification_to_queue(notification=n, research_mode=n.service.research_mode)
 
+    # if the letter has not be send after an hour, then create a zendesk ticket
+    letters = letters_missing_from_sending_bucket(resend_created_notifications_older_than)
+
+    if len(letters) > 0:
+        msg = "{} letters were created over an hour ago, " \
+              "but do not have an updated_at timestamp or billable units. " \
+              "\n Creating app.celery.letters_pdf_tasks.create_letters tasks to upload letter to S3 " \
+              "and update notifications for the following notification ids: " \
+              "\n {}".format(len(letters), [x.id for x in letters])
+
+        current_app.logger.info(msg)
+        for letter in letters:
+            create_letters_pdf.apply_async([letter.id], queue=QueueNames.LETTERS)
+
 
 @notify_celery.task(name='check-precompiled-letter-state')
 @statsd(namespace="tasks")

app/dao/notifications_dao.py

@@ -32,6 +32,7 @@ from app.models import (
     Notification,
     NotificationHistory,
     ScheduledNotification,
+    KEY_TYPE_NORMAL,
     KEY_TYPE_TEST,
     LETTER_TYPE,
     NOTIFICATION_CREATED,
@@ -46,7 +47,7 @@ from app.models import (
     SMS_TYPE,
     EMAIL_TYPE,
     ServiceDataRetention,
-    Service
+    Service,
 )
 from app.utils import get_london_midnight_in_utc
 from app.utils import midnight_n_days_ago, escape_special_characters
@@ -698,15 +699,32 @@ def dao_old_letters_with_created_status():
     last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0)
 
     notifications = Notification.query.filter(
-        Notification.updated_at < convert_bst_to_utc(last_processing_deadline),
+        Notification.created_at < convert_bst_to_utc(last_processing_deadline),
         Notification.notification_type == LETTER_TYPE,
         Notification.status == NOTIFICATION_CREATED
     ).order_by(
-        Notification.updated_at
+        Notification.created_at
     ).all()
     return notifications
 
 
+def letters_missing_from_sending_bucket(seconds_to_subtract):
+    older_than_date = datetime.utcnow() - timedelta(seconds=seconds_to_subtract)
+    # We expect letters to have a `created` status, updated_at timestamp and billable units greater than zero.
+    notifications = Notification.query.filter(
+        Notification.billable_units == 0,
+        Notification.updated_at == None,  # noqa
+        Notification.status == NOTIFICATION_CREATED,
+        Notification.created_at <= older_than_date,
+        Notification.notification_type == LETTER_TYPE,
+        Notification.key_type == KEY_TYPE_NORMAL
+    ).order_by(
+        Notification.created_at
+    ).all()
+
+    return notifications
+
+
 def dao_precompiled_letters_still_pending_virus_check():
     ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400)
 

tests/app/celery/test_scheduled_tasks.py

@@ -285,7 +285,7 @@ def test_replay_created_notifications(notify_db_session, sample_service, mocker)
 
     sms_template = create_template(service=sample_service, template_type='sms')
     email_template = create_template(service=sample_service, template_type='email')
-    older_than = (60 * 60 * 4) + (60 * 15)  # 4 hours 15 minutes
+    older_than = (60 * 60) + (60 * 15)  # 1 hour 15 minutes
     # notifications expected to be resent
     old_sms = create_notification(template=sms_template, created_at=datetime.utcnow() - timedelta(seconds=older_than),
                                   status='created')
@@ -309,6 +309,28 @@ def test_replay_created_notifications(notify_db_session, sample_service, mocker)
                                                queue="send-sms-tasks")
 
 
+def test_replay_created_notifications_create_letters_pdf_tasks_for_letters_not_ready_to_send(
+        sample_letter_template, mocker
+):
+    mock_task = mocker.patch('app.celery.scheduled_tasks.create_letters_pdf.apply_async')
+    create_notification(template=sample_letter_template, billable_units=0,
+                        created_at=datetime.utcnow() - timedelta(hours=4))
+
+    create_notification(template=sample_letter_template, billable_units=0,
+                        created_at=datetime.utcnow() - timedelta(minutes=20))
+    notification_1 = create_notification(template=sample_letter_template, billable_units=0,
+                                         created_at=datetime.utcnow() - timedelta(hours=1, minutes=20))
+    notification_2 = create_notification(template=sample_letter_template, billable_units=0,
+                                         created_at=datetime.utcnow() - timedelta(hours=5))
+
+    replay_created_notifications()
+
+    calls = [call([notification_1.id], queue=QueueNames.LETTERS),
+             call([notification_2.id], queue=QueueNames.LETTERS),
+             ]
+    mock_task.assert_has_calls(calls, any_order=True)
+
+
 def test_check_job_status_task_does_not_raise_error(sample_template):
     create_job(
         template=sample_template,
@@ -363,12 +385,12 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
     mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
-    noti_1 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 1, 12, 0))
+    noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0))
-    noti_2 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 16, 29))
+    noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 16, 30))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 30))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 17, 29))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 17, 29))
-    create_notification(template=sample_letter_template, status='delivered', updated_at=datetime(2019, 5, 28, 10, 0))
+    create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 30, 10, 0))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0))
 
     check_templated_letter_state()
 
@@ -386,14 +408,14 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
 @freeze_time("2019-01-30 14:00:00")
 def test_check_templated_letter_state_during_utc(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
-    mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
+    mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket')
 
-    noti_1 = create_notification(template=sample_letter_template, updated_at=datetime(2018, 12, 1, 12, 0))
+    noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0))
-    noti_2 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 17, 29))
+    noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 17, 30))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 30))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 18, 29))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 18, 29))
-    create_notification(template=sample_letter_template, status='delivered', updated_at=datetime(2019, 1, 29, 10, 0))
+    create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 30, 10, 0))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0))
 
     check_templated_letter_state()