Add an alert when a letter is created but doesn't have a file in S3 for sending. We can tell this is the case because there is no updated_at and billable units are still 0.

At this point we are just creating a zendesk ticket - perhaps we can just call the create_letter_pdf task.
2026-02-03 18:01:08 -05:00 · 2019-11-13 16:39:59 +00:00
parent 0979b41562
commit c42420c329
3 changed files with 82 additions and 16 deletions
--- a/app/celery/scheduled_tasks.py
+++ b/app/celery/scheduled_tasks.py
@@ -30,6 +30,7 @@ from app.dao.notifications_dao import (
    notifications_not_yet_sent,
    dao_precompiled_letters_still_pending_virus_check,
    dao_old_letters_with_created_status,
+    letters_missing_from_sending_bucket
 )
 from app.dao.provider_details_dao import (
    get_current_provider,
@@ -189,6 +190,25 @@ def replay_created_notifications():
        for n in notifications_to_resend:
            send_notification_to_queue(notification=n, research_mode=n.service.research_mode)

+    # if the letter has not be send after 4 hours + 15 minutes, then create a zendesk ticket
+    letters = letters_missing_from_sending_bucket(resend_created_notifications_older_than)
+
+    if len(letters) > 0:
+        msg = "{} letters were created four hours and 15 minutes ago, " \
+              "but do not have an updated_at timestamp or billable units. " \
+              "It is likely you need to run the " \
+              "app.celery.letters_pdf_tasks.create_letters_pdf task again with " \
+              "the notification id.\n {}".format(len(letters),
+                                                 [x.id for x in letters])
+        current_app.logger.info(msg)
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still in created status might be missing from S3".format(
+                    current_app.config['NOTIFY_ENVIRONMENT']),
+                message=msg,
+                ticket_type=zendesk_client.TYPE_INCIDENT
+            )
+

@notify_celery.task(name='check-precompiled-letter-state')
@statsd(namespace="tasks")
--- a/app/dao/notifications_dao.py
+++ b/app/dao/notifications_dao.py
@@ -32,6 +32,7 @@ from app.models import (
    Notification,
    NotificationHistory,
    ScheduledNotification,
+    KEY_TYPE_NORMAL,
    KEY_TYPE_TEST,
    LETTER_TYPE,
    NOTIFICATION_CREATED,
@@ -46,7 +47,7 @@ from app.models import (
    SMS_TYPE,
    EMAIL_TYPE,
    ServiceDataRetention,
-    Service
+    Service,
 )
 from app.utils import get_london_midnight_in_utc
 from app.utils import midnight_n_days_ago, escape_special_characters
@@ -698,15 +699,32 @@ def dao_old_letters_with_created_status():
    last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0)

    notifications = Notification.query.filter(
-        Notification.updated_at < convert_bst_to_utc(last_processing_deadline),
+        Notification.created_at < convert_bst_to_utc(last_processing_deadline),
        Notification.notification_type == LETTER_TYPE,
        Notification.status == NOTIFICATION_CREATED
    ).order_by(
-        Notification.updated_at
+        Notification.created_at
    ).all()
    return notifications


+def letters_missing_from_sending_bucket(seconds_to_subtract):
+    older_than_date = datetime.utcnow() - timedelta(seconds=seconds_to_subtract)
+
+    notifications = Notification.query.filter(
+        Notification.updated_at == None,  # noqa
+        Notification.status == NOTIFICATION_CREATED,
+        Notification.billable_units == 0,
+        Notification.created_at <= older_than_date,
+        Notification.notification_type == LETTER_TYPE,
+        Notification.key_type == KEY_TYPE_NORMAL
+    ).order_by(
+        Notification.created_at
+    ).all()
+
+    return notifications
+
+
 def dao_precompiled_letters_still_pending_virus_check():
    ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400)

--- a/tests/app/celery/test_scheduled_tasks.py
+++ b/tests/app/celery/test_scheduled_tasks.py
@@ -309,6 +309,34 @@ def test_replay_created_notifications(notify_db_session, sample_service, mocker)
                                               queue="send-sms-tasks")


+def test_replay_created_notifications_create_zendesk_ticket_for_letters_not_ready_to_send(
+        sample_letter_template, mocker
+):
+    mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket')
+    create_notification(template=sample_letter_template, billable_units=0,
+                        created_at=datetime.utcnow() - timedelta(hours=4))
+
+    notification_1 = create_notification(template=sample_letter_template, billable_units=0,
+                                         created_at=datetime.utcnow() - timedelta(hours=4, minutes=20))
+    notification_2 = create_notification(template=sample_letter_template, billable_units=0,
+                                         created_at=datetime.utcnow() - timedelta(hours=5))
+
+    replay_created_notifications()
+
+    message = "{} letters were created four hours and 15 minutes ago, " \
+              "but do not have an updated_at timestamp or billable units. " \
+              "It is likely you need to run the " \
+              "app.celery.letters_pdf_tasks.create_letters_pdf task again with " \
+              "the notification id.\n {}".format(2,
+                                                 [notification_2.id, notification_1.id])
+
+    mock_create_ticket.assert_called_with(
+        message=message,
+        subject='[test] Letters still in created status might be missing from S3',
+        ticket_type='incident'
+    )
+
+
 def test_check_job_status_task_does_not_raise_error(sample_template):
    create_job(
        template=sample_template,
@@ -363,12 +391,12 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
    mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')

-    noti_1 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 1, 12, 0))
-    noti_2 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 16, 29))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 16, 30))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 17, 29))
-    create_notification(template=sample_letter_template, status='delivered', updated_at=datetime(2019, 5, 28, 10, 0))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 30, 10, 0))
+    noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0))
+    noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 30))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 17, 29))
+    create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0))

    check_templated_letter_state()

@@ -386,14 +414,14 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
@freeze_time("2019-01-30 14:00:00")
 def test_check_templated_letter_state_during_utc(mocker, sample_letter_template):
    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
-    mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
+    mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket')

-    noti_1 = create_notification(template=sample_letter_template, updated_at=datetime(2018, 12, 1, 12, 0))
-    noti_2 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 17, 29))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 17, 30))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 18, 29))
-    create_notification(template=sample_letter_template, status='delivered', updated_at=datetime(2019, 1, 29, 10, 0))
-    create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 30, 10, 0))
+    noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0))
+    noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 30))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 18, 29))
+    create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0))
+    create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0))

    check_templated_letter_state()