Merge pull request #3132 from alphagov/created-letters-runbook

Improvements to our letter checking tasks

app/celery/scheduled_tasks.py

@@ -192,9 +192,9 @@ def replay_created_notifications():
             get_pdf_for_templated_letter.apply_async([str(letter.id)], queue=QueueNames.CREATE_LETTERS_PDF)
 
 
-@notify_celery.task(name='check-precompiled-letter-state')
+@notify_celery.task(name='check-if-letters-still-pending-virus-check')
 @statsd(namespace="tasks")
-def check_precompiled_letter_state():
+def check_if_letters_still_pending_virus_check():
     letters = dao_precompiled_letters_still_pending_virus_check()
 
     if len(letters) > 0:
@@ -214,16 +214,18 @@ def check_precompiled_letter_state():
             )
 
 
-@notify_celery.task(name='check-templated-letter-state')
+@notify_celery.task(name='check-if-letters-still-in-created')
 @statsd(namespace="tasks")
-def check_templated_letter_state():
+def check_if_letters_still_in_created():
     letters = dao_old_letters_with_created_status()
 
     if len(letters) > 0:
         letter_ids = [str(letter.id) for letter in letters]
 
         msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
-              "Notifications: {}".format(len(letters), letter_ids)
+            "Follow runbook to resolve: " \
+            "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created. " \
+            "Notifications: {}".format(len(letters), letter_ids)
 
         current_app.logger.warning(msg)
 

app/config.py

@@ -286,13 +286,13 @@ class Config(object):
             # since we mark jobs as archived
             'options': {'queue': QueueNames.PERIODIC},
         },
-        'check-templated-letter-state': {
-            'task': 'check-templated-letter-state',
+        'check-if-letters-still-in-created': {
+            'task': 'check-if-letters-still-in-created',
             'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0),
             'options': {'queue': QueueNames.PERIODIC}
         },
-        'check-precompiled-letter-state': {
-            'task': 'check-precompiled-letter-state',
+        'check-if-letters-still-pending-virus-check': {
+            'task': 'check-if-letters-still-pending-virus-check',
             'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
             'options': {'queue': QueueNames.PERIODIC}
         },

tests/app/celery/test_scheduled_tasks.py

@@ -14,8 +14,8 @@ from app.celery.scheduled_tasks import (
     delete_verify_codes,
     run_scheduled_jobs,
     replay_created_notifications,
-    check_precompiled_letter_state,
-    check_templated_letter_state,
+    check_if_letters_still_pending_virus_check,
+    check_if_letters_still_in_created,
     check_for_missing_rows_in_completed_jobs,
     check_for_services_with_high_failure_rates_or_sending_to_tv_numbers,
     switch_current_sms_provider_on_slow_delivery,
@@ -319,7 +319,7 @@ def test_check_job_status_task_does_not_raise_error(sample_template):
 
 
 @freeze_time("2019-05-30 14:00:00")
-def test_check_precompiled_letter_state(mocker, sample_letter_template):
+def test_check_if_letters_still_pending_virus_check(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning')
     mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
@@ -338,7 +338,7 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template):
                                          created_at=datetime.utcnow() - timedelta(seconds=70000),
                                          reference='two')
 
-    check_precompiled_letter_state()
+    check_if_letters_still_pending_virus_check()
 
     id_references = sorted([(str(notification_1.id), notification_1.reference),
                             (str(notification_2.id), notification_2.reference)])
@@ -356,7 +356,7 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template):
 
 
 @freeze_time("2019-05-30 14:00:00")
-def test_check_templated_letter_state_during_bst(mocker, sample_letter_template):
+def test_check_if_letters_still_in_created_during_bst(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning')
     mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
@@ -367,10 +367,12 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
     create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0))
     create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0))
 
-    check_templated_letter_state()
+    check_if_letters_still_in_created()
 
     message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
-              "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
+        "Follow runbook to resolve: " \
+        "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created. " \
+        "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
 
     mock_logger.assert_called_once_with(message)
     mock_create_ticket.assert_called_with(
@@ -381,7 +383,7 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
 
 
 @freeze_time("2019-01-30 14:00:00")
-def test_check_templated_letter_state_during_utc(mocker, sample_letter_template):
+def test_check_if_letters_still_in_created_during_utc(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning')
     mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket')
 
@@ -392,10 +394,12 @@ def test_check_templated_letter_state_during_utc(mocker, sample_letter_template)
     create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0))
     create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0))
 
-    check_templated_letter_state()
+    check_if_letters_still_in_created()
 
     message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
-              "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
+        "Follow runbook to resolve: " \
+        "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created. " \
+        "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
 
     mock_logger.assert_called_once_with(message)
     mock_create_ticket.assert_called_with(