From 3c0e609cc94fa5e10a78744551dd367a4789721d Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 10 Feb 2021 14:53:45 +0000 Subject: [PATCH 1/3] Add link to runbook for created letter alert We've got the entry in the runbook, this will make it clear to go and look at it. --- app/celery/scheduled_tasks.py | 4 +++- tests/app/celery/test_scheduled_tasks.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 77d4b12af..fcc5edf29 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -223,7 +223,9 @@ def check_templated_letter_state(): letter_ids = [str(letter.id) for letter in letters] msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \ - "Notifications: {}".format(len(letters), letter_ids) + "Follow runbook to resolve: " \ + "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created. " \ + "Notifications: {}".format(len(letters), letter_ids) current_app.logger.warning(msg) diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 2254fd6f0..ccc5b250a 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -370,7 +370,9 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template) check_templated_letter_state() message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \ - "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id) + "Follow runbook to resolve: " \ + "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created. " \ + "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id) mock_logger.assert_called_once_with(message) mock_create_ticket.assert_called_with( @@ -395,7 +397,9 @@ def test_check_templated_letter_state_during_utc(mocker, sample_letter_template) check_templated_letter_state() message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \ - "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id) + "Follow runbook to resolve: " \ + "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created. " \ + "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id) mock_logger.assert_called_once_with(message) mock_create_ticket.assert_called_with( From 1b9d8252ec76b1cfc276d67c0bfeb26a675e14ba Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 10 Feb 2021 14:58:49 +0000 Subject: [PATCH 2/3] Rename task and function for clarity This doesn't just relate to templated letters, it's actually just checking that there are not any letters still in created that should not be. This change to the naming makes it more accurate and therefore easy to understand --- app/celery/scheduled_tasks.py | 4 ++-- app/config.py | 4 ++-- tests/app/celery/test_scheduled_tasks.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index fcc5edf29..e344985f0 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -214,9 +214,9 @@ def check_precompiled_letter_state(): ) -@notify_celery.task(name='check-templated-letter-state') +@notify_celery.task(name='check-if-letters-still-in-created') @statsd(namespace="tasks") -def check_templated_letter_state(): +def check_if_letters_still_in_created(): letters = dao_old_letters_with_created_status() if len(letters) > 0: diff --git a/app/config.py b/app/config.py index 4bf384fe3..9580a92bc 100644 --- a/app/config.py +++ b/app/config.py @@ -284,8 +284,8 @@ class Config(object): # since we mark jobs as archived 'options': {'queue': QueueNames.PERIODIC}, }, - 'check-templated-letter-state': { - 'task': 'check-templated-letter-state', + 'check-if-letters-still-in-created': { + 'task': 'check-if-letters-still-in-created', 'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0), 'options': {'queue': QueueNames.PERIODIC} }, diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index ccc5b250a..f5310ebaa 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -15,7 +15,7 @@ from app.celery.scheduled_tasks import ( run_scheduled_jobs, replay_created_notifications, check_precompiled_letter_state, - check_templated_letter_state, + check_if_letters_still_in_created, check_for_missing_rows_in_completed_jobs, check_for_services_with_high_failure_rates_or_sending_to_tv_numbers, switch_current_sms_provider_on_slow_delivery, @@ -356,7 +356,7 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template): @freeze_time("2019-05-30 14:00:00") -def test_check_templated_letter_state_during_bst(mocker, sample_letter_template): +def test_check_if_letters_still_in_created_during_bst(mocker, sample_letter_template): mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning') mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket') @@ -367,7 +367,7 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template) create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0)) create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0)) - check_templated_letter_state() + check_if_letters_still_in_created() message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \ "Follow runbook to resolve: " \ @@ -383,7 +383,7 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template) @freeze_time("2019-01-30 14:00:00") -def test_check_templated_letter_state_during_utc(mocker, sample_letter_template): +def test_check_if_letters_still_in_created_during_utc(mocker, sample_letter_template): mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning') mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket') @@ -394,7 +394,7 @@ def test_check_templated_letter_state_during_utc(mocker, sample_letter_template) create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0)) create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0)) - check_templated_letter_state() + check_if_letters_still_in_created() message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \ "Follow runbook to resolve: " \ From 5526c89c343d3a6f517828bd520f395acb44386b Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 10 Feb 2021 15:03:00 +0000 Subject: [PATCH 3/3] Rename task and function for clarity This doesn't just relate to precompiled letters, it's actually just checking that there are not any letters still waiting for a virus check that should not be. This change to the naming makes it more accurate and therefore easy to understand --- app/celery/scheduled_tasks.py | 4 ++-- app/config.py | 4 ++-- tests/app/celery/test_scheduled_tasks.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index e344985f0..22ba8a60b 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -192,9 +192,9 @@ def replay_created_notifications(): get_pdf_for_templated_letter.apply_async([str(letter.id)], queue=QueueNames.CREATE_LETTERS_PDF) -@notify_celery.task(name='check-precompiled-letter-state') +@notify_celery.task(name='check-if-letters-still-pending-virus-check') @statsd(namespace="tasks") -def check_precompiled_letter_state(): +def check_if_letters_still_pending_virus_check(): letters = dao_precompiled_letters_still_pending_virus_check() if len(letters) > 0: diff --git a/app/config.py b/app/config.py index 9580a92bc..96a55f1ac 100644 --- a/app/config.py +++ b/app/config.py @@ -289,8 +289,8 @@ class Config(object): 'schedule': crontab(day_of_week='mon-fri', hour=9, minute=0), 'options': {'queue': QueueNames.PERIODIC} }, - 'check-precompiled-letter-state': { - 'task': 'check-precompiled-letter-state', + 'check-if-letters-still-pending-virus-check': { + 'task': 'check-if-letters-still-pending-virus-check', 'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0), 'options': {'queue': QueueNames.PERIODIC} }, diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index f5310ebaa..6823a85b9 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -14,7 +14,7 @@ from app.celery.scheduled_tasks import ( delete_verify_codes, run_scheduled_jobs, replay_created_notifications, - check_precompiled_letter_state, + check_if_letters_still_pending_virus_check, check_if_letters_still_in_created, check_for_missing_rows_in_completed_jobs, check_for_services_with_high_failure_rates_or_sending_to_tv_numbers, @@ -319,7 +319,7 @@ def test_check_job_status_task_does_not_raise_error(sample_template): @freeze_time("2019-05-30 14:00:00") -def test_check_precompiled_letter_state(mocker, sample_letter_template): +def test_check_if_letters_still_pending_virus_check(mocker, sample_letter_template): mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning') mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket') @@ -338,7 +338,7 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template): created_at=datetime.utcnow() - timedelta(seconds=70000), reference='two') - check_precompiled_letter_state() + check_if_letters_still_pending_virus_check() id_references = sorted([(str(notification_1.id), notification_1.reference), (str(notification_2.id), notification_2.reference)])