From e1f517ebd530493eefe3df3c2e20dcf16348742b Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 7 Dec 2022 11:37:25 -0500 Subject: [PATCH] remove more letter tasks --- app/celery/nightly_tasks.py | 104 ------------------- app/celery/scheduled_tasks.py | 68 ------------ app/config.py | 22 ---- tests/app/celery/test_nightly_tasks.py | 126 ----------------------- tests/app/celery/test_scheduled_tasks.py | 2 - 5 files changed, 322 deletions(-) diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py index fda994397..13b5a5e25 100644 --- a/app/celery/nightly_tasks.py +++ b/app/celery/nightly_tasks.py @@ -185,110 +185,6 @@ def delete_inbound_sms(): raise -@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending") -@cronitor("raise-alert-if-letter-notifications-still-sending") -def raise_alert_if_letter_notifications_still_sending(): - still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be() - - if still_sending_count: - message = "There are {} letters in the 'sending' state from {}".format( - still_sending_count, - sent_date.strftime('%A %d %B') - ) - # Only send alerts in production - if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: - message += ". Resolve using https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending" # noqa - - ticket = NotifySupportTicket( - subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still sending", - email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'], - message=message, - ticket_type=NotifySupportTicket.TYPE_INCIDENT, - technical_ticket=True, - ticket_categories=['notify_letters'] - ) - zendesk_client.send_ticket_to_zendesk(ticket) - else: - current_app.logger.info(message) - - -def get_letter_notifications_still_sending_when_they_shouldnt_be(): - today = datetime.utcnow().date() - - # Do nothing on the weekend - if today.isoweekday() in {6, 7}: # sat, sun - return 0, None - - if today.isoweekday() in {1, 2}: # mon, tues. look for files from before the weekend - offset_days = 4 - else: - offset_days = 2 - - expected_sent_date = today - timedelta(days=offset_days) - - q = Notification.query.filter( - Notification.notification_type == LETTER_TYPE, - Notification.status == NOTIFICATION_SENDING, - Notification.key_type == KEY_TYPE_NORMAL, - func.date(Notification.sent_at) <= expected_sent_date - ) - - return q.count(), expected_sent_date - - -@notify_celery.task(name='raise-alert-if-no-letter-ack-file') -@cronitor('raise-alert-if-no-letter-ack-file') -def letter_raise_alert_if_no_ack_file_for_zip(): - # get a list of zip files since yesterday - zip_file_set = set() - today_str = datetime.utcnow().strftime('%Y-%m-%d') - yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format - - for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], - subfolder=today_str + '/zips_sent', - suffix='.TXT'): - subname = key.split('/')[-1] # strip subfolder in name - zip_file_set.add(subname.upper().replace('.ZIP.TXT', '')) - - # get acknowledgement file - ack_file_set = set() - - for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], - subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday): - ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', '')) # noqa - - message = '\n'.join([ - "Letter ack file does not contain all zip files sent." - "", - f"See runbook at https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#letter-ack-file-does-not-contain-all-zip-files-sent\n", # noqa - f"pdf bucket: {current_app.config['LETTERS_PDF_BUCKET_NAME']}, subfolder: {datetime.utcnow().strftime('%Y-%m-%d')}/zips_sent", # noqa - f"ack bucket: {current_app.config['DVLA_RESPONSE_BUCKET_NAME']}", - "", - f"Missing ack for zip files: {str(sorted(zip_file_set - ack_file_set))}", - ]) - - # strip empty element before comparison - ack_file_set.discard('') - zip_file_set.discard('') - - if len(zip_file_set - ack_file_set) > 0: - if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: - ticket = NotifySupportTicket( - subject="Letter acknowledge error", - message=message, - ticket_type=NotifySupportTicket.TYPE_INCIDENT, - technical_ticket=True, - ticket_categories=['notify_letters'] - ) - zendesk_client.send_ticket_to_zendesk(ticket) - current_app.logger.error(message) - - if len(ack_file_set - zip_file_set) > 0: - current_app.logger.info( - "letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set) - ) - - @notify_celery.task(name='save-daily-notification-processing-time') @cronitor("save-daily-notification-processing-time") def save_daily_notification_processing_time(local_date=None): diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 28298aa75..1baac023d 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -180,74 +180,6 @@ def replay_created_notifications(): get_pdf_for_templated_letter.apply_async([str(letter.id)], queue=QueueNames.CREATE_LETTERS_PDF) -@notify_celery.task(name='check-if-letters-still-pending-virus-check') -def check_if_letters_still_pending_virus_check(): - letters = [] - - for letter in dao_precompiled_letters_still_pending_virus_check(): - # find letter in the scan bucket - filename = generate_letter_pdf_filename( - letter.reference, - letter.created_at, - ignore_folder=True, - postage=letter.postage - ) - - if s3.file_exists(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename): - current_app.logger.warning( - f'Letter id {letter.id} got stuck in pending-virus-check. Sending off for scan again.' - ) - notify_celery.send_task( - name=TaskNames.SCAN_FILE, - kwargs={'filename': filename}, - queue=QueueNames.ANTIVIRUS, - ) - else: - letters.append(letter) - - if len(letters) > 0: - letter_ids = [(str(letter.id), letter.reference) for letter in letters] - - msg = f"""{len(letters)} precompiled letters have been pending-virus-check for over 90 minutes. - We couldn't find them in the scan bucket. We'll need to find out where the files are and kick them off - again or move them to technical failure. - - Notifications: {sorted(letter_ids)}""" - - if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: - ticket = NotifySupportTicket( - subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still pending virus check", - message=msg, - ticket_type=NotifySupportTicket.TYPE_INCIDENT, - technical_ticket=True, - ticket_categories=['notify_letters'] - ) - zendesk_client.send_ticket_to_zendesk(ticket) - current_app.logger.error(msg) - - -@notify_celery.task(name='check-if-letters-still-in-created') -def check_if_letters_still_in_created(): - letters = dao_old_letters_with_created_status() - - if len(letters) > 0: - msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \ - "Follow runbook to resolve: " \ - "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook" \ - "#deal-with-Letters-still-in-created.".format(len(letters)) - - if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: - ticket = NotifySupportTicket( - subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still in 'created' status", - message=msg, - ticket_type=NotifySupportTicket.TYPE_INCIDENT, - technical_ticket=True, - ticket_categories=['notify_letters'] - ) - zendesk_client.send_ticket_to_zendesk(ticket) - current_app.logger.error(msg) - - @notify_celery.task(name='check-for-missing-rows-in-completed-jobs') def check_for_missing_rows_in_completed_jobs(): jobs = find_jobs_with_missing_rows() diff --git a/app/config.py b/app/config.py index bf477dbbb..2e520267c 100644 --- a/app/config.py +++ b/app/config.py @@ -278,33 +278,11 @@ class Config(object): # since we mark jobs as archived 'options': {'queue': QueueNames.PERIODIC}, }, - 'check-if-letters-still-in-created': { - 'task': 'check-if-letters-still-in-created', - 'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0), - 'options': {'queue': QueueNames.PERIODIC} - }, - 'check-if-letters-still-pending-virus-check': { - 'task': 'check-if-letters-still-pending-virus-check', - 'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0), - 'options': {'queue': QueueNames.PERIODIC} - }, 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': { 'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers', 'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30), 'options': {'queue': QueueNames.PERIODIC} }, - 'raise-alert-if-letter-notifications-still-sending': { - 'task': 'raise-alert-if-letter-notifications-still-sending', - 'schedule': crontab(hour=17, minute=00), - 'options': {'queue': QueueNames.PERIODIC} - }, - # The collate-letter-pdf does assume it is called in an hour that BST does not make a - # difference to the truncate date which translates to the filename to process - 'raise-alert-if-no-letter-ack-file': { - 'task': 'raise-alert-if-no-letter-ack-file', - 'schedule': crontab(hour=23, minute=00), - 'options': {'queue': QueueNames.PERIODIC} - }, } } diff --git a/tests/app/celery/test_nightly_tasks.py b/tests/app/celery/test_nightly_tasks.py index 5849038cd..2c9cbfd25 100644 --- a/tests/app/celery/test_nightly_tasks.py +++ b/tests/app/celery/test_nightly_tasks.py @@ -16,9 +16,6 @@ from app.celery.nightly_tasks import ( delete_inbound_sms, delete_letter_notifications_older_than_retention, delete_sms_notifications_older_than_retention, - get_letter_notifications_still_sending_when_they_shouldnt_be, - letter_raise_alert_if_no_ack_file_for_zip, - raise_alert_if_letter_notifications_still_sending, remove_letter_csv_files, remove_sms_email_csv_files, s3, @@ -195,129 +192,6 @@ def test_delete_inbound_sms_calls_child_task(notify_api, mocker): assert nightly_tasks.delete_inbound_sms_older_than_retention.call_count == 1 -def test_create_ticket_if_letter_notifications_still_sending(notify_api, mocker): - mock_get_letters = mocker.patch( - "app.celery.nightly_tasks.get_letter_notifications_still_sending_when_they_shouldnt_be" - ) - - mock_get_letters.return_value = 1, date(2018, 1, 15) - mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__') - mock_send_ticket_to_zendesk = mocker.patch( - 'app.celery.nightly_tasks.zendesk_client.send_ticket_to_zendesk', - autospec=True, - ) - - raise_alert_if_letter_notifications_still_sending() - mock_create_ticket.assert_called_once_with( - ANY, - subject='[test] Letters still sending', - email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'], - message=( - "There are 1 letters in the 'sending' state from Monday 15 January. Resolve using " - "https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending" - ), - ticket_type='incident', - technical_ticket=True, - ticket_categories=['notify_letters'] - ) - mock_send_ticket_to_zendesk.assert_called_once() - - -def test_dont_create_ticket_if_letter_notifications_not_still_sending(notify_api, mocker): - mock_get_letters = mocker.patch( - "app.celery.nightly_tasks.get_letter_notifications_still_sending_when_they_shouldnt_be" - ) - - mock_get_letters.return_value = 0, None - mock_send_ticket_to_zendesk = mocker.patch( - "app.celery.nightly_tasks.zendesk_client.send_ticket_to_zendesk", - autospec=True - ) - - raise_alert_if_letter_notifications_still_sending() - - mock_send_ticket_to_zendesk.assert_not_called() - - -@freeze_time("Thursday 17th January 2018 17:00") -def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_no_letters_if_sent_a_day_ago( - sample_letter_template -): - today = datetime.utcnow() - one_day_ago = today - timedelta(days=1) - create_notification(template=sample_letter_template, status='sending', sent_at=one_day_ago) - - count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be() - assert count == 0 - - -@freeze_time("Thursday 17th January 2018 17:00") -def test_get_letter_notifications_still_sending_when_they_shouldnt_only_finds_letters_still_in_sending_status( - sample_letter_template -): - two_days_ago = datetime(2018, 1, 15, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=two_days_ago) - create_notification(template=sample_letter_template, status='delivered', sent_at=two_days_ago) - create_notification(template=sample_letter_template, status='failed', sent_at=two_days_ago) - - count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be() - assert count == 1 - assert expected_sent_date == date(2018, 1, 15) - - -@freeze_time("Thursday 17th January 2018 17:00") -def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_letters_older_than_offset( - sample_letter_template -): - three_days_ago = datetime(2018, 1, 14, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=three_days_ago) - - count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be() - assert count == 1 - assert expected_sent_date == date(2018, 1, 15) - - -@freeze_time("Sunday 14th January 2018 17:00") -def test_get_letter_notifications_still_sending_when_they_shouldnt_be_finds_no_letters_on_weekend( - sample_letter_template -): - yesterday = datetime(2018, 1, 13, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=yesterday) - - count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be() - assert count == 0 - - -@freeze_time("Monday 15th January 2018 17:00") -def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_thursday_letters_when_run_on_monday( - sample_letter_template -): - thursday = datetime(2018, 1, 11, 13, 30) - yesterday = datetime(2018, 1, 14, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=thursday, postage='first') - create_notification(template=sample_letter_template, status='sending', sent_at=thursday, postage='second') - create_notification(template=sample_letter_template, status='sending', sent_at=yesterday, postage='second') - - count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be() - assert count == 2 - assert expected_sent_date == date(2018, 1, 11) - - -@freeze_time("Tuesday 16th January 2018 17:00") -def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_friday_letters_when_run_on_tuesday( - sample_letter_template -): - friday = datetime(2018, 1, 12, 13, 30) - yesterday = datetime(2018, 1, 14, 13, 30) - create_notification(template=sample_letter_template, status='sending', sent_at=friday, postage='first') - create_notification(template=sample_letter_template, status='sending', sent_at=friday, postage='second') - create_notification(template=sample_letter_template, status='sending', sent_at=yesterday, postage='first') - - count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be() - assert count == 2 - assert expected_sent_date == date(2018, 1, 12) - - @freeze_time('2018-01-11T23:00:00') @pytest.mark.skip(reason="Skipping letter-related functionality for now") def test_letter_raise_alert_if_no_ack_file_for_zip_does_not_raise_when_files_match_zip_list(mocker, notify_db_session): diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 55298af2c..bae2e0c5b 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -13,8 +13,6 @@ from app.celery import scheduled_tasks from app.celery.scheduled_tasks import ( check_for_missing_rows_in_completed_jobs, check_for_services_with_high_failure_rates_or_sending_to_tv_numbers, - check_if_letters_still_in_created, - check_if_letters_still_pending_virus_check, check_job_status, delete_invitations, delete_verify_codes,