darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-05-22 09:41:05 -04:00
Code Issues Packages Projects Releases Wiki Activity

remove more letter tasks

Browse Source
This commit is contained in:
stvnrlly
2022-12-07 11:37:25 -05:00
parent 0e679c846d
commit e1f517ebd5
5 changed files with 0 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
app/celery/nightly_tasks.py
View File

@@ -185,110 +185,6 @@ def delete_inbound_sms():
raise
@notify_celery.task(name="raise-alert-if-letter-notifications-still-sending")
@cronitor("raise-alert-if-letter-notifications-still-sending")
def raise_alert_if_letter_notifications_still_sending():
still_sending_count, sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
if still_sending_count:
message = "There are {} letters in the 'sending' state from {}".format(
still_sending_count,
sent_date.strftime('%A %d %B')
)
# Only send alerts in production
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
message += ". Resolve using https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending" # noqa
ticket = NotifySupportTicket(
subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still sending",
email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'],
message=message,
ticket_type=NotifySupportTicket.TYPE_INCIDENT,
technical_ticket=True,
ticket_categories=['notify_letters']
)
zendesk_client.send_ticket_to_zendesk(ticket)
else:
current_app.logger.info(message)
def get_letter_notifications_still_sending_when_they_shouldnt_be():
today = datetime.utcnow().date()
# Do nothing on the weekend
if today.isoweekday() in {6, 7}: # sat, sun
return 0, None
if today.isoweekday() in {1, 2}: # mon, tues. look for files from before the weekend
offset_days = 4
else:
offset_days = 2
expected_sent_date = today - timedelta(days=offset_days)
q = Notification.query.filter(
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_SENDING,
Notification.key_type == KEY_TYPE_NORMAL,
func.date(Notification.sent_at) <= expected_sent_date
)
return q.count(), expected_sent_date
@notify_celery.task(name='raise-alert-if-no-letter-ack-file')
@cronitor('raise-alert-if-no-letter-ack-file')
def letter_raise_alert_if_no_ack_file_for_zip():
# get a list of zip files since yesterday
zip_file_set = set()
today_str = datetime.utcnow().strftime('%Y-%m-%d')
yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder=today_str + '/zips_sent',
suffix='.TXT'):
subname = key.split('/')[-1] # strip subfolder in name
zip_file_set.add(subname.upper().replace('.ZIP.TXT', ''))
# get acknowledgement file
ack_file_set = set()
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
ack_file_set.add(key.lstrip('root/dispatch').upper().replace('.ACK.TXT', '')) # noqa
message = '\n'.join([
"Letter ack file does not contain all zip files sent."
"",
f"See runbook at https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#letter-ack-file-does-not-contain-all-zip-files-sent\n", # noqa
f"pdf bucket: {current_app.config['LETTERS_PDF_BUCKET_NAME']}, subfolder: {datetime.utcnow().strftime('%Y-%m-%d')}/zips_sent", # noqa
f"ack bucket: {current_app.config['DVLA_RESPONSE_BUCKET_NAME']}",
"",
f"Missing ack for zip files: {str(sorted(zip_file_set - ack_file_set))}",
])
# strip empty element before comparison
ack_file_set.discard('')
zip_file_set.discard('')
if len(zip_file_set - ack_file_set) > 0:
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
ticket = NotifySupportTicket(
subject="Letter acknowledge error",
message=message,
ticket_type=NotifySupportTicket.TYPE_INCIDENT,
technical_ticket=True,
ticket_categories=['notify_letters']
)
zendesk_client.send_ticket_to_zendesk(ticket)
current_app.logger.error(message)
if len(ack_file_set - zip_file_set) > 0:
current_app.logger.info(
"letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
)
@notify_celery.task(name='save-daily-notification-processing-time')
@cronitor("save-daily-notification-processing-time")
def save_daily_notification_processing_time(local_date=None):

68
app/celery/scheduled_tasks.py
View File

@@ -180,74 +180,6 @@ def replay_created_notifications():
get_pdf_for_templated_letter.apply_async([str(letter.id)], queue=QueueNames.CREATE_LETTERS_PDF)
@notify_celery.task(name='check-if-letters-still-pending-virus-check')
def check_if_letters_still_pending_virus_check():
letters = []
for letter in dao_precompiled_letters_still_pending_virus_check():
# find letter in the scan bucket
filename = generate_letter_pdf_filename(
letter.reference,
letter.created_at,
ignore_folder=True,
postage=letter.postage
)
if s3.file_exists(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename):
current_app.logger.warning(
f'Letter id {letter.id} got stuck in pending-virus-check. Sending off for scan again.'
)
notify_celery.send_task(
name=TaskNames.SCAN_FILE,
kwargs={'filename': filename},
queue=QueueNames.ANTIVIRUS,
)
else:
letters.append(letter)
if len(letters) > 0:
letter_ids = [(str(letter.id), letter.reference) for letter in letters]
msg = f"""{len(letters)} precompiled letters have been pending-virus-check for over 90 minutes.
We couldn't find them in the scan bucket. We'll need to find out where the files are and kick them off
again or move them to technical failure.
Notifications: {sorted(letter_ids)}"""
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
ticket = NotifySupportTicket(
subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still pending virus check",
message=msg,
ticket_type=NotifySupportTicket.TYPE_INCIDENT,
technical_ticket=True,
ticket_categories=['notify_letters']
)
zendesk_client.send_ticket_to_zendesk(ticket)
current_app.logger.error(msg)
@notify_celery.task(name='check-if-letters-still-in-created')
def check_if_letters_still_in_created():
letters = dao_old_letters_with_created_status()
if len(letters) > 0:
msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
"Follow runbook to resolve: " \
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook" \
"#deal-with-Letters-still-in-created.".format(len(letters))
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
ticket = NotifySupportTicket(
subject=f"[{current_app.config['NOTIFY_ENVIRONMENT']}] Letters still in 'created' status",
message=msg,
ticket_type=NotifySupportTicket.TYPE_INCIDENT,
technical_ticket=True,
ticket_categories=['notify_letters']
)
zendesk_client.send_ticket_to_zendesk(ticket)
current_app.logger.error(msg)
@notify_celery.task(name='check-for-missing-rows-in-completed-jobs')
def check_for_missing_rows_in_completed_jobs():
jobs = find_jobs_with_missing_rows()

22
app/config.py
View File

@@ -278,33 +278,11 @@ class Config(object):
# since we mark jobs as archived
'options': {'queue': QueueNames.PERIODIC},
},
'check-if-letters-still-in-created': {
'task': 'check-if-letters-still-in-created',
'schedule': crontab(day_of_week='mon-fri', hour=7, minute=0),
'options': {'queue': QueueNames.PERIODIC}
},
'check-if-letters-still-pending-virus-check': {
'task': 'check-if-letters-still-pending-virus-check',
'schedule': crontab(day_of_week='mon-fri', hour='9,15', minute=0),
'options': {'queue': QueueNames.PERIODIC}
},
'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers': {
'task': 'check-for-services-with-high-failure-rates-or-sending-to-tv-numbers',
'schedule': crontab(day_of_week='mon-fri', hour=10, minute=30),
'options': {'queue': QueueNames.PERIODIC}
},
'raise-alert-if-letter-notifications-still-sending': {
'task': 'raise-alert-if-letter-notifications-still-sending',
'schedule': crontab(hour=17, minute=00),
'options': {'queue': QueueNames.PERIODIC}
},
# The collate-letter-pdf does assume it is called in an hour that BST does not make a
# difference to the truncate date which translates to the filename to process
'raise-alert-if-no-letter-ack-file': {
'task': 'raise-alert-if-no-letter-ack-file',
'schedule': crontab(hour=23, minute=00),
'options': {'queue': QueueNames.PERIODIC}
},
}
}

126
tests/app/celery/test_nightly_tasks.py
View File

@@ -16,9 +16,6 @@ from app.celery.nightly_tasks import (
delete_inbound_sms,
delete_letter_notifications_older_than_retention,
delete_sms_notifications_older_than_retention,
get_letter_notifications_still_sending_when_they_shouldnt_be,
letter_raise_alert_if_no_ack_file_for_zip,
raise_alert_if_letter_notifications_still_sending,
remove_letter_csv_files,
remove_sms_email_csv_files,
s3,
@@ -195,129 +192,6 @@ def test_delete_inbound_sms_calls_child_task(notify_api, mocker):
assert nightly_tasks.delete_inbound_sms_older_than_retention.call_count == 1
def test_create_ticket_if_letter_notifications_still_sending(notify_api, mocker):
mock_get_letters = mocker.patch(
"app.celery.nightly_tasks.get_letter_notifications_still_sending_when_they_shouldnt_be"
)
mock_get_letters.return_value = 1, date(2018, 1, 15)
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.nightly_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
raise_alert_if_letter_notifications_still_sending()
mock_create_ticket.assert_called_once_with(
ANY,
subject='[test] Letters still sending',
email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'],
message=(
"There are 1 letters in the 'sending' state from Monday 15 January. Resolve using "
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending"
),
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
mock_send_ticket_to_zendesk.assert_called_once()
def test_dont_create_ticket_if_letter_notifications_not_still_sending(notify_api, mocker):
mock_get_letters = mocker.patch(
"app.celery.nightly_tasks.get_letter_notifications_still_sending_when_they_shouldnt_be"
)
mock_get_letters.return_value = 0, None
mock_send_ticket_to_zendesk = mocker.patch(
"app.celery.nightly_tasks.zendesk_client.send_ticket_to_zendesk",
autospec=True
)
raise_alert_if_letter_notifications_still_sending()
mock_send_ticket_to_zendesk.assert_not_called()
@freeze_time("Thursday 17th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_no_letters_if_sent_a_day_ago(
sample_letter_template
):
today = datetime.utcnow()
one_day_ago = today - timedelta(days=1)
create_notification(template=sample_letter_template, status='sending', sent_at=one_day_ago)
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 0
@freeze_time("Thursday 17th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_only_finds_letters_still_in_sending_status(
sample_letter_template
):
two_days_ago = datetime(2018, 1, 15, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=two_days_ago)
create_notification(template=sample_letter_template, status='delivered', sent_at=two_days_ago)
create_notification(template=sample_letter_template, status='failed', sent_at=two_days_ago)
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 1
assert expected_sent_date == date(2018, 1, 15)
@freeze_time("Thursday 17th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_letters_older_than_offset(
sample_letter_template
):
three_days_ago = datetime(2018, 1, 14, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=three_days_ago)
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 1
assert expected_sent_date == date(2018, 1, 15)
@freeze_time("Sunday 14th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_be_finds_no_letters_on_weekend(
sample_letter_template
):
yesterday = datetime(2018, 1, 13, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=yesterday)
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 0
@freeze_time("Monday 15th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_thursday_letters_when_run_on_monday(
sample_letter_template
):
thursday = datetime(2018, 1, 11, 13, 30)
yesterday = datetime(2018, 1, 14, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=thursday, postage='first')
create_notification(template=sample_letter_template, status='sending', sent_at=thursday, postage='second')
create_notification(template=sample_letter_template, status='sending', sent_at=yesterday, postage='second')
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 2
assert expected_sent_date == date(2018, 1, 11)
@freeze_time("Tuesday 16th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_friday_letters_when_run_on_tuesday(
sample_letter_template
):
friday = datetime(2018, 1, 12, 13, 30)
yesterday = datetime(2018, 1, 14, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=friday, postage='first')
create_notification(template=sample_letter_template, status='sending', sent_at=friday, postage='second')
create_notification(template=sample_letter_template, status='sending', sent_at=yesterday, postage='first')
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 2
assert expected_sent_date == date(2018, 1, 12)
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_raise_alert_if_no_ack_file_for_zip_does_not_raise_when_files_match_zip_list(mocker, notify_db_session):

2
tests/app/celery/test_scheduled_tasks.py
View File

@@ -13,8 +13,6 @@ from app.celery import scheduled_tasks
from app.celery.scheduled_tasks import (
check_for_missing_rows_in_completed_jobs,
check_for_services_with_high_failure_rates_or_sending_to_tv_numbers,
check_if_letters_still_in_created,
check_if_letters_still_pending_virus_check,
check_job_status,
delete_invitations,
delete_verify_codes,