darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-07-13 01:44:21 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2431 from alphagov/change-file-format-for-ack-files

Browse Source 
Update alert for new DVLA file format
This commit is contained in:
Rebecca Law
2019-04-03 12:09:59 +01:00
committed by GitHub
parent fcc25abf18 dc8159104e
commit b17cfd50b7
2 changed files with 17 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
app/celery/nightly_tasks.py
View File

@@ -288,31 +288,21 @@ def raise_alert_if_letter_notifications_still_sending():
def letter_raise_alert_if_no_ack_file_for_zip():
# get a list of zip files since yesterday
zip_file_set = set()
today_str = datetime.utcnow().strftime('%Y-%m-%d')
yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
subfolder=today_str + '/zips_sent',
suffix='.TXT'):
subname = key.split('/')[-1] # strip subfolder in name
zip_file_set.add(subname.upper().rstrip('.TXT'))
zip_file_set.add(subname.upper().rstrip('ZIP.TXT'))
# get acknowledgement file
ack_file_set = set()
yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
ack_file_set.add(key)
today_str = datetime.utcnow().strftime('%Y%m%d')
ack_content_set = set()
for key in ack_file_set:
if today_str in key:
content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key)
for zip_file in content.split('\n'): # each line
s = zip_file.split('|')
ack_content_set.add(s[0].upper())
ack_file_set.add(key.lstrip('root/dispatch').upper().rstrip('ACK.TXT'))
message = (
"Letter ack file does not contain all zip files sent. "
@@ -320,16 +310,16 @@ def letter_raise_alert_if_no_ack_file_for_zip():
"pdf bucket: {}, subfolder: {}, "
"ack bucket: {}"
).format(
str(sorted(zip_file_set - ack_content_set)),
str(sorted(zip_file_set - ack_file_set)),
current_app.config['LETTERS_PDF_BUCKET_NAME'],
datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
current_app.config['DVLA_RESPONSE_BUCKET_NAME']
)
# strip empty element before comparison
ack_content_set.discard('')
ack_file_set.discard('')
zip_file_set.discard('')
if len(zip_file_set - ack_content_set) > 0:
if len(zip_file_set - ack_file_set) > 0:
if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
zendesk_client.create_ticket(
subject="Letter acknowledge error",
@@ -338,7 +328,7 @@ def letter_raise_alert_if_no_ack_file_for_zip():
)
current_app.logger.error(message)
if len(ack_content_set - zip_file_set) > 0:
if len(ack_file_set - zip_file_set) > 0:
current_app.logger.info(
"letter ack contains zip that is not for today: {}".format(ack_content_set - zip_file_set)
"letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
)

29
tests/app/celery/test_nightly_tasks.py
View File

@@ -50,17 +50,17 @@ from tests.app.conftest import datetime_in_past
def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None):
if subfolder == '2018-01-11/zips_sent':
return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT']
return ['NOTIFY.2018-01-11175007.ZIP.TXT', 'NOTIFY.2018-01-11175008.ZIP.TXT']
if subfolder == 'root/dispatch':
return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
return ['root/dispatch/NOTIFY.2018-01-11175007.ACK.txt', 'root/dispatch/NOTIFY.2018-01-11175008.ACK.txt']
def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None):
if subfolder == '2018-01-11/zips_sent':
return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT', 'NOTIFY.20180111175009.ZIP.TXT',
'NOTIFY.20180111175010.ZIP.TXT']
return ['NOTIFY.2018-01-11175007.ZIP.TXT', 'NOTIFY.2018-01-11175008.ZIP.TXT', 'NOTIFY.2018-01-11175009.ZIP.TXT',
'NOTIFY.2018-01-11175010.ZIP.TXT']
if subfolder == 'root/dispatch':
return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
return ['root/disoatch/NOTIFY.2018-01-11175007.ACK.TXT', 'root/disoatch/NOTIFY.2018-01-11175008.ACK.TXT']
@freeze_time('2016-10-18T10:00:00')
@@ -504,12 +504,8 @@ def test_tuesday_alert_if_letter_notifications_still_sending_reports_friday_lett
@freeze_time('2018-01-11T23:00:00')
def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db):
def test_letter_raise_alert_if_no_ack_file_for_zip_does_not_raise_when_files_match_zip_list(mocker, notify_db):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match)
mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
'NOTIFY.20180111175008.ZIP|20180111175734')
letter_raise_alert_if_no_ack_file_for_zip()
yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # Datatime format on AWS
@@ -520,30 +516,24 @@ def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db):
call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch',
suffix='.ACK.txt', last_modified=yesterday),
]
assert mock_get_file.call_count == 1
@freeze_time('2018-01-11T23:00:00')
def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff)
mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
'NOTIFY.20180111175008.ZIP|20180111175734')
mock_zendesk = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket")
letter_raise_alert_if_no_ack_file_for_zip()
assert mock_file_list.call_count == 2
assert mock_get_file.call_count == 1
message = "Letter ack file does not contain all zip files sent. " \
"Missing ack for zip files: {}, " \
"pdf bucket: {}, subfolder: {}, " \
"ack bucket: {}".format(str(['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']),
"ack bucket: {}".format(str(['NOTIFY.2018-01-11175009', 'NOTIFY.2018-01-11175010']),
current_app.config['LETTERS_PDF_BUCKET_NAME'],
datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
current_app.config['DVLA_RESPONSE_BUCKET_NAME'])
mock_zendesk.assert_called_once_with(
subject="Letter acknowledge error",
message=message,
@@ -554,11 +544,6 @@ def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db):
@freeze_time('2018-01-11T23:00:00')
def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None)
mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
'NOTIFY.20180111175008.ZIP|20180111175734')
letter_raise_alert_if_no_ack_file_for_zip()
assert mock_file_list.call_count == 2
assert mock_get_file.call_count == 0