From dc8159104e807738725f7a1702fc5d158dc004d5 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 3 Apr 2019 11:03:42 +0100 Subject: [PATCH] Update letter_raise_alert_if_no_ack_file_for_zip for new DVLA file format When we send a zip file of letters to DVLA we expect them to send back an acknowledgement of those files. Previously they named the files like NOTIFY.20180202091254.ACK.TXT and the contents would contain the name of the zip file we sent with a date of when they got it. They have updated this format to mirror the format of the zip file because there was an instance where they sent 2 files of the same name so the later overwrote the first. Since the name matches our name, there is no need to get the file from S3 but just compare file names. --- app/celery/nightly_tasks.py | 30 +++++++++----------------- tests/app/celery/test_nightly_tasks.py | 29 ++++++------------------- 2 files changed, 17 insertions(+), 42 deletions(-) diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py index 5eac8acc2..30df04de8 100644 --- a/app/celery/nightly_tasks.py +++ b/app/celery/nightly_tasks.py @@ -288,31 +288,21 @@ def raise_alert_if_letter_notifications_still_sending(): def letter_raise_alert_if_no_ack_file_for_zip(): # get a list of zip files since yesterday zip_file_set = set() + today_str = datetime.utcnow().strftime('%Y-%m-%d') + yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], - subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', + subfolder=today_str + '/zips_sent', suffix='.TXT'): subname = key.split('/')[-1] # strip subfolder in name - zip_file_set.add(subname.upper().rstrip('.TXT')) + zip_file_set.add(subname.upper().rstrip('ZIP.TXT')) # get acknowledgement file ack_file_set = set() - yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # AWS datetime format - for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday): - ack_file_set.add(key) - - today_str = datetime.utcnow().strftime('%Y%m%d') - - ack_content_set = set() - for key in ack_file_set: - if today_str in key: - content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key) - for zip_file in content.split('\n'): # each line - s = zip_file.split('|') - ack_content_set.add(s[0].upper()) + ack_file_set.add(key.lstrip('root/dispatch').upper().rstrip('ACK.TXT')) message = ( "Letter ack file does not contain all zip files sent. " @@ -320,16 +310,16 @@ def letter_raise_alert_if_no_ack_file_for_zip(): "pdf bucket: {}, subfolder: {}, " "ack bucket: {}" ).format( - str(sorted(zip_file_set - ack_content_set)), + str(sorted(zip_file_set - ack_file_set)), current_app.config['LETTERS_PDF_BUCKET_NAME'], datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', current_app.config['DVLA_RESPONSE_BUCKET_NAME'] ) # strip empty element before comparison - ack_content_set.discard('') + ack_file_set.discard('') zip_file_set.discard('') - if len(zip_file_set - ack_content_set) > 0: + if len(zip_file_set - ack_file_set) > 0: if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: zendesk_client.create_ticket( subject="Letter acknowledge error", @@ -338,7 +328,7 @@ def letter_raise_alert_if_no_ack_file_for_zip(): ) current_app.logger.error(message) - if len(ack_content_set - zip_file_set) > 0: + if len(ack_file_set - zip_file_set) > 0: current_app.logger.info( - "letter ack contains zip that is not for today: {}".format(ack_content_set - zip_file_set) + "letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set) ) diff --git a/tests/app/celery/test_nightly_tasks.py b/tests/app/celery/test_nightly_tasks.py index 7fb0dc85d..7f07592f5 100644 --- a/tests/app/celery/test_nightly_tasks.py +++ b/tests/app/celery/test_nightly_tasks.py @@ -50,17 +50,17 @@ from tests.app.conftest import datetime_in_past def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None): if subfolder == '2018-01-11/zips_sent': - return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT'] + return ['NOTIFY.2018-01-11175007.ZIP.TXT', 'NOTIFY.2018-01-11175008.ZIP.TXT'] if subfolder == 'root/dispatch': - return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] + return ['root/dispatch/NOTIFY.2018-01-11175007.ACK.txt', 'root/dispatch/NOTIFY.2018-01-11175008.ACK.txt'] def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None): if subfolder == '2018-01-11/zips_sent': - return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT', 'NOTIFY.20180111175009.ZIP.TXT', - 'NOTIFY.20180111175010.ZIP.TXT'] + return ['NOTIFY.2018-01-11175007.ZIP.TXT', 'NOTIFY.2018-01-11175008.ZIP.TXT', 'NOTIFY.2018-01-11175009.ZIP.TXT', + 'NOTIFY.2018-01-11175010.ZIP.TXT'] if subfolder == 'root/dispatch': - return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] + return ['root/disoatch/NOTIFY.2018-01-11175007.ACK.TXT', 'root/disoatch/NOTIFY.2018-01-11175008.ACK.TXT'] @freeze_time('2016-10-18T10:00:00') @@ -504,12 +504,8 @@ def test_tuesday_alert_if_letter_notifications_still_sending_reports_friday_lett @freeze_time('2018-01-11T23:00:00') -def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db): +def test_letter_raise_alert_if_no_ack_file_for_zip_does_not_raise_when_files_match_zip_list(mocker, notify_db): mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match) - mock_get_file = mocker.patch("app.aws.s3.get_s3_file", - return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' - 'NOTIFY.20180111175008.ZIP|20180111175734') - letter_raise_alert_if_no_ack_file_for_zip() yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # Datatime format on AWS @@ -520,30 +516,24 @@ def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db): call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday), ] - assert mock_get_file.call_count == 1 @freeze_time('2018-01-11T23:00:00') def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db): mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff) - mock_get_file = mocker.patch("app.aws.s3.get_s3_file", - return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' - 'NOTIFY.20180111175008.ZIP|20180111175734') mock_zendesk = mocker.patch("app.celery.nightly_tasks.zendesk_client.create_ticket") letter_raise_alert_if_no_ack_file_for_zip() assert mock_file_list.call_count == 2 - assert mock_get_file.call_count == 1 message = "Letter ack file does not contain all zip files sent. " \ "Missing ack for zip files: {}, " \ "pdf bucket: {}, subfolder: {}, " \ - "ack bucket: {}".format(str(['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']), + "ack bucket: {}".format(str(['NOTIFY.2018-01-11175009', 'NOTIFY.2018-01-11175010']), current_app.config['LETTERS_PDF_BUCKET_NAME'], datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', current_app.config['DVLA_RESPONSE_BUCKET_NAME']) - mock_zendesk.assert_called_once_with( subject="Letter acknowledge error", message=message, @@ -554,11 +544,6 @@ def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db): @freeze_time('2018-01-11T23:00:00') def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db): mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None) - mock_get_file = mocker.patch("app.aws.s3.get_s3_file", - return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' - 'NOTIFY.20180111175008.ZIP|20180111175734') - letter_raise_alert_if_no_ack_file_for_zip() assert mock_file_list.call_count == 2 - assert mock_get_file.call_count == 0