darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-03 01:41:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Parse acknowledgement files against .ZIP.TXT created by ftp app.

Browse Source 
- Also convert the files info to upper() for comparison rather than lower
because original file names are in upper case. The unit tests contain examples of the returned lists.
This commit is contained in:
venusbb
2018-01-18 10:44:36 +00:00
parent dec8a191a3
commit 8f5a5f8105
3 changed files with 26 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/aws/s3.py
View File

@@ -120,7 +120,7 @@ def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modif
for page in page_iterator: for page in page_iterator:
for obj in page.get('Contents', []): for obj in page.get('Contents', []):
key = obj['Key'].lower() key = obj['Key']
if key.endswith(suffix.lower()): if key.lower().endswith(suffix.lower()):
if not last_modified or obj['LastModified'] >= last_modified: if not last_modified or obj['LastModified'] >= last_modified:
yield key yield key

9
app/celery/scheduled_tasks.py
View File

@@ -490,9 +490,9 @@ def letter_raise_alert_if_no_ack_file_for_zip():
zip_file_list = [] zip_file_list = []
for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder=datetime.utcnow().strftime('%Y-%m-%d'), subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
suffix='.zip'): suffix='.TXT'):
zip_file_list.append(key) zip_file_list.append(key.upper().rstrip('.TXT'))
# get acknowledgement file # get acknowledgement file
ack_file_list = [] ack_file_list = []
@@ -511,13 +511,12 @@ def letter_raise_alert_if_no_ack_file_for_zip():
for zip_file in content.split('\n'): # each line for zip_file in content.split('\n'): # each line
s = zip_file.split('|') s = zip_file.split('|')
for zf in zip_file_list: for zf in zip_file_list:
if s[0].lower() in zf.lower(): if s[0].upper() in zf:
zip_file_list.remove(zf) zip_file_list.remove(zf)
else: else:
zip_not_today.append(s[0]) zip_not_today.append(s[0])
if zip_file_list: if zip_file_list:
raise NoAckFileReceived(message=zip_file_list) raise NoAckFileReceived(message=zip_file_list)
if zip_not_today: if zip_not_today:

27
tests/app/celery/test_scheduled_tasks.py
View File

@@ -1109,16 +1109,16 @@ def test_dao_fetch_monthly_historical_stats_by_template_null_template_id_not_cou
def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None): def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None):
if subfolder == '2018-01-11': if subfolder == '2018-01-11/zips_sent':
return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP'] return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT']
if subfolder == 'root/dispatch': if subfolder == 'root/dispatch':
return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None): def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None):
if subfolder == '2018-01-11': if subfolder == '2018-01-11/zips_sent':
return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP', 'NOTIFY.20180111175009.ZIP', return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT', 'NOTIFY.20180111175009.ZIP.TXT',
'NOTIFY.20180111175010.ZIP'] 'NOTIFY.20180111175010.ZIP.TXT']
if subfolder == 'root/dispatch': if subfolder == 'root/dispatch':
return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
@@ -1133,10 +1133,10 @@ def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db):
letter_raise_alert_if_no_ack_file_for_zip() letter_raise_alert_if_no_ack_file_for_zip()
yesterday = datetime.utcnow() - timedelta(days=1) yesterday = datetime.utcnow() - timedelta(days=1)
subfoldername = datetime.utcnow().strftime('%Y-%m-%d') subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent'
assert mock_file_list.call_count == 2 assert mock_file_list.call_count == 2
assert mock_file_list.call_args_list == [ assert mock_file_list.call_args_list == [
call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.zip'), call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'),
call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch', call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch',
suffix='.ACK.txt', last_modified=yesterday), suffix='.ACK.txt', last_modified=yesterday),
] ]
@@ -1155,3 +1155,16 @@ def test_letter_not_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_d
assert e.value.message == ['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP'] assert e.value.message == ['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']
assert mock_file_list.call_count == 2 assert mock_file_list.call_count == 2
assert mock_get_file.call_count == 1 assert mock_get_file.call_count == 1
@freeze_time('2018-01-11T23:00:00')
def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None)
mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
'NOTIFY.20180111175008.ZIP|20180111175734')
letter_raise_alert_if_no_ack_file_for_zip()
assert mock_file_list.call_count == 2
assert mock_get_file.call_count == 0