darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-24 16:23:44 -04:00
Code Issues Packages Projects Releases Wiki Activity

Include postage in zip folder filenames when sending letters

Browse Source 
Also split letters into zips in 4 categories based on their postage.

Also have a separate count for zipfiles in each postage category.
This commit is contained in:
Pea Tyczynska
2020-06-24 14:59:10 +01:00
parent 9db6fc83f9
commit f3c7c098d6
2 changed files with 26 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
app/celery/letters_pdf_tasks.py
View File

@@ -42,6 +42,7 @@ from app.models import (
NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_VALIDATION_FAILED, NOTIFICATION_VALIDATION_FAILED,
NOTIFICATION_VIRUS_SCAN_FAILED, NOTIFICATION_VIRUS_SCAN_FAILED,
RESOLVE_POSTAGE_FOR_FILE_NAME
) )
from app.cronitor import cronitor from app.cronitor import cronitor
@@ -137,16 +138,17 @@ def collate_letter_pdfs_to_be_sent():
letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline) letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline)
i = 0
for zip_folder, letters_list in letters_to_print.items(): for zip_folder, letters_list in letters_to_print.items():
i = 0
for letters in group_letters(letters_list): for letters in group_letters(letters_list):
i += 1 i += 1
filenames = [letter['Key'] for letter in letters] filenames = [letter['Key'] for letter in letters]
hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
# eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format( dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.ZIP'.format(
date=print_run_deadline.strftime("%Y-%m-%d"), date=print_run_deadline.strftime("%Y-%m-%d"),
postage=RESOLVE_POSTAGE_FOR_FILE_NAME[zip_folder],
num=i, num=i,
hash=hash hash=hash
) )
@@ -171,10 +173,7 @@ def collate_letter_pdfs_to_be_sent():
def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline) letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline)
zip_folders_by_postage = { letter_pdfs = {"first": [], "second": [], "europe": [], "rest-of-world": []}
"first": "first", "second": "second", "europe": "international", "rest-of-world": "international"
}
letter_pdfs = {"first": [], "second": [], "international": []}
for letter in letters_awaiting_sending: for letter in letters_awaiting_sending:
try: try:
letter_file_name = get_letter_pdf_filename( letter_file_name = get_letter_pdf_filename(
@@ -184,9 +183,7 @@ def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
postage=letter.postage postage=letter.postage
) )
letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name) letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name)
letter_pdfs[ letter_pdfs[letter.postage].append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
zip_folders_by_postage[letter.postage]
].append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
except BotoClientError as e: except BotoClientError as e:
current_app.logger.exception( current_app.logger.exception(
f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e) f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)

28
tests/app/celery/test_letters_pdf_tasks.py
View File

@@ -262,9 +262,11 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam
{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 3}, {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 3},
{'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 1}, {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 1},
], ],
"international": [ "europe": [
{'Key': '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', 'Size': 1},
{'Key': '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF', 'Size': 1}, {'Key': '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF', 'Size': 1},
],
"rest-of-world": [
{'Key': '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', 'Size': 1},
] ]
} }
@@ -377,14 +379,14 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock
with freeze_time(time_to_run_task): with freeze_time(time_to_run_task):
collate_letter_pdfs_to_be_sent() collate_letter_pdfs_to_be_sent()
assert len(mock_celery.call_args_list) == 4 assert len(mock_celery.call_args_list) == 5
assert mock_celery.call_args_list[0] == call( assert mock_celery.call_args_list[0] == call(
name='zip-and-send-letter-pdfs', name='zip-and-send-letter-pdfs',
kwargs={ kwargs={
'filenames_to_zip': [ 'filenames_to_zip': [
'2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF' '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF'
], ],
'upload_filename': 'NOTIFY.2020-02-17.001.kHh01fdUxT9iEIYUt5Wx.ZIP' 'upload_filename': 'NOTIFY.2020-02-17.1.001.kHh01fdUxT9iEIYUt5Wx.ZIP'
}, },
queue='process-ftp-tasks', queue='process-ftp-tasks',
compression='zlib' compression='zlib'
@@ -396,7 +398,7 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock
'2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF',
'2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF' '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'
], ],
'upload_filename': 'NOTIFY.2020-02-17.002.k3x_WqC5KhB6e2DWv9Ma.ZIP' 'upload_filename': 'NOTIFY.2020-02-17.2.001.k3x_WqC5KhB6e2DWv9Ma.ZIP'
}, },
queue='process-ftp-tasks', queue='process-ftp-tasks',
compression='zlib' compression='zlib'
@@ -407,7 +409,7 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock
'filenames_to_zip': [ 'filenames_to_zip': [
'2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF' '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF'
], ],
'upload_filename': 'NOTIFY.2020-02-17.003.J85cUw-FWlKuAIOcwdLS.ZIP' 'upload_filename': 'NOTIFY.2020-02-17.2.002.J85cUw-FWlKuAIOcwdLS.ZIP'
}, },
queue='process-ftp-tasks', queue='process-ftp-tasks',
compression='zlib' compression='zlib'
@@ -416,10 +418,20 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock
name='zip-and-send-letter-pdfs', name='zip-and-send-letter-pdfs',
kwargs={ kwargs={
'filenames_to_zip': [ 'filenames_to_zip': [
'2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF',
'2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF' '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF'
], ],
'upload_filename': 'NOTIFY.2020-02-17.004.ArkHQVgyuvwCZA-dVExE.ZIP' 'upload_filename': 'NOTIFY.2020-02-17.E.001.4YajCZzgzIl7zf8bjWK2.ZIP'
},
queue='process-ftp-tasks',
compression='zlib'
)
assert mock_celery.call_args_list[4] == call(
name='zip-and-send-letter-pdfs',
kwargs={
'filenames_to_zip': [
'2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF',
],
'upload_filename': 'NOTIFY.2020-02-17.N.001.eSvP8Ph6EBKhh3k7BSA2.ZIP'
}, },
queue='process-ftp-tasks', queue='process-ftp-tasks',
compression='zlib' compression='zlib'