diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index 9a34ba729..bb2f82537 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -42,6 +42,7 @@ from app.models import ( NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_VALIDATION_FAILED, NOTIFICATION_VIRUS_SCAN_FAILED, + RESOLVE_POSTAGE_FOR_FILE_NAME ) from app.cronitor import cronitor @@ -137,16 +138,17 @@ def collate_letter_pdfs_to_be_sent(): letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline) - i = 0 for zip_folder, letters_list in letters_to_print.items(): + i = 0 for letters in group_letters(letters_list): i += 1 filenames = [letter['Key'] for letter in letters] hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP - dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format( + dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.ZIP'.format( date=print_run_deadline.strftime("%Y-%m-%d"), + postage=RESOLVE_POSTAGE_FOR_FILE_NAME[zip_folder], num=i, hash=hash ) @@ -171,10 +173,7 @@ def collate_letter_pdfs_to_be_sent(): def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline) - zip_folders_by_postage = { - "first": "first", "second": "second", "europe": "international", "rest-of-world": "international" - } - letter_pdfs = {"first": [], "second": [], "international": []} + letter_pdfs = {"first": [], "second": [], "europe": [], "rest-of-world": []} for letter in letters_awaiting_sending: try: letter_file_name = get_letter_pdf_filename( @@ -184,9 +183,7 @@ def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): postage=letter.postage ) letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name) - letter_pdfs[ - zip_folders_by_postage[letter.postage] - ].append({"Key": letter_file_name, "Size": letter_head['ContentLength']}) + letter_pdfs[letter.postage].append({"Key": letter_file_name, "Size": letter_head['ContentLength']}) except BotoClientError as e: current_app.logger.exception( f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e) diff --git a/tests/app/celery/test_letters_pdf_tasks.py b/tests/app/celery/test_letters_pdf_tasks.py index 31cde9044..287203af3 100644 --- a/tests/app/celery/test_letters_pdf_tasks.py +++ b/tests/app/celery/test_letters_pdf_tasks.py @@ -262,9 +262,11 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 3}, {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 1}, ], - "international": [ - {'Key': '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', 'Size': 1}, + "europe": [ {'Key': '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF', 'Size': 1}, + ], + "rest-of-world": [ + {'Key': '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', 'Size': 1}, ] } @@ -377,14 +379,14 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock with freeze_time(time_to_run_task): collate_letter_pdfs_to_be_sent() - assert len(mock_celery.call_args_list) == 4 + assert len(mock_celery.call_args_list) == 5 assert mock_celery.call_args_list[0] == call( name='zip-and-send-letter-pdfs', kwargs={ 'filenames_to_zip': [ '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF' ], - 'upload_filename': 'NOTIFY.2020-02-17.001.kHh01fdUxT9iEIYUt5Wx.ZIP' + 'upload_filename': 'NOTIFY.2020-02-17.1.001.kHh01fdUxT9iEIYUt5Wx.ZIP' }, queue='process-ftp-tasks', compression='zlib' @@ -396,7 +398,7 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF' ], - 'upload_filename': 'NOTIFY.2020-02-17.002.k3x_WqC5KhB6e2DWv9Ma.ZIP' + 'upload_filename': 'NOTIFY.2020-02-17.2.001.k3x_WqC5KhB6e2DWv9Ma.ZIP' }, queue='process-ftp-tasks', compression='zlib' @@ -407,7 +409,7 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock 'filenames_to_zip': [ '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF' ], - 'upload_filename': 'NOTIFY.2020-02-17.003.J85cUw-FWlKuAIOcwdLS.ZIP' + 'upload_filename': 'NOTIFY.2020-02-17.2.002.J85cUw-FWlKuAIOcwdLS.ZIP' }, queue='process-ftp-tasks', compression='zlib' @@ -416,10 +418,20 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock name='zip-and-send-letter-pdfs', kwargs={ 'filenames_to_zip': [ - '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF' ], - 'upload_filename': 'NOTIFY.2020-02-17.004.ArkHQVgyuvwCZA-dVExE.ZIP' + 'upload_filename': 'NOTIFY.2020-02-17.E.001.4YajCZzgzIl7zf8bjWK2.ZIP' + }, + queue='process-ftp-tasks', + compression='zlib' + ) + assert mock_celery.call_args_list[4] == call( + name='zip-and-send-letter-pdfs', + kwargs={ + 'filenames_to_zip': [ + '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', + ], + 'upload_filename': 'NOTIFY.2020-02-17.N.001.eSvP8Ph6EBKhh3k7BSA2.ZIP' }, queue='process-ftp-tasks', compression='zlib'