diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index bb2f82537..75d13e18c 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -42,6 +42,7 @@ from app.models import ( NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_VALIDATION_FAILED, NOTIFICATION_VIRUS_SCAN_FAILED, + POSTAGE_TYPES, RESOLVE_POSTAGE_FOR_FILE_NAME ) from app.cronitor import cronitor @@ -135,21 +136,18 @@ def collate_letter_pdfs_to_be_sent(): print_run_deadline = print_run_date.replace( hour=17, minute=30, second=0, microsecond=0 ) + for postage in POSTAGE_TYPES: + letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage) - letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline) - - for zip_folder, letters_list in letters_to_print.items(): - i = 0 - for letters in group_letters(letters_list): - i += 1 + for i, letters in enumerate(group_letters(letters_to_print)): filenames = [letter['Key'] for letter in letters] hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.ZIP'.format( date=print_run_deadline.strftime("%Y-%m-%d"), - postage=RESOLVE_POSTAGE_FOR_FILE_NAME[zip_folder], - num=i, + postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage], + num=i + 1, hash=hash ) @@ -171,9 +169,9 @@ def collate_letter_pdfs_to_be_sent(): ) -def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): - letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline) - letter_pdfs = {"first": [], "second": [], "europe": [], "rest-of-world": []} +def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage): + letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage) + letter_pdfs = [] for letter in letters_awaiting_sending: try: letter_file_name = get_letter_pdf_filename( @@ -183,7 +181,7 @@ def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): postage=letter.postage ) letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name) - letter_pdfs[letter.postage].append({"Key": letter_file_name, "Size": letter_head['ContentLength']}) + letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']}) except BotoClientError as e: current_app.logger.exception( f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 6629a118c..db52bc5af 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -740,7 +740,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ return notifications -def dao_get_letters_to_be_printed(print_run_deadline): +def dao_get_letters_to_be_printed(print_run_deadline, postage): """ Return all letters created before the print run deadline that have not yet been sent """ @@ -748,7 +748,8 @@ def dao_get_letters_to_be_printed(print_run_deadline): Notification.created_at < convert_bst_to_utc(print_run_deadline), Notification.notification_type == LETTER_TYPE, Notification.status == NOTIFICATION_CREATED, - Notification.key_type == KEY_TYPE_NORMAL + Notification.key_type == KEY_TYPE_NORMAL, + Notification.postage == postage, ).order_by( Notification.created_at ).all() diff --git a/tests/app/celery/test_letters_pdf_tasks.py b/tests/app/celery/test_letters_pdf_tasks.py index 287203af3..4b331f279 100644 --- a/tests/app/celery/test_letters_pdf_tasks.py +++ b/tests/app/celery/test_letters_pdf_tasks.py @@ -175,31 +175,6 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam created_at=(datetime.now() - timedelta(days=2)) ) - # first class - create_notification( - template=sample_letter_template, - status='created', - reference='first_class', - created_at=(datetime.now() - timedelta(hours=4)), - postage="first" - ) - - # international - create_notification( - template=sample_letter_template, - status='created', - reference='international', - created_at=(datetime.now() - timedelta(days=3)), - postage="europe" - ) - create_notification( - template=sample_letter_template, - status='created', - reference='international', - created_at=(datetime.now() - timedelta(days=4)), - postage="rest-of-world" - ) - # notifications we don't expect to get sent to print as they are in the wrong status for status in ['delivered', 'validation-failed', 'cancelled', 'sending']: create_notification( @@ -227,48 +202,29 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam ) mock_s3 = mocker.patch('app.celery.tasks.s3.head_s3_object', side_effect=[ - {'ContentLength': 1}, - {'ContentLength': 1}, {'ContentLength': 2}, {'ContentLength': 1}, {'ContentLength': 3}, - {'ContentLength': 1}, ]) - results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30)) + results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30), postage='second') - assert mock_s3.call_count == 6 + assert mock_s3.call_count == 3 mock_s3.assert_has_calls( [ - call(current_app.config[ - 'LETTERS_PDF_BUCKET_NAME' - ], '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF'), - call(current_app.config[ - 'LETTERS_PDF_BUCKET_NAME' - ], '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF'), call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF'), - call(current_app.config[ - 'LETTERS_PDF_BUCKET_NAME' - ], '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF'), call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'), call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF'), ] ) - assert results == { - "first": [{'Key': '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF', 'Size': 1}], - "second": [ - {'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2}, - {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 3}, - {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 1}, - ], - "europe": [ - {'Key': '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF', 'Size': 1}, - ], - "rest-of-world": [ - {'Key': '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', 'Size': 1}, - ] - } + assert len(results) == 3 + + assert results == [ + {'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2}, + {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 1}, + {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 3}, + ] @freeze_time('2020-02-17 18:00:00') @@ -300,7 +256,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception( ClientError(error_response, "File not found") ]) - results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30)) + results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30), postage='second') assert mock_head_s3_object.call_count == 2 mock_head_s3_object.assert_has_calls( @@ -310,7 +266,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception( ] ) - assert results["second"] == [{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 2}] + assert results == [{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 2}] @pytest.mark.parametrize('time_to_run_task', [