diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index 8c1ce0e3c..9a34ba729 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -137,39 +137,44 @@ def collate_letter_pdfs_to_be_sent(): letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline) - for i, letters in enumerate(group_letters(letters_to_print)): - filenames = [letter['Key'] for letter in letters] + i = 0 + for zip_folder, letters_list in letters_to_print.items(): + for letters in group_letters(letters_list): + i += 1 + filenames = [letter['Key'] for letter in letters] - hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() - # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP - dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format( - date=print_run_deadline.strftime("%Y-%m-%d"), - num=i + 1, - hash=hash - ) - - current_app.logger.info( - 'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format( - len(filenames), - dvla_filename, - sum(letter['Size'] for letter in letters) + hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() + # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP + dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format( + date=print_run_deadline.strftime("%Y-%m-%d"), + num=i, + hash=hash + ) + + current_app.logger.info( + 'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format( + len(filenames), + dvla_filename, + sum(letter['Size'] for letter in letters) + ) + ) + notify_celery.send_task( + name=TaskNames.ZIP_AND_SEND_LETTER_PDFS, + kwargs={ + 'filenames_to_zip': filenames, + 'upload_filename': dvla_filename + }, + queue=QueueNames.PROCESS_FTP, + compression='zlib' ) - ) - notify_celery.send_task( - name=TaskNames.ZIP_AND_SEND_LETTER_PDFS, - kwargs={ - 'filenames_to_zip': filenames, - 'upload_filename': dvla_filename - }, - queue=QueueNames.PROCESS_FTP, - compression='zlib' - ) def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline) - - letter_pdfs = [] + zip_folders_by_postage = { + "first": "first", "second": "second", "europe": "international", "rest-of-world": "international" + } + letter_pdfs = {"first": [], "second": [], "international": []} for letter in letters_awaiting_sending: try: letter_file_name = get_letter_pdf_filename( @@ -179,7 +184,9 @@ def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): postage=letter.postage ) letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name) - letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']}) + letter_pdfs[ + zip_folders_by_postage[letter.postage] + ].append({"Key": letter_file_name, "Size": letter_head['ContentLength']}) except BotoClientError as e: current_app.logger.exception( f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e) diff --git a/tests/app/celery/test_letters_pdf_tasks.py b/tests/app/celery/test_letters_pdf_tasks.py index c587b646e..31cde9044 100644 --- a/tests/app/celery/test_letters_pdf_tasks.py +++ b/tests/app/celery/test_letters_pdf_tasks.py @@ -155,20 +155,19 @@ def test_update_billable_units_for_letter_doesnt_update_if_sent_with_test_key(mo @freeze_time('2020-02-17 18:00:00') def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sample_letter_template): + # second class create_notification( template=sample_letter_template, status='created', reference='ref0', created_at=(datetime.now() - timedelta(hours=2)) ) - create_notification( template=sample_letter_template, status='created', reference='ref1', created_at=(datetime.now() - timedelta(hours=3)) ) - create_notification( template=sample_letter_template, status='created', @@ -176,6 +175,31 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam created_at=(datetime.now() - timedelta(days=2)) ) + # first class + create_notification( + template=sample_letter_template, + status='created', + reference='first_class', + created_at=(datetime.now() - timedelta(hours=4)), + postage="first" + ) + + # international + create_notification( + template=sample_letter_template, + status='created', + reference='international', + created_at=(datetime.now() - timedelta(days=3)), + postage="europe" + ) + create_notification( + template=sample_letter_template, + status='created', + reference='international', + created_at=(datetime.now() - timedelta(days=4)), + postage="rest-of-world" + ) + # notifications we don't expect to get sent to print as they are in the wrong status for status in ['delivered', 'validation-failed', 'cancelled', 'sending']: create_notification( @@ -203,28 +227,46 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam ) mock_s3 = mocker.patch('app.celery.tasks.s3.head_s3_object', side_effect=[ + {'ContentLength': 1}, + {'ContentLength': 1}, {'ContentLength': 2}, {'ContentLength': 1}, {'ContentLength': 3}, + {'ContentLength': 1}, ]) results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30)) - assert mock_s3.call_count == 3 + assert mock_s3.call_count == 6 mock_s3.assert_has_calls( [ + call(current_app.config[ + 'LETTERS_PDF_BUCKET_NAME' + ], '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF'), + call(current_app.config[ + 'LETTERS_PDF_BUCKET_NAME' + ], '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF'), call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF'), + call(current_app.config[ + 'LETTERS_PDF_BUCKET_NAME' + ], '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF'), call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'), call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF'), ] ) - assert len(results) == 3 - assert results == [ - {'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2}, - {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 1}, - {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 3}, - ] + assert results == { + "first": [{'Key': '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF', 'Size': 1}], + "second": [ + {'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2}, + {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 3}, + {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 1}, + ], + "international": [ + {'Key': '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', 'Size': 1}, + {'Key': '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF', 'Size': 1}, + ] + } @freeze_time('2020-02-17 18:00:00') @@ -266,7 +308,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception( ] ) - assert results == [{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 2}] + assert results["second"] == [{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 2}] @pytest.mark.parametrize('time_to_run_task', [ @@ -275,20 +317,19 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception( ]) def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mocker, time_to_run_task): with freeze_time("2020-02-17 18:00:00"): + # second class create_notification( template=sample_letter_template, status='created', reference='ref0', created_at=(datetime.now() - timedelta(hours=2)) ) - create_notification( template=sample_letter_template, status='created', reference='ref1', created_at=(datetime.now() - timedelta(hours=3)) ) - create_notification( template=sample_letter_template, status='created', @@ -296,10 +337,38 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock created_at=(datetime.now() - timedelta(days=2)) ) + # first class + create_notification( + template=sample_letter_template, + status='created', + reference='first_class', + created_at=(datetime.now() - timedelta(hours=4)), + postage="first" + ) + + # international + create_notification( + template=sample_letter_template, + status='created', + reference='international', + created_at=(datetime.now() - timedelta(days=3)), + postage="europe" + ) + create_notification( + template=sample_letter_template, + status='created', + reference='international', + created_at=(datetime.now() - timedelta(days=4)), + postage="rest-of-world" + ) + mocker.patch('app.celery.tasks.s3.head_s3_object', side_effect=[ + {'ContentLength': 1}, + {'ContentLength': 1}, {'ContentLength': 2}, {'ContentLength': 1}, {'ContentLength': 3}, + {'ContentLength': 1}, ]) mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task') @@ -308,26 +377,49 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock with freeze_time(time_to_run_task): collate_letter_pdfs_to_be_sent() - assert len(mock_celery.call_args_list) == 2 + assert len(mock_celery.call_args_list) == 4 assert mock_celery.call_args_list[0] == call( name='zip-and-send-letter-pdfs', kwargs={ 'filenames_to_zip': [ - '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', - '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF' + '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF' ], - 'upload_filename': 'NOTIFY.2020-02-17.001.k3x_WqC5KhB6e2DWv9Ma.ZIP' + 'upload_filename': 'NOTIFY.2020-02-17.001.kHh01fdUxT9iEIYUt5Wx.ZIP' }, queue='process-ftp-tasks', compression='zlib' ) assert mock_celery.call_args_list[1] == call( + name='zip-and-send-letter-pdfs', + kwargs={ + 'filenames_to_zip': [ + '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', + '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF' + ], + 'upload_filename': 'NOTIFY.2020-02-17.002.k3x_WqC5KhB6e2DWv9Ma.ZIP' + }, + queue='process-ftp-tasks', + compression='zlib' + ) + assert mock_celery.call_args_list[2] == call( name='zip-and-send-letter-pdfs', kwargs={ 'filenames_to_zip': [ '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF' ], - 'upload_filename': 'NOTIFY.2020-02-17.002.J85cUw-FWlKuAIOcwdLS.ZIP' + 'upload_filename': 'NOTIFY.2020-02-17.003.J85cUw-FWlKuAIOcwdLS.ZIP' + }, + queue='process-ftp-tasks', + compression='zlib' + ) + assert mock_celery.call_args_list[3] == call( + name='zip-and-send-letter-pdfs', + kwargs={ + 'filenames_to_zip': [ + '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', + '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF' + ], + 'upload_filename': 'NOTIFY.2020-02-17.004.ArkHQVgyuvwCZA-dVExE.ZIP' }, queue='process-ftp-tasks', compression='zlib'