From 1a4baf4283ea38ae0859fbe9c4ad5158c3be89a8 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 19 Mar 2019 13:48:17 +0000 Subject: [PATCH] pass upload filename to notify-ftp previously ftp would name the files itself by giving them a timestamp when uploading. we ran into issues with tasks being picked up multiple times and as such, uploading duplicate files. By naming the file before creating the task, we can avoid this issue. Files are now named `NOTIFY.YYYYMMDD######.ZIP` where the number is a counter that increments with each task we've issued in that run of collate-letter-pdfs-for-day --- app/celery/letters_pdf_tasks.py | 11 ++++++++--- tests/app/celery/test_letters_pdf_tasks.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index 9149b72cc..409ed7808 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -119,17 +119,22 @@ def collate_letter_pdfs_for_day(date=None): current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=date ) - for letters in group_letters(letter_pdfs): + for i, letters in enumerate(group_letters(letter_pdfs)): + dvla_filename = 'NOTIFY.{date}{num:06}.ZIP'.format(date=date.replace('-', ''), num=i + 1) filenames = [letter['Key'] for letter in letters] current_app.logger.info( - 'Calling task zip-and-send-letter-pdfs for {} pdfs of total size {:,} bytes'.format( + 'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format( len(filenames), + dvla_filename, sum(letter['Size'] for letter in letters) ) ) notify_celery.send_task( name=TaskNames.ZIP_AND_SEND_LETTER_PDFS, - kwargs={'filenames_to_zip': filenames}, + kwargs={ + 'filenames_to_zip': filenames, + 'upload_filename': dvla_filename + }, queue=QueueNames.PROCESS_FTP, compression='zlib' ) diff --git a/tests/app/celery/test_letters_pdf_tasks.py b/tests/app/celery/test_letters_pdf_tasks.py index 4a849a005..d35301ae5 100644 --- a/tests/app/celery/test_letters_pdf_tasks.py +++ b/tests/app/celery/test_letters_pdf_tasks.py @@ -231,13 +231,13 @@ def test_collate_letter_pdfs_for_day(notify_api, mocker): mock_group_letters.assert_called_once_with(mock_s3.return_value) assert mock_celery.call_args_list[0] == call( name='zip-and-send-letter-pdfs', - kwargs={'filenames_to_zip': ['A.PDF', 'B.pDf']}, + kwargs={'filenames_to_zip': ['A.PDF', 'B.pDf'], 'upload_filename': 'NOTIFY.20170102000001.ZIP'}, queue='process-ftp-tasks', compression='zlib' ) assert mock_celery.call_args_list[1] == call( name='zip-and-send-letter-pdfs', - kwargs={'filenames_to_zip': ['C.pdf']}, + kwargs={'filenames_to_zip': ['C.pdf'], 'upload_filename': 'NOTIFY.20170102000002.ZIP'}, queue='process-ftp-tasks', compression='zlib' )