darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-22 08:21:13 -05:00
Code Issues Packages Projects Releases Wiki Activity

pass upload filename to notify-ftp

Browse Source 
previously ftp would name the files itself by giving them a timestamp
when uploading. we ran into issues with tasks being picked up multiple
times and as such, uploading duplicate files. By naming the file before
creating the task, we can avoid this issue.

Files are now named `NOTIFY.YYYYMMDD######.ZIP` where the number is a
counter that increments with each task we've issued in that run of
collate-letter-pdfs-for-day
This commit is contained in:
Leo Hemsted
2019-03-19 13:48:17 +00:00
parent 7b8a478b7a
commit 1a4baf4283
2 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/celery/letters_pdf_tasks.py
View File

@@ -119,17 +119,22 @@ def collate_letter_pdfs_for_day(date=None):
current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder=date
)
for letters in group_letters(letter_pdfs):
for i, letters in enumerate(group_letters(letter_pdfs)):
dvla_filename = 'NOTIFY.{date}{num:06}.ZIP'.format(date=date.replace('-', ''), num=i + 1)
filenames = [letter['Key'] for letter in letters]
current_app.logger.info(
'Calling task zip-and-send-letter-pdfs for {} pdfs of total size {:,} bytes'.format(
'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
len(filenames),
dvla_filename,
sum(letter['Size'] for letter in letters)
)
)
notify_celery.send_task(
name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
kwargs={'filenames_to_zip': filenames},
kwargs={
'filenames_to_zip': filenames,
'upload_filename': dvla_filename
},
queue=QueueNames.PROCESS_FTP,
compression='zlib'
)

4
tests/app/celery/test_letters_pdf_tasks.py
View File

@@ -231,13 +231,13 @@ def test_collate_letter_pdfs_for_day(notify_api, mocker):
mock_group_letters.assert_called_once_with(mock_s3.return_value)
assert mock_celery.call_args_list[0] == call(
name='zip-and-send-letter-pdfs',
kwargs={'filenames_to_zip': ['A.PDF', 'B.pDf']},
kwargs={'filenames_to_zip': ['A.PDF', 'B.pDf'], 'upload_filename': 'NOTIFY.20170102000001.ZIP'},
queue='process-ftp-tasks',
compression='zlib'
)
assert mock_celery.call_args_list[1] == call(
name='zip-and-send-letter-pdfs',
kwargs={'filenames_to_zip': ['C.pdf']},
kwargs={'filenames_to_zip': ['C.pdf'], 'upload_filename': 'NOTIFY.20170102000002.ZIP'},
queue='process-ftp-tasks',
compression='zlib'
)