darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-02 17:31:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

exclude non-pdfs from collate task

Browse Source 
and fix celery kwargsg
This commit is contained in:
Leo Hemsted
2017-12-22 15:38:49 +00:00
committed by Ken Tsang
parent 309b4d7d33
commit 9debc96c8c
2 changed files with 27 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
app/celery/letters_pdf_tasks.py
View File

@@ -97,7 +97,7 @@ def collate_letter_pdfs_for_day(date):
) )
notify_celery.send_task( notify_celery.send_task(
name=TaskNames.ZIP_AND_SEND_LETTER_PDFS, name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
kwargs={'filenames': filenames}, kwargs={'filenames_to_zip': filenames},
queue=QueueNames.PROCESS_FTP queue=QueueNames.PROCESS_FTP
) )
@@ -111,13 +111,14 @@ def group_letters(letter_pdfs):
running_filesize = 0 running_filesize = 0
list_of_files = [] list_of_files = []
for letter in letter_pdfs: for letter in letter_pdfs:
if running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE']: if letter['Key'].lower().endswith('.pdf'):
yield list_of_files if running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE']:
running_filesize = 0 yield list_of_files
list_of_files = [] running_filesize = 0
list_of_files = []
running_filesize += letter['Size'] running_filesize += letter['Size']
list_of_files.append(letter) list_of_files.append(letter)
if list_of_files: if list_of_files:
yield list_of_files yield list_of_files

33
tests/app/celery/test_letters_pdf_tasks.py
View File

@@ -143,8 +143,8 @@ def test_create_letters_pdf_sets_technical_failure_max_retries(mocker, sample_le
def test_collate_letter_pdfs_for_day(notify_api, mocker): def test_collate_letter_pdfs_for_day(notify_api, mocker):
mock_s3 = mocker.patch('app.celery.tasks.s3.get_s3_bucket_objects') mock_s3 = mocker.patch('app.celery.tasks.s3.get_s3_bucket_objects')
mock_group_letters = mocker.patch('app.celery.letters_pdf_tasks.group_letters', return_value=[ mock_group_letters = mocker.patch('app.celery.letters_pdf_tasks.group_letters', return_value=[
[{'Key': 'A', 'Size': 1}, {'Key': 'B', 'Size': 2}], [{'Key': 'A.PDF', 'Size': 1}, {'Key': 'B.pDf', 'Size': 2}],
[{'Key': 'C', 'Size': 3}] [{'Key': 'C.pdf', 'Size': 3}]
]) ])
mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task') mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task')
@@ -154,12 +154,12 @@ def test_collate_letter_pdfs_for_day(notify_api, mocker):
mock_group_letters.assert_called_once_with(mock_s3.return_value) mock_group_letters.assert_called_once_with(mock_s3.return_value)
assert mock_celery.call_args_list[0] == call( assert mock_celery.call_args_list[0] == call(
name='zip-and-send-letter-pdfs', name='zip-and-send-letter-pdfs',
kwargs={'filenames': ['A', 'B']}, kwargs={'filenames_to_zip': ['A.PDF', 'B.pDf']},
queue='process-ftp-tasks' queue='process-ftp-tasks'
) )
assert mock_celery.call_args_list[1] == call( assert mock_celery.call_args_list[1] == call(
name='zip-and-send-letter-pdfs', name='zip-and-send-letter-pdfs',
kwargs={'filenames': ['C']}, kwargs={'filenames_to_zip': ['C.pdf']},
queue='process-ftp-tasks' queue='process-ftp-tasks'
) )
@@ -167,28 +167,33 @@ def test_collate_letter_pdfs_for_day(notify_api, mocker):
def test_group_letters(notify_api): def test_group_letters(notify_api):
letters = [ letters = [
# ends under max but next one is too big # ends under max but next one is too big
{'Key': 'A', 'Size': 1}, {'Key': 'B', 'Size': 2}, {'Key': 'A.pdf', 'Size': 1}, {'Key': 'B.pdf', 'Size': 2},
# ends on exactly max # ends on exactly max
{'Key': 'C', 'Size': 3}, {'Key': 'D', 'Size': 1}, {'Key': 'E', 'Size': 1}, {'Key': 'C.pdf', 'Size': 3}, {'Key': 'D.pdf', 'Size': 1}, {'Key': 'E.pdf', 'Size': 1},
# exactly max goes in next file # exactly max goes in next file
{'Key': 'F', 'Size': 5}, {'Key': 'F.pdf', 'Size': 5},
# if it's bigger than the max, still gets included # if it's bigger than the max, still gets included
{'Key': 'G', 'Size': 6}, {'Key': 'G.pdf', 'Size': 6},
# whatever's left goes in last list # whatever's left goes in last list
{'Key': 'H', 'Size': 1}, {'Key': 'I', 'Size': 1}, {'Key': 'H.pdf', 'Size': 1}, {'Key': 'I.pdf', 'Size': 1},
] ]
with set_config_values(notify_api, {'MAX_LETTER_PDF_ZIP_FILESIZE': 5}): with set_config_values(notify_api, {'MAX_LETTER_PDF_ZIP_FILESIZE': 5}):
x = group_letters(letters) x = group_letters(letters)
assert next(x) == [{'Key': 'A', 'Size': 1}, {'Key': 'B', 'Size': 2}] assert next(x) == [{'Key': 'A.pdf', 'Size': 1}, {'Key': 'B.pdf', 'Size': 2}]
assert next(x) == [{'Key': 'C', 'Size': 3}, {'Key': 'D', 'Size': 1}, {'Key': 'E', 'Size': 1}] assert next(x) == [{'Key': 'C.pdf', 'Size': 3}, {'Key': 'D.pdf', 'Size': 1}, {'Key': 'E.pdf', 'Size': 1}]
assert next(x) == [{'Key': 'F', 'Size': 5}] assert next(x) == [{'Key': 'F.pdf', 'Size': 5}]
assert next(x) == [{'Key': 'G', 'Size': 6}] assert next(x) == [{'Key': 'G.pdf', 'Size': 6}]
assert next(x) == [{'Key': 'H', 'Size': 1}, {'Key': 'I', 'Size': 1}] assert next(x) == [{'Key': 'H.pdf', 'Size': 1}, {'Key': 'I.pdf', 'Size': 1}]
# make sure iterator is exhausted # make sure iterator is exhausted
assert next(x, None) is None assert next(x, None) is None
def test_group_letters_ignores_non_pdfs(notify_api):
letters = [{'Key': 'A.zip'}]
assert list(group_letters(letters)) == []
def test_group_letters_with_no_letters(notify_api): def test_group_letters_with_no_letters(notify_api):
assert list(group_letters([])) == [] assert list(group_letters([])) == []