diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index 74fc22fcf..a5832ead9 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -52,7 +52,7 @@ from app.models import ( NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_VALIDATION_FAILED, NOTIFICATION_VIRUS_SCAN_FAILED, -) + POSTAGE_TYPES) from app.cronitor import cronitor @@ -136,39 +136,40 @@ def collate_letter_pdfs_to_be_sent(): hour=17, minute=30, second=0, microsecond=0 ) - letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline) + for postage in POSTAGE_TYPES: + letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage) - for i, letters in enumerate(group_letters(letters_to_print)): - filenames = [letter['Key'] for letter in letters] + for i, letters in enumerate(group_letters(letters_to_print)): + filenames = [letter['Key'] for letter in letters] - hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() - # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP - dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format( - date=print_run_deadline.strftime("%Y-%m-%d"), - num=i + 1, - hash=hash - ) - - current_app.logger.info( - 'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format( - len(filenames), - dvla_filename, - sum(letter['Size'] for letter in letters) + hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() + # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP + dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format( + date=print_run_deadline.strftime("%Y-%m-%d"), + num=i + 1, + hash=hash + ) + + current_app.logger.info( + 'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format( + len(filenames), + dvla_filename, + sum(letter['Size'] for letter in letters) + ) + ) + notify_celery.send_task( + name=TaskNames.ZIP_AND_SEND_LETTER_PDFS, + kwargs={ + 'filenames_to_zip': filenames, + 'upload_filename': dvla_filename + }, + queue=QueueNames.PROCESS_FTP, + compression='zlib' ) - ) - notify_celery.send_task( - name=TaskNames.ZIP_AND_SEND_LETTER_PDFS, - kwargs={ - 'filenames_to_zip': filenames, - 'upload_filename': dvla_filename - }, - queue=QueueNames.PROCESS_FTP, - compression='zlib' - ) -def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): - letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline) +def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage): + letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage) letter_pdfs = [] for letter in letters_awaiting_sending: @@ -183,7 +184,7 @@ def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline): letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']}) except BotoClientError as e: current_app.logger.exception( - f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e) + f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}") return letter_pdfs diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 645bf9090..4c50c0972 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -731,7 +731,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ return notifications -def dao_get_letters_to_be_printed(print_run_deadline): +def dao_get_letters_to_be_printed(print_run_deadline, postage): """ Return all letters created before the print run deadline that have not yet been sent """ @@ -739,7 +739,8 @@ def dao_get_letters_to_be_printed(print_run_deadline): Notification.created_at < convert_bst_to_utc(print_run_deadline), Notification.notification_type == LETTER_TYPE, Notification.status == NOTIFICATION_CREATED, - Notification.key_type == KEY_TYPE_NORMAL + Notification.key_type == KEY_TYPE_NORMAL, + Notification.postage == postage ).order_by( Notification.created_at ).all()