diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index cd7a18021..dafb126b3 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -126,168 +126,6 @@ def update_validation_failed_for_templated_letter(self, notification_id, page_co current_app.logger.info(f"Validation failed: letter is too long {page_count} for letter with id: {notification_id}") -@notify_celery.task(name='collate-letter-pdfs-to-be-sent') -@cronitor("collate-letter-pdfs-to-be-sent") -def collate_letter_pdfs_to_be_sent(): - """ - Finds all letters which are still waiting to be sent to DVLA for printing - - This would usually be run at 5.50pm and collect up letters created between before 5:30pm today - that have not yet been sent. - If run after midnight, it will collect up letters created before 5:30pm the day before. - """ - print_run_date = convert_utc_to_local_timezone(datetime.utcnow()) - if print_run_date.time() < LETTER_PROCESSING_DEADLINE: - print_run_date = print_run_date - timedelta(days=1) - - print_run_deadline = print_run_date.replace( - hour=17, minute=30, second=0, microsecond=0 - ) - _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline) - - for postage in POSTAGE_TYPES: - current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}") - letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage) - - for i, letters in enumerate(group_letters(letters_to_print)): - filenames = [letter['Key'] for letter in letters] - - service_id = letters[0]['ServiceId'] - organisation_id = letters[0]['OrganisationId'] - - hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() - # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP - dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.{service_id}.{organisation_id}.ZIP'.format( - date=print_run_deadline.strftime("%Y-%m-%d"), - postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage], - num=i + 1, - hash=hash, - service_id=service_id, - organisation_id=organisation_id - ) - - current_app.logger.info( - 'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format( - len(filenames), - dvla_filename, - sum(letter['Size'] for letter in letters) - ) - ) - notify_celery.send_task( - name=TaskNames.ZIP_AND_SEND_LETTER_PDFS, - kwargs={ - 'filenames_to_zip': filenames, - 'upload_filename': dvla_filename - }, - queue=QueueNames.PROCESS_FTP, - compression='zlib' - ) - current_app.logger.info(f"finished collate-letter-pdfs-to-be-sent processing for postage class {postage}") - - current_app.logger.info("finished collate-letter-pdfs-to-be-sent") - - -def _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline): - letters_volumes = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline) - send_letters_volume_email_to_dvla(letters_volumes, print_run_deadline.date()) - - -def send_letters_volume_email_to_dvla(letters_volumes, date): - personalisation = { - 'total_volume': 0, - 'first_class_volume': 0, - 'second_class_volume': 0, - 'international_volume': 0, - 'total_sheets': 0, - 'first_class_sheets': 0, - "second_class_sheets": 0, - 'international_sheets': 0, - 'date': date.strftime("%d %B %Y") - } - for item in letters_volumes: - personalisation['total_volume'] += item.letters_count - personalisation['total_sheets'] += item.sheets_count - if f"{item.postage}_class_volume" in personalisation: - personalisation[f"{item.postage}_class_volume"] = item.letters_count - personalisation[f"{item.postage}_class_sheets"] = item.sheets_count - else: - personalisation["international_volume"] += item.letters_count - personalisation["international_sheets"] += item.sheets_count - - template = dao_get_template_by_id(current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID']) - recipients = current_app.config['DVLA_EMAIL_ADDRESSES'] - reply_to = template.service.get_default_reply_to_email_address() - service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID']) - - # avoid circular imports: - from app.notifications.process_notifications import ( - persist_notification, - send_notification_to_queue, - ) - for recipient in recipients: - saved_notification = persist_notification( - template_id=template.id, - template_version=template.version, - recipient=recipient, - service=service, - personalisation=personalisation, - notification_type=template.template_type, - api_key_id=None, - key_type=KEY_TYPE_NORMAL, - reply_to_text=reply_to - ) - - send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) - - -def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage): - letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage) - for letter in letters_awaiting_sending: - try: - letter_pdf = find_letter_pdf_in_s3(letter) - yield { - "Key": letter_pdf.key, - "Size": letter_pdf.size, - "ServiceId": str(letter.service_id), - "OrganisationId": str(letter.service.organisation_id) - } - except (BotoClientError, LetterPDFNotFound) as e: - current_app.logger.exception( - f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e) - - -def group_letters(letter_pdfs): - """ - Group letters in chunks of MAX_LETTER_PDF_ZIP_FILESIZE. Will add files to lists, never going over that size. - If a single file is (somehow) larger than MAX_LETTER_PDF_ZIP_FILESIZE that'll be in a list on it's own. - If there are no files, will just exit (rather than yielding an empty list). - """ - running_filesize = 0 - list_of_files = [] - service_id = None - for letter in letter_pdfs: - if letter['Key'].lower().endswith('.pdf'): - if not service_id: - service_id = letter['ServiceId'] - if ( - running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE'] - or len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP'] - or letter['ServiceId'] != service_id - ): - yield list_of_files - running_filesize = 0 - list_of_files = [] - service_id = None - - if not service_id: - service_id = letter['ServiceId'] - running_filesize += letter['Size'] - list_of_files.append(letter) - - if list_of_files: - yield list_of_files - - @notify_celery.task(bind=True, name='sanitise-letter', max_retries=15, default_retry_delay=300) def sanitise_letter(self, filename): try: diff --git a/app/config.py b/app/config.py index 2ec06aa69..bf477dbbb 100644 --- a/app/config.py +++ b/app/config.py @@ -300,11 +300,6 @@ class Config(object): }, # The collate-letter-pdf does assume it is called in an hour that BST does not make a # difference to the truncate date which translates to the filename to process - 'collate-letter-pdfs-to-be-sent': { - 'task': 'collate-letter-pdfs-to-be-sent', - 'schedule': crontab(hour=17, minute=50), - 'options': {'queue': QueueNames.PERIODIC} - }, 'raise-alert-if-no-letter-ack-file': { 'task': 'raise-alert-if-no-letter-ack-file', 'schedule': crontab(hour=23, minute=00),