diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index ad0c767c2..fa42d0716 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -55,32 +55,6 @@ from app.models import ( ) -@notify_celery.task(bind=True, name="update-billable-units-for-letter", max_retries=15, default_retry_delay=300) -def update_billable_units_for_letter(self, notification_id, page_count): - notification = get_notification_by_id(notification_id, _raise=True) - - billable_units = get_billable_units_for_letter_page_count(page_count) - - if notification.key_type != KEY_TYPE_TEST: - notification.billable_units = billable_units - dao_update_notification(notification) - - current_app.logger.info( - f"Letter notification id: {notification_id} reference {notification.reference}: " - f"billable units set to {billable_units}" - ) - - -@notify_celery.task( - bind=True, name="update-validation-failed-for-templated-letter", max_retries=15, default_retry_delay=300 -) -def update_validation_failed_for_templated_letter(self, notification_id, page_count): - notification = get_notification_by_id(notification_id, _raise=True) - notification.status = NOTIFICATION_VALIDATION_FAILED - dao_update_notification(notification) - current_app.logger.info(f"Validation failed: letter is too long {page_count} for letter with id: {notification_id}") - - @notify_celery.task(bind=True, name='sanitise-letter', max_retries=15, default_retry_delay=300) def sanitise_letter(self, filename): try: @@ -117,247 +91,3 @@ def sanitise_letter(self, filename): "Notification has been updated to technical-failure".format(notification.id) update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message) - - -@notify_celery.task(bind=True, name='process-sanitised-letter', max_retries=15, default_retry_delay=300) -def process_sanitised_letter(self, sanitise_data): - letter_details = encryption.decrypt(sanitise_data) - - filename = letter_details['filename'] - notification_id = letter_details['notification_id'] - - current_app.logger.info('Processing sanitised letter with id {}'.format(notification_id)) - notification = get_notification_by_id(notification_id, _raise=True) - - if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK: - current_app.logger.info( - 'process-sanitised-letter task called for notification {} which is in {} state'.format( - notification.id, notification.status) - ) - return - - try: - original_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename) - - if letter_details['validation_status'] == 'failed': - current_app.logger.info('Processing invalid precompiled pdf with id {} (file {})'.format( - notification_id, filename)) - - _move_invalid_letter_and_update_status( - notification=notification, - filename=filename, - scan_pdf_object=original_pdf_object, - message=letter_details['message'], - invalid_pages=letter_details['invalid_pages'], - page_count=letter_details['page_count'], - ) - return - - current_app.logger.info('Processing valid precompiled pdf with id {} (file {})'.format( - notification_id, filename)) - - billable_units = get_billable_units_for_letter_page_count(letter_details['page_count']) - is_test_key = notification.key_type == KEY_TYPE_TEST - - # Updating the notification needs to happen before the file is moved. This is so that if updating the - # notification fails, the task can retry because the file is in the same place. - update_letter_pdf_status( - reference=notification.reference, - status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED, - billable_units=billable_units, - recipient_address=letter_details['address'] - ) - - # The original filename could be wrong because we didn't know the postage. - # Now we know if the letter is international, we can check what the filename should be. - upload_file_name = generate_letter_pdf_filename( - reference=notification.reference, - created_at=notification.created_at, - ignore_folder=True, - postage=notification.postage - ) - - move_sanitised_letter_to_test_or_live_pdf_bucket( - filename, - is_test_key, - notification.created_at, - upload_file_name, - ) - # We've moved the sanitised PDF from the sanitise bucket, but still need to delete the original file: - original_pdf_object.delete() - - except BotoClientError: - # Boto exceptions are likely to be caused by the file(s) being in the wrong place, so retrying won't help - - # we'll need to manually investigate - current_app.logger.exception( - f"Boto error when processing sanitised letter for notification {notification.id} (file {filename})" - ) - update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE) - raise NotificationTechnicalFailureException - except Exception: - try: - current_app.logger.exception( - "RETRY: calling process_sanitised_letter task for notification {} failed".format(notification.id) - ) - self.retry(queue=QueueNames.RETRY) - except self.MaxRetriesExceededError: - message = "RETRY FAILED: Max retries reached. " \ - "The task process_sanitised_letter failed for notification {}. " \ - "Notification has been updated to technical-failure".format(notification.id) - update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE) - raise NotificationTechnicalFailureException(message) - - -def _move_invalid_letter_and_update_status( - *, notification, filename, scan_pdf_object, message=None, invalid_pages=None, page_count=None -): - try: - move_scan_to_invalid_pdf_bucket( - source_filename=filename, - message=message, - invalid_pages=invalid_pages, - page_count=page_count - ) - scan_pdf_object.delete() - - update_letter_pdf_status( - reference=notification.reference, - status=NOTIFICATION_VALIDATION_FAILED, - billable_units=0) - except BotoClientError: - current_app.logger.exception( - "Error when moving letter with id {} to invalid PDF bucket".format(notification.id) - ) - update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE) - raise NotificationTechnicalFailureException - - -@notify_celery.task(name='process-virus-scan-failed') -def process_virus_scan_failed(filename): - move_failed_pdf(filename, ScanErrorType.FAILURE) - reference = get_reference_from_filename(filename) - notification = dao_get_notification_by_reference(reference) - updated_count = update_letter_pdf_status(reference, NOTIFICATION_VIRUS_SCAN_FAILED, billable_units=0) - - if updated_count != 1: - raise Exception( - "There should only be one letter notification for each reference. Found {} notifications".format( - updated_count - ) - ) - - error = VirusScanError('notification id {} Virus scan failed: {}'.format(notification.id, filename)) - current_app.logger.exception(error) - raise error - - -@notify_celery.task(name='process-virus-scan-error') -def process_virus_scan_error(filename): - move_failed_pdf(filename, ScanErrorType.ERROR) - reference = get_reference_from_filename(filename) - notification = dao_get_notification_by_reference(reference) - updated_count = update_letter_pdf_status(reference, NOTIFICATION_TECHNICAL_FAILURE, billable_units=0) - - if updated_count != 1: - raise Exception( - "There should only be one letter notification for each reference. Found {} notifications".format( - updated_count - ) - ) - error = VirusScanError('notification id {} Virus scan error: {}'.format(notification.id, filename)) - current_app.logger.exception(error) - raise error - - -def update_letter_pdf_status(reference, status, billable_units, recipient_address=None): - postage = None - if recipient_address: - # fix allow_international_letters - postage = PostalAddress(raw_address=recipient_address.replace(',', '\n'), - allow_international_letters=True - ).postage - postage = postage if postage in INTERNATIONAL_POSTAGE_TYPES else None - update_dict = {'status': status, 'billable_units': billable_units, 'updated_at': datetime.utcnow()} - if postage: - update_dict.update({'postage': postage, 'international': True}) - if recipient_address: - update_dict['to'] = recipient_address - update_dict['normalised_to'] = ''.join(recipient_address.split()).lower() - return dao_update_notifications_by_reference( - references=[reference], - update_dict=update_dict)[0] - - -def replay_letters_in_error(filename=None): - # This method can be used to replay letters that end up in the ERROR directory. - # We had an incident where clamAV was not processing the virus scan. - if filename: - move_error_pdf_to_scan_bucket(filename) - # call task to add the filename to anti virus queue - current_app.logger.info("Calling scan_file for: {}".format(filename)) - - if current_app.config['ANTIVIRUS_ENABLED']: - notify_celery.send_task( - name=TaskNames.SCAN_FILE, - kwargs={'filename': filename}, - queue=QueueNames.ANTIVIRUS, - ) - else: - # stub out antivirus in dev - sanitise_letter.apply_async( - [filename], - queue=QueueNames.LETTERS - ) - else: - error_files = get_file_names_from_error_bucket() - for item in error_files: - moved_file_name = item.key.split('/')[1] - current_app.logger.info("Calling scan_file for: {}".format(moved_file_name)) - move_error_pdf_to_scan_bucket(moved_file_name) - # call task to add the filename to anti virus queue - if current_app.config['ANTIVIRUS_ENABLED']: - notify_celery.send_task( - name=TaskNames.SCAN_FILE, - kwargs={'filename': moved_file_name}, - queue=QueueNames.ANTIVIRUS, - ) - else: - # stub out antivirus in dev - sanitise_letter.apply_async( - [filename], - queue=QueueNames.LETTERS - ) - - -@notify_celery.task(name='resanitise-pdf') -def resanitise_pdf(notification_id): - """ - `notification_id` is the notification id for a PDF letter which was either uploaded or sent using the API. - - This task calls the `recreate_pdf_for_precompiled_letter` template preview task which recreates the - PDF for a letter which is already sanitised and in the letters-pdf bucket. The new file that is generated - will then overwrite the existing letter in the letters-pdf bucket. - """ - notification = get_notification_by_id(notification_id) - - # folder_name is the folder that the letter is in the letters-pdf bucket e.g. '2021-10-10/' - folder_name = get_folder_name(notification.created_at) - - filename = generate_letter_pdf_filename( - reference=notification.reference, - created_at=notification.created_at, - ignore_folder=True, - postage=notification.postage - ) - - notify_celery.send_task( - name=TaskNames.RECREATE_PDF_FOR_PRECOMPILED_LETTER, - kwargs={ - 'notification_id': str(notification.id), - 'file_location': f'{folder_name}{filename}', - 'allow_international_letters': notification.service.has_permission( - INTERNATIONAL_LETTERS - ), - }, - queue=QueueNames.SANITISE_LETTERS, - ) diff --git a/app/commands.py b/app/commands.py index bf2c6353b..c7a9b1dea 100644 --- a/app/commands.py +++ b/app/commands.py @@ -19,7 +19,6 @@ from sqlalchemy.orm.exc import NoResultFound from app import db from app.aws import s3 -from app.celery.letters_pdf_tasks import resanitise_pdf from app.celery.tasks import process_row, record_daily_sorted_counts from app.config import QueueNames from app.dao.annual_billing_dao import ( @@ -159,14 +158,6 @@ def insert_inbound_numbers_from_file(file_name): db.session.commit() -@notify_command(name='recreate-pdf-for-precompiled-or-uploaded-letter') -@click.option('-n', '--notification_id', type=click.UUID, required=True, - help="Notification ID of the precompiled or uploaded letter") -def recreate_pdf_for_precompiled_or_uploaded_letter(notification_id): - print(f"Call resanitise_pdf task for notification: {notification_id}") - resanitise_pdf.apply_async([str(notification_id)], queue=QueueNames.LETTERS) - - def setup_commands(application): application.cli.add_command(command_group) diff --git a/app/config.py b/app/config.py index 2e520267c..394434fa6 100644 --- a/app/config.py +++ b/app/config.py @@ -26,7 +26,6 @@ class QueueNames(object): LETTERS = 'letter-tasks' SMS_CALLBACKS = 'sms-callbacks' ANTIVIRUS = 'antivirus-tasks' - SANITISE_LETTERS = 'sanitise-letter-tasks' SAVE_API_EMAIL = 'save-api-email-tasks' SAVE_API_SMS = 'save-api-sms-tasks'