From 4260576fb2231f8ebe07bbd3af60e0371a322f18 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 7 Dec 2022 11:55:24 -0500 Subject: [PATCH] remove pdf sanitize tasks --- app/celery/letters_pdf_tasks.py | 93 ------------------- app/celery/tasks.py | 2 +- app/config.py | 1 - app/letters/utils.py | 14 --- .../process_letter_notifications.py | 2 +- app/v2/notifications/post_notifications.py | 9 -- docs/queues-and-tasks.md | 4 - tests/app/letters/test_letter_utils.py | 61 ------------ .../test_send_notification.py | 2 +- 9 files changed, 3 insertions(+), 185 deletions(-) delete mode 100644 app/celery/letters_pdf_tasks.py diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py deleted file mode 100644 index fa42d0716..000000000 --- a/app/celery/letters_pdf_tasks.py +++ /dev/null @@ -1,93 +0,0 @@ -from base64 import urlsafe_b64encode -from datetime import datetime, timedelta -from hashlib import sha512 - -from botocore.exceptions import ClientError as BotoClientError -from flask import current_app -from notifications_utils.letter_timings import LETTER_PROCESSING_DEADLINE -from notifications_utils.postal_address import PostalAddress -from notifications_utils.timezones import convert_utc_to_local_timezone - -from app import encryption, notify_celery -from app.aws import s3 -from app.config import QueueNames, TaskNames -from app.cronitor import cronitor -from app.dao.notifications_dao import ( - dao_get_letters_and_sheets_volume_by_postage, - dao_get_letters_to_be_printed, - dao_get_notification_by_reference, - dao_update_notification, - dao_update_notifications_by_reference, - get_notification_by_id, - update_notification_status_by_id, -) -from app.dao.templates_dao import dao_get_template_by_id -from app.errors import VirusScanError -from app.exceptions import NotificationTechnicalFailureException -from app.letters.utils import ( - LetterPDFNotFound, - ScanErrorType, - find_letter_pdf_in_s3, - generate_letter_pdf_filename, - get_billable_units_for_letter_page_count, - get_file_names_from_error_bucket, - get_folder_name, - get_reference_from_filename, - move_error_pdf_to_scan_bucket, - move_failed_pdf, - move_sanitised_letter_to_test_or_live_pdf_bucket, - move_scan_to_invalid_pdf_bucket, -) -from app.models import ( - INTERNATIONAL_LETTERS, - INTERNATIONAL_POSTAGE_TYPES, - KEY_TYPE_NORMAL, - KEY_TYPE_TEST, - NOTIFICATION_CREATED, - NOTIFICATION_DELIVERED, - NOTIFICATION_PENDING_VIRUS_CHECK, - NOTIFICATION_TECHNICAL_FAILURE, - NOTIFICATION_VALIDATION_FAILED, - NOTIFICATION_VIRUS_SCAN_FAILED, - POSTAGE_TYPES, - RESOLVE_POSTAGE_FOR_FILE_NAME, - Service, -) - - -@notify_celery.task(bind=True, name='sanitise-letter', max_retries=15, default_retry_delay=300) -def sanitise_letter(self, filename): - try: - reference = get_reference_from_filename(filename) - notification = dao_get_notification_by_reference(reference) - - current_app.logger.info('Notification ID {} Virus scan passed: {}'.format(notification.id, filename)) - - if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK: - current_app.logger.info('Sanitise letter called for notification {} which is in {} state'.format( - notification.id, notification.status)) - return - - notify_celery.send_task( - name=TaskNames.SANITISE_LETTER, - kwargs={ - 'notification_id': str(notification.id), - 'filename': filename, - 'allow_international_letters': notification.service.has_permission( - INTERNATIONAL_LETTERS - ), - }, - queue=QueueNames.SANITISE_LETTERS, - ) - except Exception: - try: - current_app.logger.exception( - "RETRY: calling sanitise_letter task for notification {} failed".format(notification.id) - ) - self.retry(queue=QueueNames.RETRY) - except self.MaxRetriesExceededError: - message = "RETRY FAILED: Max retries reached. " \ - "The task sanitise_letter failed for notification {}. " \ - "Notification has been updated to technical-failure".format(notification.id) - update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE) - raise NotificationTechnicalFailureException(message) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 676e71904..2880a6726 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -12,7 +12,7 @@ from sqlalchemy.exc import IntegrityError, SQLAlchemyError from app import create_random_identifier, create_uuid, encryption, notify_celery from app.aws import s3 -from app.celery import letters_pdf_tasks, provider_tasks, research_mode_tasks +from app.celery import provider_tasks, research_mode_tasks from app.config import QueueNames from app.dao.daily_sorted_letter_dao import ( dao_create_or_update_daily_sorted_letter, diff --git a/app/config.py b/app/config.py index 394434fa6..7baeed5a6 100644 --- a/app/config.py +++ b/app/config.py @@ -56,7 +56,6 @@ class TaskNames(object): PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs' ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs' SCAN_FILE = 'scan-file' - SANITISE_LETTER = 'sanitise-and-upload-letter' CREATE_PDF_FOR_TEMPLATED_LETTER = 'create-pdf-for-templated-letter' RECREATE_PDF_FOR_PRECOMPILED_LETTER = 'recreate-pdf-for-precompiled-letter' diff --git a/app/letters/utils.py b/app/letters/utils.py index 564549eb9..797e461b6 100644 --- a/app/letters/utils.py +++ b/app/letters/utils.py @@ -161,20 +161,6 @@ def move_uploaded_pdf_to_letters_bucket(source_filename, upload_filename): ) -def move_sanitised_letter_to_test_or_live_pdf_bucket(filename, is_test_letter, created_at, new_filename): - target_bucket_config = 'TEST_LETTERS_BUCKET_NAME' if is_test_letter else 'LETTERS_PDF_BUCKET_NAME' - target_bucket_name = current_app.config[target_bucket_config] - target_folder = '' if is_test_letter else get_folder_name(created_at) - target_filename = target_folder + new_filename - - _move_s3_object( - source_bucket=current_app.config['LETTER_SANITISE_BUCKET_NAME'], - source_filename=filename, - target_bucket=target_bucket_name, - target_filename=target_filename, - ) - - def get_file_names_from_error_bucket(): s3 = boto3.resource('s3') scan_bucket = current_app.config['LETTERS_SCAN_BUCKET_NAME'] diff --git a/app/notifications/process_letter_notifications.py b/app/notifications/process_letter_notifications.py index 3cce47433..3fe7e7cbf 100644 --- a/app/notifications/process_letter_notifications.py +++ b/app/notifications/process_letter_notifications.py @@ -33,7 +33,7 @@ def create_letter_notification( status=status, reply_to_text=reply_to_text, billable_units=billable_units, - # letter_data.get('postage') is only set for precompiled letters (if international it is set after sanitise) + # letter_data.get('postage') is only set for precompiled letters # letters from a template will pass in 'europe' or 'rest-of-world' if None then use postage from template postage=postage or letter_data.get('postage') or template.postage, updated_at=updated_at diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index 2e51e639c..f47fb7e22 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -15,9 +15,6 @@ from app import ( encryption, notify_celery, ) -from app.celery.letters_pdf_tasks import ( - sanitise_letter, -) from app.celery.research_mode_tasks import create_fake_letter_response_file from app.celery.tasks import save_api_email, save_api_sms from app.clients.document_download import DocumentDownloadError @@ -433,12 +430,6 @@ def process_precompiled_letter_notifications(*, letter_data, api_key, service, t kwargs={'filename': filename}, queue=QueueNames.ANTIVIRUS, ) - else: - # stub out antivirus in dev - sanitise_letter.apply_async( - [filename], - queue=QueueNames.LETTERS - ) return resp diff --git a/docs/queues-and-tasks.md b/docs/queues-and-tasks.md index 3e40cde4a..675e50dd0 100644 --- a/docs/queues-and-tasks.md +++ b/docs/queues-and-tasks.md @@ -19,7 +19,6 @@ There are a bunch of queues: - letter tasks - sms callbacks - antivirus tasks -- sanitise letter tasks - save api email tasks - save api sms tasks @@ -49,7 +48,6 @@ And these tasks: - process incomplete jobs - process job - process returned letters list -- process sanitised letter - process ses result - process virus scan error - process virus scan failed @@ -59,9 +57,7 @@ And these tasks: - remove letter jobs - remove sms email jobs - replay created notifications -- resanitise pdf - run scheduled jobs -- sanitise letter - save api email - save api sms - save daily notification processing time diff --git a/tests/app/letters/test_letter_utils.py b/tests/app/letters/test_letter_utils.py index deb3639f3..71c8ce59a 100644 --- a/tests/app/letters/test_letter_utils.py +++ b/tests/app/letters/test_letter_utils.py @@ -18,7 +18,6 @@ from app.letters.utils import ( get_letter_pdf_and_metadata, letter_print_day, move_failed_pdf, - move_sanitised_letter_to_test_or_live_pdf_bucket, upload_letter_pdf, ) from app.models import ( @@ -353,66 +352,6 @@ def test_get_folder_name_in_british_summer_time(notify_api, timestamp, expected_ assert folder_name == expected_folder_name -@mock_s3 -def test_move_sanitised_letter_to_live_pdf_bucket(notify_api, mocker): - filename = 'my_letter.pdf' - source_bucket_name = current_app.config['LETTER_SANITISE_BUCKET_NAME'] - target_bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME'] - - conn = boto3.resource('s3', region_name='eu-west-1') - source_bucket = conn.create_bucket( - Bucket=source_bucket_name, - CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'} - ) - target_bucket = conn.create_bucket( - Bucket=target_bucket_name, - CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'} - ) - - s3 = boto3.client('s3', region_name='eu-west-1') - s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b'pdf_content') - - move_sanitised_letter_to_test_or_live_pdf_bucket( - filename=filename, - is_test_letter=False, - created_at=datetime.utcnow(), - new_filename=filename - ) - - assert not [x for x in source_bucket.objects.all()] - assert len([x for x in target_bucket.objects.all()]) == 1 - - -@mock_s3 -def test_move_sanitised_letter_to_test_pdf_bucket(notify_api, mocker): - filename = 'my_letter.pdf' - source_bucket_name = current_app.config['LETTER_SANITISE_BUCKET_NAME'] - target_bucket_name = current_app.config['TEST_LETTERS_BUCKET_NAME'] - - conn = boto3.resource('s3', region_name='eu-west-1') - source_bucket = conn.create_bucket( - Bucket=source_bucket_name, - CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'} - ) - target_bucket = conn.create_bucket( - Bucket=target_bucket_name, - CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'} - ) - - s3 = boto3.client('s3', region_name='eu-west-1') - s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b'pdf_content') - - move_sanitised_letter_to_test_or_live_pdf_bucket( - filename=filename, - is_test_letter=True, - created_at=datetime.utcnow(), - new_filename=filename - ) - - assert not [x for x in source_bucket.objects.all()] - assert len([x for x in target_bucket.objects.all()]) == 1 - - @freeze_time('2017-07-07 20:00:00') def test_letter_print_day_returns_today_if_letter_was_printed_after_1730_yesterday(): created_at = datetime(2017, 7, 6, 17, 30) diff --git a/tests/app/service/send_notification/test_send_notification.py b/tests/app/service/send_notification/test_send_notification.py index ced70f647..94d512f3d 100644 --- a/tests/app/service/send_notification/test_send_notification.py +++ b/tests/app/service/send_notification/test_send_notification.py @@ -1229,7 +1229,7 @@ def test_post_notification_should_set_reply_to_text(client, sample_service, mock assert notifications[0].reply_to_text == expected_reply_to -@pytest.mark.skip(reason="Rewrite without letters") +@pytest.mark.skip(reason="Rewrite without letters?") @pytest.mark.parametrize('reference_paceholder,', [None, 'ref2']) def test_send_notification_should_set_client_reference_from_placeholder( sample_letter_template, mocker, reference_paceholder