Merge pull request #1802 from alphagov/rc_add_virus_scan_error_task

Added a new task to handle any error cases with the anti-virus

app/celery/letters_pdf_tasks.py

@@ -22,16 +22,16 @@ from app.dao.notifications_dao import (
     dao_update_notifications_by_reference,
 )
 from app.letters.utils import (
-    delete_pdf_from_letters_scan_bucket,
     get_reference_from_filename,
     move_scanned_pdf_to_test_or_live_pdf_bucket,
-    upload_letter_pdf
-)
+    upload_letter_pdf,
+    move_failed_pdf, ScanErrorType)
 from app.models import (
     KEY_TYPE_TEST,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
     NOTIFICATION_VIRUS_SCAN_FAILED,
+    NOTIFICATION_TECHNICAL_FAILURE
 )
 
 
@@ -180,7 +180,7 @@ def process_virus_scan_passed(filename):
 @notify_celery.task(name='process-virus-scan-failed')
 def process_virus_scan_failed(filename):
     current_app.logger.exception('Virus scan failed: {}'.format(filename))
-    delete_pdf_from_letters_scan_bucket(filename)
+    move_failed_pdf(filename, ScanErrorType.FAILURE)
     reference = get_reference_from_filename(filename)
     updated_count = update_letter_pdf_status(reference, NOTIFICATION_VIRUS_SCAN_FAILED)
 
@@ -192,6 +192,21 @@ def process_virus_scan_failed(filename):
         )
 
 
+@notify_celery.task(name='process-virus-scan-error')
+def process_virus_scan_error(filename):
+    current_app.logger.exception('Virus scan error: {}'.format(filename))
+    move_failed_pdf(filename, ScanErrorType.ERROR)
+    reference = get_reference_from_filename(filename)
+    updated_count = update_letter_pdf_status(reference, NOTIFICATION_TECHNICAL_FAILURE)
+
+    if updated_count != 1:
+        raise Exception(
+            "There should only be one letter notification for each reference. Found {} notifications".format(
+                updated_count
+            )
+        )
+
+
 def update_letter_pdf_status(reference, status):
     return dao_update_notifications_by_reference(
         references=[reference],

app/letters/utils.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta
+from enum import Enum
 
 import boto3
 from flask import current_app
@@ -9,6 +10,11 @@ from app.models import KEY_TYPE_TEST
 from app.variables import Retention
 
 
+class ScanErrorType(Enum):
+    ERROR = 1
+    FAILURE = 2
+
+
 LETTERS_PDF_FILE_LOCATION_STRUCTURE = \
     '{folder}NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
 
@@ -85,36 +91,22 @@ def upload_letter_pdf(notification, pdf_data):
     return upload_file_name
 
 
-def move_scanned_pdf_to_test_or_live_pdf_bucket(filename, is_test_letter=False):
+def move_scanned_pdf_to_test_or_live_pdf_bucket(source_filename, is_test_letter=False):
     source_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
     target_bucket_config = 'TEST_LETTERS_BUCKET_NAME' if is_test_letter else 'LETTERS_PDF_BUCKET_NAME'
     target_bucket_name = current_app.config[target_bucket_config]
 
-    s3 = boto3.resource('s3')
-    copy_source = {'Bucket': source_bucket_name, 'Key': filename}
-    target_filename = get_folder_name(datetime.utcnow(), is_test_letter) + filename
+    target_filename = get_folder_name(datetime.utcnow(), is_test_letter) + source_filename
 
-    target_bucket = s3.Bucket(target_bucket_name)
-    obj = target_bucket.Object(target_filename)
-
-    # Tags are copied across but the expiration time is reset in the destination bucket
-    # e.g. if a file has 5 days left to expire on a ONE_WEEK retention in the source bucket,
-    # in the destination bucket the expiration time will be reset to 7 days left to expire
-    obj.copy(copy_source, ExtraArgs={'ServerSideEncryption': 'AES256'})
-
-    s3.Object(source_bucket_name, filename).delete()
-
-    current_app.logger.info("Moved letter PDF: {}/{} to {}/{}".format(
-        source_bucket_name, filename, target_bucket_name, target_filename))
+    _move_s3_object(source_bucket_name, source_filename, target_bucket_name, target_filename)
 
 
-def delete_pdf_from_letters_scan_bucket(filename):
-    bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
+def move_failed_pdf(source_filename, scan_error_type):
+    scan_bucket = current_app.config['LETTERS_SCAN_BUCKET_NAME']
 
-    s3 = boto3.resource('s3')
-    s3.Object(bucket_name, filename).delete()
+    target_filename = ('ERROR/' if scan_error_type == ScanErrorType.ERROR else 'FAILURE/') + source_filename
 
-    current_app.logger.info("Deleted letter PDF: {}/{}".format(bucket_name, filename))
+    _move_s3_object(scan_bucket, source_filename, scan_bucket, target_filename)
 
 
 def get_letter_pdf(notification):
@@ -135,3 +127,21 @@ def get_letter_pdf(notification):
         file_content = obj.get()["Body"].read()
 
     return file_content
+
+
+def _move_s3_object(source_bucket, source_filename, target_bucket, target_filename):
+    s3 = boto3.resource('s3')
+    copy_source = {'Bucket': source_bucket, 'Key': source_filename}
+
+    target_bucket = s3.Bucket(target_bucket)
+    obj = target_bucket.Object(target_filename)
+
+    # Tags are copied across but the expiration time is reset in the destination bucket
+    # e.g. if a file has 5 days left to expire on a ONE_WEEK retention in the source bucket,
+    # in the destination bucket the expiration time will be reset to 7 days left to expire
+    obj.copy(copy_source, ExtraArgs={'ServerSideEncryption': 'AES256'})
+
+    s3.Object(source_bucket, source_filename).delete()
+
+    current_app.logger.info("Moved letter PDF: {}/{} to {}/{}".format(
+        source_bucket, source_filename, target_bucket, target_filename))