Sanitise uploaded letters and store in S3

This sanitises uploaded letters and stores the sanitised result in S3
with if it passes validation or the original PDF in S3 if validation
fails. A metadata value of 'status' is set to either 'valid' or
'invalid'.

app/config.py

@@ -72,6 +72,7 @@ class Config(object):
     NOTIFY_ENVIRONMENT = 'development'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-local'
     MOU_BUCKET_NAME = 'local-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'local-transient-uploaded-letters'
     ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
     ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
     CHECK_PROXY_HEADER = False
@@ -94,6 +95,7 @@ class Development(Config):
     CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-tools'
     MOU_BUCKET_NAME = 'notify.tools-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
 
     ADMIN_CLIENT_SECRET = 'dev-notify-secret-key'
     API_HOST_NAME = 'http://localhost:6011'
@@ -115,6 +117,7 @@ class Test(Development):
     CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test'
     MOU_BUCKET_NAME = 'test-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
     NOTIFY_ENVIRONMENT = 'test'
     API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to'
     TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'
@@ -132,6 +135,7 @@ class Preview(Config):
     CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview'
     MOU_BUCKET_NAME = 'notify.works-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
     NOTIFY_ENVIRONMENT = 'preview'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.notify.works'
@@ -146,6 +150,7 @@ class Staging(Config):
     CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging'
     MOU_BUCKET_NAME = 'staging-notify.works-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
     NOTIFY_ENVIRONMENT = 'staging'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.staging-notify.works'
@@ -160,6 +165,7 @@ class Live(Config):
     CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-production'
     MOU_BUCKET_NAME = 'notifications.service.gov.uk-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
     NOTIFY_ENVIRONMENT = 'live'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.notifications.service.gov.uk'

app/main/views/uploads.py

@@ -11,11 +11,17 @@ from flask import (
 )
 from notifications_utils.pdf import pdf_page_count
 from PyPDF2.utils import PdfReadError
+from requests import RequestException
 
 from app import current_service
 from app.extensions import antivirus_client
 from app.main import main
 from app.main.forms import PDFUploadForm
+from app.s3_client.s3_letter_upload_client import (
+    get_transient_letter_file_location,
+    upload_letter_to_s3,
+)
+from app.template_previews import sanitise_letter
 from app.utils import user_has_permissions
 
 MAX_FILE_UPLOAD_SIZE = 2 * 1024 * 1024  # 2MB
@@ -49,6 +55,20 @@ def upload_letter(service_id):
             return invalid_upload_error('Your file must be a valid PDF')
 
         upload_id = uuid.uuid4()
+        file_location = get_transient_letter_file_location(service_id, upload_id)
+
+        try:
+            response = sanitise_letter(BytesIO(pdf_file_bytes))
+            response.raise_for_status()
+        except RequestException as ex:
+            if ex.response is not None and ex.response.status_code == 400:
+                status = 'invalid'
+                upload_letter_to_s3(pdf_file_bytes, file_location, status)
+            else:
+                raise ex
+        else:
+            status = 'valid'
+            upload_letter_to_s3(response.content, file_location, status)
 
         return redirect(
             url_for(
@@ -56,6 +76,7 @@ def upload_letter(service_id):
                 service_id=current_service.id,
                 file_id=upload_id,
                 original_filename=form.file.data.filename,
+                status=status,
             )
         )
 
@@ -71,5 +92,6 @@ def invalid_upload_error(message):
 @user_has_permissions('send_messages')
 def uploaded_letter_preview(service_id, file_id):
     original_filename = request.args.get('original_filename')
+    status = request.args.get('status')
 
-    return render_template('views/uploads/preview.html', original_filename=original_filename)
+    return render_template('views/uploads/preview.html', original_filename=original_filename, status=status)

app/s3_client/s3_letter_upload_client.py

@@ -0,0 +1,16 @@
+from flask import current_app
+from notifications_utils.s3 import s3upload as utils_s3upload
+
+
+def get_transient_letter_file_location(service_id, upload_id):
+    return 'service-{}/{}.pdf'.format(service_id, upload_id)
+
+
+def upload_letter_to_s3(data, file_location, status):
+    utils_s3upload(
+        filedata=data,
+        region=current_app.config['AWS_REGION'],
+        bucket_name=current_app.config['TRANSIENT_UPLOADED_LETTERS'],
+        file_location=file_location,
+        metadata={'status': status}
+    )

app/template_previews.py

@@ -66,3 +66,11 @@ def validate_letter(pdf_file):
         data=pdf_file,
         headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
     )
+
+
+def sanitise_letter(pdf_file):
+    return requests.post(
+        '{}/precompiled/sanitise'.format(current_app.config['TEMPLATE_PREVIEW_API_HOST']),
+        data=pdf_file,
+        headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
+    )

app/templates/views/uploads/preview.html

@@ -11,4 +11,9 @@
         back_link=url_for('main.upload_letter', service_id=current_service.id)
     ) }}
 
+    {% if status == 'invalid' %}
+    <p class="notification-status-cancelled" id="validation-error-message">
+      Validation failed
+    </p>
+    {% endif %}
 {% endblock %}