Merge pull request #3121 from alphagov/upload-precompiled-letters

Upload precompiled letters

app/config.py

@@ -72,6 +72,7 @@ class Config(object):
     NOTIFY_ENVIRONMENT = 'development'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-local'
     MOU_BUCKET_NAME = 'local-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'local-transient-uploaded-letters'
     ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
     ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
     CHECK_PROXY_HEADER = False
@@ -94,6 +95,7 @@ class Development(Config):
     CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-tools'
     MOU_BUCKET_NAME = 'notify.tools-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
 
     ADMIN_CLIENT_SECRET = 'dev-notify-secret-key'
     API_HOST_NAME = 'http://localhost:6011'
@@ -115,6 +117,7 @@ class Test(Development):
     CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test'
     MOU_BUCKET_NAME = 'test-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
     NOTIFY_ENVIRONMENT = 'test'
     API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to'
     TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'
@@ -132,6 +135,7 @@ class Preview(Config):
     CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview'
     MOU_BUCKET_NAME = 'notify.works-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
     NOTIFY_ENVIRONMENT = 'preview'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.notify.works'
@@ -146,6 +150,7 @@ class Staging(Config):
     CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging'
     MOU_BUCKET_NAME = 'staging-notify.works-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
     NOTIFY_ENVIRONMENT = 'staging'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.staging-notify.works'
@@ -160,6 +165,7 @@ class Live(Config):
     CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-production'
     MOU_BUCKET_NAME = 'notifications.service.gov.uk-mou'
+    TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
     NOTIFY_ENVIRONMENT = 'live'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.notifications.service.gov.uk'

app/main/forms.py

@@ -1109,9 +1109,9 @@ class ServiceLetterBrandingDetails(StripWhitespaceForm):
 
 class PDFUploadForm(StripWhitespaceForm):
     file = FileField_wtf(
-        'Upload a letter in PDF format to check if it fits in the printable area',
+        'Upload a letter in PDF format',
         validators=[
-            FileAllowed(['pdf'], 'PDF documents only!'),
+            FileAllowed(['pdf'], 'Letters must be saved as a PDF'),
             DataRequired(message="You need to upload a file to submit")
         ]
     )

app/main/views/uploads.py

@@ -1,10 +1,161 @@
-from flask import render_template
+import base64
+import uuid
+from io import BytesIO
 
+from flask import (
+    abort,
+    current_app,
+    flash,
+    redirect,
+    render_template,
+    request,
+    url_for,
+)
+from notifications_utils.pdf import pdf_page_count
+from PyPDF2.utils import PdfReadError
+from requests import RequestException
+
+from app import current_service, notification_api_client, service_api_client
+from app.extensions import antivirus_client
 from app.main import main
-from app.utils import user_has_permissions
+from app.main.forms import PDFUploadForm
+from app.s3_client.s3_letter_upload_client import (
+    get_letter_pdf_and_metadata,
+    get_transient_letter_file_location,
+    upload_letter_to_s3,
+)
+from app.template_previews import TemplatePreview, sanitise_letter
+from app.utils import get_template, user_has_permissions
+
+MAX_FILE_UPLOAD_SIZE = 2 * 1024 * 1024  # 2MB
 
 
 @main.route("/services/<service_id>/uploads")
 @user_has_permissions('send_messages')
 def uploads(service_id):
     return render_template('views/uploads/index.html')
+
+
+@main.route("/services/<service_id>/upload-letter", methods=['GET', 'POST'])
+@user_has_permissions('send_messages')
+def upload_letter(service_id):
+    form = PDFUploadForm()
+
+    if form.validate_on_submit():
+        pdf_file_bytes = form.file.data.read()
+
+        virus_free = antivirus_client.scan(BytesIO(pdf_file_bytes))
+        if not virus_free:
+            return invalid_upload_error('Your file has failed the virus check')
+
+        if len(pdf_file_bytes) > MAX_FILE_UPLOAD_SIZE:
+            return invalid_upload_error('Your file must be smaller than 2MB')
+
+        try:
+            # TODO: get page count from the sanitise response once template preview handles malformed files nicely
+            page_count = pdf_page_count(BytesIO(pdf_file_bytes))
+        except PdfReadError:
+            current_app.logger.info('Invalid PDF uploaded for service_id: {}'.format(service_id))
+            return invalid_upload_error('Your file must be a valid PDF')
+
+        upload_id = uuid.uuid4()
+        file_location = get_transient_letter_file_location(service_id, upload_id)
+
+        try:
+            response = sanitise_letter(BytesIO(pdf_file_bytes))
+            response.raise_for_status()
+        except RequestException as ex:
+            if ex.response is not None and ex.response.status_code == 400:
+                status = 'invalid'
+                upload_letter_to_s3(pdf_file_bytes, file_location, status)
+            else:
+                raise ex
+        else:
+            status = 'valid'
+            file_contents = base64.b64decode(response.json()['file'].encode())
+            upload_letter_to_s3(file_contents, file_location, status)
+
+        return redirect(
+            url_for(
+                'main.uploaded_letter_preview',
+                service_id=current_service.id,
+                file_id=upload_id,
+                original_filename=form.file.data.filename,
+                page_count=page_count,
+                status=status,
+            )
+        )
+
+    return render_template('views/uploads/choose-file.html', form=form)
+
+
+def invalid_upload_error(message):
+    flash(message, 'dangerous')
+    return render_template('views/uploads/choose-file.html', form=PDFUploadForm()), 400
+
+
+@main.route("/services/<service_id>/preview-letter/<file_id>")
+@user_has_permissions('send_messages')
+def uploaded_letter_preview(service_id, file_id):
+    original_filename = request.args.get('original_filename')
+    page_count = request.args.get('page_count')
+    status = request.args.get('status')
+
+    template_dict = service_api_client.get_precompiled_template(service_id)
+
+    template = get_template(
+        template_dict,
+        service_id,
+        letter_preview_url=url_for(
+            '.view_letter_upload_as_preview',
+            service_id=service_id,
+            file_id=file_id
+        ),
+        page_count=page_count
+    )
+
+    return render_template(
+        'views/uploads/preview.html',
+        original_filename=original_filename,
+        template=template,
+        status=status,
+        file_id=file_id,
+    )
+
+
+@main.route("/services/<service_id>/preview-letter-image/<file_id>")
+@user_has_permissions('send_messages')
+def view_letter_upload_as_preview(service_id, file_id):
+    file_location = get_transient_letter_file_location(service_id, file_id)
+    pdf_file, metadata = get_letter_pdf_and_metadata(file_location)
+
+    page = request.args.get('page')
+
+    if metadata['status'] == 'invalid':
+        return TemplatePreview.from_invalid_pdf_file(pdf_file, page)
+    else:
+        return TemplatePreview.from_valid_pdf_file(pdf_file, page)
+
+
+@main.route("/services/<service_id>/upload-letter/send", methods=['POST'])
+@user_has_permissions('send_messages', restrict_admin_usage=True)
+def send_uploaded_letter(service_id):
+    filename = request.form['filename']
+    file_id = request.form['file_id']
+
+    if not (current_service.has_permission('letter') and current_service.has_permission('upload_letters')):
+        abort(403)
+
+    file_location = get_transient_letter_file_location(service_id, file_id)
+    _, metadata = get_letter_pdf_and_metadata(file_location)
+
+    if metadata.get('status') != 'valid':
+        abort(403)
+
+    notification_api_client.send_precompiled_letter(service_id, filename, file_id)
+
+    return redirect(url_for(
+        '.view_notification',
+        service_id=service_id,
+        notification_id=file_id,
+    ))

app/navigation.py

@@ -244,6 +244,7 @@ class HeaderNavigation(Navigation):
         'send_test',
         'send_test_preview',
         'send_test_step',
+        'send_uploaded_letter',
         'service_add_email_reply_to',
         'service_add_letter_contact',
         'service_add_sms_sender',
@@ -304,6 +305,8 @@ class HeaderNavigation(Navigation):
         'template_history',
         'template_usage',
         'trial_mode',
+        'upload_letter',
+        'uploaded_letter_preview',
         'uploads',
         'usage',
         'view_job',
@@ -312,6 +315,7 @@ class HeaderNavigation(Navigation):
         'view_jobs',
         'view_letter_notification_as_preview',
         'view_letter_template_preview',
+        'view_letter_upload_as_preview',
         'view_notification',
         'view_notification_updates',
         'view_notifications',
@@ -365,6 +369,8 @@ class MainNavigation(Navigation):
             'view_template_versions',
         },
         'uploads': {
+            'upload_letter',
+            'uploaded_letter_preview',
             'uploads',
         },
         'team-members': {
@@ -551,6 +557,7 @@ class MainNavigation(Navigation):
         'robots',
         'security',
         'send_notification',
+        'send_uploaded_letter',
         'service_dashboard_updates',
         'service_delete_email_reply_to',
         'service_delete_letter_contact',
@@ -603,6 +610,7 @@ class MainNavigation(Navigation):
         'view_job_updates',
         'view_letter_notification_as_preview',
         'view_letter_template_preview',
+        'view_letter_upload_as_preview',
         'view_notification_updates',
         'view_notifications_csv',
         'view_provider',
@@ -786,6 +794,7 @@ class CaseworkNavigation(Navigation):
         'send_messages',
         'send_notification',
         'send_test_preview',
+        'send_uploaded_letter',
         'service_add_email_reply_to',
         'service_add_letter_contact',
         'service_add_sms_sender',
@@ -859,6 +868,8 @@ class CaseworkNavigation(Navigation):
         'two_factor_email_sent',
         'update_email_branding',
         'update_letter_branding',
+        'upload_letter',
+        'uploaded_letter_preview',
         'uploads',
         'usage',
         'usage_for_all_services',
@@ -881,6 +892,7 @@ class CaseworkNavigation(Navigation):
         'view_job_updates',
         'view_letter_notification_as_preview',
         'view_letter_template_preview',
+        'view_letter_upload_as_preview',
         'view_notification_updates',
         'view_notifications_csv',
         'view_provider',
@@ -1065,6 +1077,7 @@ class OrgNavigation(Navigation):
         'send_test',
         'send_test_preview',
         'send_test_step',
+        'send_uploaded_letter',
         'service_add_email_reply_to',
         'service_add_letter_contact',
         'service_add_sms_sender',
@@ -1138,6 +1151,8 @@ class OrgNavigation(Navigation):
         'two_factor_email_sent',
         'update_email_branding',
         'update_letter_branding',
+        'upload_letter',
+        'uploaded_letter_preview',
         'uploads',
         'usage',
         'usage_for_all_services',
@@ -1162,6 +1177,7 @@ class OrgNavigation(Navigation):
         'view_jobs',
         'view_letter_notification_as_preview',
         'view_letter_template_preview',
+        'view_letter_upload_as_preview',
         'view_notification',
         'view_notification_updates',
         'view_notifications',

app/notify_client/notification_api_client.py

@@ -59,6 +59,14 @@ class NotificationApiClient(NotifyAdminAPIClient):
         data = _attach_current_user(data)
         return self.post(url='/service/{}/send-notification'.format(service_id), data=data)
 
+    def send_precompiled_letter(self, service_id, filename, file_id):
+        data = {
+            'filename': filename,
+            'file_id': file_id,
+        }
+        data = _attach_current_user(data)
+        return self.post(url='/service/{}/send-pdf-letter'.format(service_id), data=data)
+
     def get_notification(self, service_id, notification_id):
         return self.get(url='/service/{}/notifications/{}'.format(service_id, notification_id))
 

app/notify_client/service_api_client.py

@@ -261,6 +261,12 @@ class ServiceAPIClient(NotifyAdminAPIClient):
         )
         return self.get(endpoint)
 
+    def get_precompiled_template(self, service_id):
+        """
+        Returns the precompiled template for a service, creating it if it doesn't already exist
+        """
+        return self.get('/service/{}/template/precompiled'.format(service_id))
+
     @cache.set('service-{service_id}-templates')
     def get_service_templates(self, service_id):
         """

app/s3_client/s3_letter_upload_client.py

@@ -0,0 +1,27 @@
+from boto3 import resource
+from flask import current_app
+from notifications_utils.s3 import s3upload as utils_s3upload
+
+
+def get_transient_letter_file_location(service_id, upload_id):
+    return 'service-{}/{}.pdf'.format(service_id, upload_id)
+
+
+def upload_letter_to_s3(data, file_location, status):
+    utils_s3upload(
+        filedata=data,
+        region=current_app.config['AWS_REGION'],
+        bucket_name=current_app.config['TRANSIENT_UPLOADED_LETTERS'],
+        file_location=file_location,
+        metadata={'status': status}
+    )
+
+
+def get_letter_pdf_and_metadata(file_location):
+    s3 = resource('s3')
+    s3_object = s3.Object(current_app.config['TRANSIENT_UPLOADED_LETTERS'], file_location).get()
+
+    pdf = s3_object['Body'].read()
+    metadata = s3_object['Metadata']
+
+    return pdf, metadata

app/template_previews.py

@@ -1,5 +1,9 @@
+import base64
+from io import BytesIO
+
 import requests
 from flask import current_app, json
+from notifications_utils.pdf import extract_page_from_pdf
 
 from app import current_service
 
@@ -24,6 +28,36 @@ class TemplatePreview:
         )
         return (resp.content, resp.status_code, resp.headers.items())
 
+    @classmethod
+    def from_valid_pdf_file(cls, pdf_file, page):
+        pdf_page = extract_page_from_pdf(BytesIO(pdf_file), int(page) - 1)
+
+        response = requests.post(
+            '{}/precompiled-preview.png{}'.format(
+                current_app.config['TEMPLATE_PREVIEW_API_HOST'],
+                '?hide_notify=true' if page == '1' else ''
+            ),
+            data=base64.b64encode(pdf_page).decode('utf-8'),
+            headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
+        )
+
+        return (response.content, response.status_code, response.headers.items())
+
+    @classmethod
+    def from_invalid_pdf_file(cls, pdf_file, page):
+        pdf_page = extract_page_from_pdf(BytesIO(pdf_file), int(page) - 1)
+
+        response = requests.post(
+            '{}/precompiled/overlay.png{}'.format(
+                current_app.config['TEMPLATE_PREVIEW_API_HOST'],
+                '?page_number={}'.format(page)
+            ),
+            data=pdf_page,
+            headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
+        )
+
+        return (response.content, response.status_code, response.headers.items())
+
     @classmethod
     def from_example_template(cls, template, filename):
         data = {
@@ -66,3 +100,11 @@ def validate_letter(pdf_file):
         data=pdf_file,
         headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
     )
+
+
+def sanitise_letter(pdf_file):
+    return requests.post(
+        '{}/precompiled/sanitise'.format(current_app.config['TEMPLATE_PREVIEW_API_HOST']),
+        data=pdf_file,
+        headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
+    )

app/templates/views/uploads/choose-file.html

@@ -0,0 +1,28 @@
+{% extends "withnav_template.html" %}
+{% from "components/file-upload.html" import file_upload %}
+{% from "components/page-header.html" import page_header %}
+
+{% block service_page_title %}
+  Upload a letter
+{% endblock %}
+
+{% block maincolumn_content %}
+<div class="grid-row">
+  <div class="column-five-sixths">
+    {{ page_header(
+        'Upload a letter',
+        back_link=url_for('main.uploads', service_id=current_service.id)
+    ) }}
+
+  <p>
+    {{ file_upload(
+      form.file,
+      action = url_for('main.upload_letter', service_id=current_service.id),
+    )}}
+  </p>
+  <p>You can upload a single letter as a PDF.</p>
+  <p>Your file must meet our <a href="https://docs.notifications.service.gov.uk/documentation/images/notify-pdf-letter-spec-v2.3.pdf">letter specification</a>.</p>
+
+  </div>
+</div>
+{% endblock %}

app/templates/views/uploads/index.html

@@ -7,10 +7,12 @@
 
 {% block maincolumn_content %}
 <div class="grid-row">
-  <div class="column-two-thirds">
+  <div class="column-five-sixths">
     {{ page_header('Uploads') }}
 
     <p>Upload a letter and Notify will print, pack and post it for you.</p>
+
+    <a href="{{ url_for('.upload_letter', service_id=current_service.id) }}" class="button-secondary">Upload a letter</a>
   </div>
 </div>
 {% endblock %}

app/templates/views/uploads/preview.html

@@ -0,0 +1,37 @@
+{% extends "withnav_template.html" %}
+{% from "components/page-header.html" import page_header %}
+
+{% block service_page_title %}
+  {{ original_filename }}
+{% endblock %}
+
+{% block maincolumn_content %}
+    {{ page_header(
+        original_filename,
+        back_link=url_for('main.upload_letter', service_id=current_service.id)
+    ) }}
+
+    {% if status == 'invalid' %}
+    <p class="notification-status-cancelled" id="validation-error-message">
+      Validation failed
+    </p>
+    {% endif %}
+
+    <div class="letter-sent">
+      {{ template|string }}
+    </div>
+
+    {% if status == 'valid' %}
+    <div class="js-stick-at-bottom-when-scrolling">
+      <form method="post" enctype="multipart/form-data" action="{{url_for(
+          'main.send_uploaded_letter',
+          service_id=current_service.id,
+        )}}" class='page-footer'>
+        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+        <input type="hidden" name="filename" value="{{ original_filename }}" />
+        <input type="hidden" name="file_id" value="{{ file_id }}" />
+        <button type="submit" class="button">Send 1 letter</button>
+      </form>
+    </div>
+  {% endif %}
+{% endblock %}

requirements-app.txt

@@ -23,4 +23,4 @@ awscli-cwlogs>=1.4,<1.5
 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default
 itsdangerous==0.24  # pyup: <1.0.0
 
-git+https://github.com/alphagov/notifications-utils.git@34.0.1#egg=notifications-utils==34.0.1
+git+https://github.com/alphagov/notifications-utils.git@34.1.0#egg=notifications-utils==34.1.0

requirements.txt

@@ -25,13 +25,13 @@ awscli-cwlogs>=1.4,<1.5
 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default
 itsdangerous==0.24  # pyup: <1.0.0
 
-git+https://github.com/alphagov/notifications-utils.git@34.0.1#egg=notifications-utils==34.0.1
+git+https://github.com/alphagov/notifications-utils.git@34.1.0#egg=notifications-utils==34.1.0
 
 ## The following requirements were added by pip freeze:
-awscli==1.16.231
+awscli==1.16.233
 bleach==3.1.0
 boto3==1.6.16
-botocore==1.12.221
+botocore==1.12.223
 certifi==2019.6.16
 chardet==3.0.4
 Click==7.0
@@ -72,7 +72,7 @@ statsd==3.3.0
 texttable==1.6.2
 urllib3==1.25.3
 webencodings==0.5.1
-Werkzeug==0.15.5
+Werkzeug==0.15.6
 WTForms==2.2.1
 xlrd==1.2.0
 xlwt==1.3.0

tests/app/main/views/test_platform_admin.py

@@ -805,7 +805,7 @@ def test_letter_validation_preview_doesnt_call_template_preview_when_file_not_pd
     antivirus_scan.assert_not_called()
     validate_letter.assert_not_called()
     page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
-    assert page.find('span', class_='error-message').text.strip() == "PDF documents only!"
+    assert page.find('span', class_='error-message').text.strip() == "Letters must be saved as a PDF"
 
 
 def test_letter_validation_preview_doesnt_call_template_preview_when_file_doesnt_pass_virus_scan(

tests/app/main/views/test_uploads.py

@@ -1,5 +1,330 @@
+from unittest.mock import Mock
+
+import pytest
+from flask import url_for
+from requests import RequestException
+
+from app.utils import normalize_spaces
 from tests.conftest import SERVICE_ONE_ID
 
 
 def test_get_upload_hub_page(client_request):
-    client_request.get('main.uploads', service_id=SERVICE_ONE_ID)
+    page = client_request.get('main.uploads', service_id=SERVICE_ONE_ID)
+
+    assert page.find('h1').text == 'Uploads'
+    assert page.find('a', text='Upload a letter').attrs['href'] == url_for(
+        'main.upload_letter', service_id=SERVICE_ONE_ID
+    )
+
+
+def test_get_upload_letter(client_request):
+    page = client_request.get('main.upload_letter', service_id=SERVICE_ONE_ID)
+
+    assert page.find('h1').text == 'Upload a letter'
+    assert page.find('input', class_='file-upload-field')
+    assert page.select('button[type=submit]')
+
+
+def test_post_upload_letter_redirects_for_valid_file(mocker, client_request):
+    mocker.patch('uuid.uuid4', return_value='fake-uuid')
+    antivirus_mock = mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=True)
+    mocker.patch(
+        'app.main.views.uploads.sanitise_letter',
+        return_value=Mock(content='The sanitised content', json=lambda: {'file': 'VGhlIHNhbml0aXNlZCBjb250ZW50'})
+    )
+    mock_s3 = mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+    mocker.patch('app.main.views.uploads.service_api_client.get_precompiled_template')
+
+    with open('tests/test_pdf_files/one_page_pdf.pdf', 'rb') as file:
+        page = client_request.post(
+            'main.upload_letter',
+            service_id=SERVICE_ONE_ID,
+            _data={'file': file},
+            _follow_redirects=True,
+        )
+    assert antivirus_mock.called
+
+    mock_s3.assert_called_once_with(
+        b'The sanitised content',
+        'service-{}/fake-uuid.pdf'.format(SERVICE_ONE_ID),
+        'valid',
+    )
+
+    assert page.find('h1').text == 'tests/test_pdf_files/one_page_pdf.pdf'
+    assert not page.find(id='validation-error-message')
+
+    assert page.find('input', {'type': 'hidden', 'name': 'filename', 'value': 'tests/test_pdf_files/one_page_pdf.pdf'})
+    assert page.find('input', {'type': 'hidden', 'name': 'file_id', 'value': 'fake-uuid'})
+    assert page.find('button', {'type': 'submit'}).text == 'Send 1 letter'
+
+
+def test_post_upload_letter_shows_letter_preview_for_valid_file(mocker, client_request):
+    letter_template = {'template_type': 'letter',
+                       'reply_to_text': '',
+                       'postage': 'second',
+                       'subject': 'hi',
+                       'content': 'my letter'}
+
+    mocker.patch('uuid.uuid4', return_value='fake-uuid')
+    mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=True)
+    mocker.patch(
+        'app.main.views.uploads.sanitise_letter',
+        return_value=Mock(content='The sanitised content', json=lambda: {'file': 'VGhlIHNhbml0aXNlZCBjb250ZW50'})
+    )
+    mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+    mocker.patch('app.main.views.uploads.pdf_page_count', return_value=3)
+    mocker.patch('app.main.views.uploads.service_api_client.get_precompiled_template', return_value=letter_template)
+
+    with open('tests/test_pdf_files/one_page_pdf.pdf', 'rb') as file:
+        page = client_request.post(
+            'main.upload_letter',
+            service_id=SERVICE_ONE_ID,
+            _data={'file': file},
+            _follow_redirects=True,
+        )
+
+    assert len(page.select('.letter-postage')) == 1
+    assert normalize_spaces(page.select_one('.letter-postage').text) == ('Postage: second class')
+    assert page.select_one('.letter-postage')['class'] == ['letter-postage', 'letter-postage-second']
+
+    letter_images = page.select('main img')
+    assert len(letter_images) == 3
+
+    for page_no, img in enumerate(letter_images, start=1):
+        assert img['src'] == url_for(
+            '.view_letter_upload_as_preview',
+            service_id=SERVICE_ONE_ID,
+            file_id='fake-uuid',
+            page=page_no)
+
+
+def test_post_upload_letter_shows_error_when_file_is_not_a_pdf(client_request):
+    with open('tests/non_spreadsheet_files/actually_a_png.csv', 'rb') as file:
+        page = client_request.post(
+            'main.upload_letter',
+            service_id=SERVICE_ONE_ID,
+            _data={'file': file},
+            _expected_status=200
+        )
+    assert page.find('span', class_='error-message').text.strip() == "Letters must be saved as a PDF"
+
+
+def test_post_upload_letter_shows_error_when_no_file_uploaded(client_request):
+    page = client_request.post(
+        'main.upload_letter',
+        service_id=SERVICE_ONE_ID,
+        _data={'file': ''},
+        _expected_status=200
+    )
+    assert page.find('span', class_='error-message').text.strip() == "You need to upload a file to submit"
+
+
+def test_post_upload_letter_shows_error_when_file_contains_virus(mocker, client_request):
+    mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=False)
+
+    with open('tests/test_pdf_files/one_page_pdf.pdf', 'rb') as file:
+        page = client_request.post(
+            'main.upload_letter',
+            service_id=SERVICE_ONE_ID,
+            _data={'file': file},
+            _expected_status=400
+        )
+    assert page.find('h1').text == 'Upload a letter'
+    assert normalize_spaces(page.select('.banner-dangerous')[0].text) == 'Your file has failed the virus check'
+
+
+def test_post_choose_upload_file_when_file_is_too_big(mocker, client_request):
+    mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=True)
+
+    with open('tests/test_pdf_files/big.pdf', 'rb') as file:
+        page = client_request.post(
+            'main.upload_letter',
+            service_id=SERVICE_ONE_ID,
+            _data={'file': file},
+            _expected_status=400
+        )
+    assert page.find('h1').text == 'Upload a letter'
+    assert normalize_spaces(page.select('.banner-dangerous')[0].text) == 'Your file must be smaller than 2MB'
+
+
+def test_post_choose_upload_file_when_file_is_malformed(mocker, client_request):
+    mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=True)
+
+    with open('tests/test_pdf_files/no_eof_marker.pdf', 'rb') as file:
+        page = client_request.post(
+            'main.upload_letter',
+            service_id=SERVICE_ONE_ID,
+            _data={'file': file},
+            _expected_status=400
+        )
+    assert page.find('h1').text == 'Upload a letter'
+    assert normalize_spaces(page.select('.banner-dangerous')[0].text) == 'Your file must be a valid PDF'
+
+
+def test_post_upload_letter_with_invalid_file(mocker, client_request):
+    mocker.patch('uuid.uuid4', return_value='fake-uuid')
+    mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=True)
+    mock_s3 = mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+
+    mock_sanitise_response = Mock()
+    mock_sanitise_response.raise_for_status.side_effect = RequestException(response=Mock(status_code=400))
+    mocker.patch('app.main.views.uploads.sanitise_letter', return_value=mock_sanitise_response)
+    mocker.patch('app.main.views.uploads.service_api_client.get_precompiled_template')
+
+    with open('tests/test_pdf_files/one_page_pdf.pdf', 'rb') as file:
+        file_contents = file.read()
+        file.seek(0)
+
+        page = client_request.post(
+            'main.upload_letter',
+            service_id=SERVICE_ONE_ID,
+            _data={'file': file},
+            _follow_redirects=True
+        )
+
+        mock_s3.assert_called_once_with(
+            file_contents,
+            'service-{}/fake-uuid.pdf'.format(SERVICE_ONE_ID),
+            'invalid',
+        )
+
+    assert page.find('h1').text == 'tests/test_pdf_files/one_page_pdf.pdf'
+    assert normalize_spaces(
+        page.find(id='validation-error-message').text
+    ) == 'Validation failed'
+    assert not page.find('button', {'type': 'submit'})
+
+
+def test_post_upload_letter_shows_letter_preview_for_invalid_file(mocker, client_request):
+    letter_template = {'template_type': 'letter',
+                       'reply_to_text': '',
+                       'postage': 'first',
+                       'subject': 'hi',
+                       'content': 'my letter'}
+
+    mocker.patch('uuid.uuid4', return_value='fake-uuid')
+    mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=True)
+    mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+    mock_sanitise_response = Mock()
+    mock_sanitise_response.raise_for_status.side_effect = RequestException(response=Mock(status_code=400))
+    mocker.patch('app.main.views.uploads.sanitise_letter', return_value=mock_sanitise_response)
+    mocker.patch('app.main.views.uploads.service_api_client.get_precompiled_template', return_value=letter_template)
+
+    with open('tests/test_pdf_files/one_page_pdf.pdf', 'rb') as file:
+        page = client_request.post(
+            'main.upload_letter',
+            service_id=SERVICE_ONE_ID,
+            _data={'file': file},
+            _follow_redirects=True,
+        )
+
+    assert len(page.select('.letter-postage')) == 1
+    assert normalize_spaces(page.select_one('.letter-postage').text) == ('Postage: first class')
+    assert page.select_one('.letter-postage')['class'] == ['letter-postage', 'letter-postage-first']
+
+    letter_images = page.select('main img')
+    assert len(letter_images) == 1
+    assert letter_images[0]['src'] == url_for(
+        '.view_letter_upload_as_preview',
+        service_id=SERVICE_ONE_ID,
+        file_id='fake-uuid',
+        page=1
+    )
+
+
+def test_post_upload_letter_does_not_upload_to_s3_if_template_preview_raises_unknown_error(mocker, client_request):
+    mocker.patch('uuid.uuid4', return_value='fake-uuid')
+    mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=True)
+    mock_s3 = mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+
+    mocker.patch('app.main.views.uploads.sanitise_letter', side_effect=RequestException())
+
+    with pytest.raises(RequestException):
+        with open('tests/test_pdf_files/one_page_pdf.pdf', 'rb') as file:
+            client_request.post(
+                'main.upload_letter',
+                service_id=SERVICE_ONE_ID,
+                _data={'file': file},
+                _follow_redirects=True
+            )
+
+    assert not mock_s3.called
+
+
+def test_uploaded_letter_preview(mocker, client_request):
+    mocker.patch('app.main.views.uploads.service_api_client')
+
+    page = client_request.get(
+        'main.uploaded_letter_preview',
+        service_id=SERVICE_ONE_ID,
+        file_id='fake-uuid',
+        original_filename='my_letter.pdf',
+        page_count=1,
+        status='valid',
+    )
+
+    assert page.find('h1').text == 'my_letter.pdf'
+    assert page.find('div', class_='letter-sent')
+
+
+def test_send_uploaded_letter_sends_letter_and_redirects_to_notification_page(mocker, service_one, client_request):
+    mocker.patch('app.main.views.uploads.get_letter_pdf_and_metadata', return_value=('file', {'status': 'valid'}))
+    mock_send = mocker.patch('app.main.views.uploads.notification_api_client.send_precompiled_letter')
+
+    service_one['permissions'] = ['letter', 'upload_letters']
+    file_id = 'abcd-1234'
+
+    client_request.post(
+        'main.send_uploaded_letter',
+        service_id=SERVICE_ONE_ID,
+        _data={'filename': 'my_file.pdf', 'file_id': file_id},
+        _expected_redirect=url_for(
+            'main.view_notification',
+            service_id=SERVICE_ONE_ID,
+            notification_id=file_id,
+            _external=True
+        )
+    )
+    mock_send.assert_called_once_with(SERVICE_ONE_ID, 'my_file.pdf', file_id)
+
+
+@pytest.mark.parametrize('permissions', [
+    ['email'],
+    ['letter'],
+    ['upload_letters'],
+])
+def test_send_uploaded_letter_when_service_does_not_have_correct_permissions(
+    mocker,
+    service_one,
+    client_request,
+    permissions,
+):
+    mocker.patch('app.main.views.uploads.get_letter_pdf_and_metadata', return_value=('file', {'status': 'valid'}))
+    mock_send = mocker.patch('app.main.views.uploads.notification_api_client.send_precompiled_letter')
+
+    service_one['permissions'] = permissions
+    file_id = 'abcd-1234'
+
+    client_request.post(
+        'main.send_uploaded_letter',
+        service_id=SERVICE_ONE_ID,
+        _data={'filename': 'my_file.pdf', 'file_id': file_id},
+        _expected_status=403
+    )
+    assert not mock_send.called
+
+
+def test_send_uploaded_letter_when_metadata_states_pdf_is_invalid(mocker, service_one, client_request):
+    mocker.patch('app.main.views.uploads.get_letter_pdf_and_metadata', return_value=('file', {'status': 'invalid'}))
+    mock_send = mocker.patch('app.main.views.uploads.notification_api_client.send_precompiled_letter')
+
+    service_one['permissions'] = ['letter', 'upload_letters']
+    file_id = 'abcd-1234'
+
+    client_request.post(
+        'main.send_uploaded_letter',
+        service_id=SERVICE_ONE_ID,
+        _data={'filename': 'my_file.pdf', 'file_id': file_id},
+        _expected_status=403
+    )
+    assert not mock_send.called

tests/app/notify_client/test_notification_client.py

@@ -59,6 +59,23 @@ def test_send_notification(mocker, logged_in_client, active_user_with_permission
     )
 
 
+def test_send_precompiled_letter(mocker, logged_in_client, active_user_with_permissions):
+    mock_post = mocker.patch('app.notify_client.notification_api_client.NotificationApiClient.post')
+    NotificationApiClient().send_precompiled_letter(
+        'abcd-1234',
+        'my_file.pdf',
+        'file-ID'
+    )
+    mock_post.assert_called_once_with(
+        url='/service/abcd-1234/send-pdf-letter',
+        data={
+            'filename': 'my_file.pdf',
+            'file_id': 'file-ID',
+            'created_by': active_user_with_permissions['id']
+        }
+    )
+
+
 def test_get_notification(mocker):
     mock_get = mocker.patch('app.notify_client.notification_api_client.NotificationApiClient.get')
     NotificationApiClient().get_notification('foo', 'bar')

tests/app/notify_client/test_service_api_client.py

@@ -93,6 +93,14 @@ def test_client_creates_service_with_correct_data(
     )
 
 
+def test_get_precompiled_template(mocker):
+    client = ServiceAPIClient()
+    mock_get = mocker.patch.object(client, 'get')
+
+    client.get_precompiled_template(SERVICE_ONE_ID)
+    mock_get.assert_called_once_with('/service/{}/template/precompiled'.format(SERVICE_ONE_ID))
+
+
 @pytest.mark.parametrize('template_data, extra_args, expected_count', (
     (
         [],

tests/app/s3_client/test_s3_letter_upload_client.py

@@ -0,0 +1,17 @@
+from flask import current_app
+
+from app.s3_client.s3_letter_upload_client import upload_letter_to_s3
+
+
+def test_upload_letter_to_s3(mocker):
+    s3_mock = mocker.patch('app.s3_client.s3_letter_upload_client.utils_s3upload')
+
+    upload_letter_to_s3('pdf_data', 'service_id/upload_id.pdf', 'valid')
+
+    s3_mock.assert_called_once_with(
+        bucket_name=current_app.config['TRANSIENT_UPLOADED_LETTERS'],
+        file_location='service_id/upload_id.pdf',
+        filedata='pdf_data',
+        metadata={'status': 'valid'},
+        region=current_app.config['AWS_REGION']
+    )

tests/app/test_template_previews.py

@@ -1,10 +1,15 @@
+import base64
 from functools import partial
 from unittest.mock import Mock
 
 import pytest
 from notifications_utils.template import LetterPreviewTemplate
 
-from app.template_previews import TemplatePreview, get_page_count_for_letter
+from app.template_previews import (
+    TemplatePreview,
+    get_page_count_for_letter,
+    sanitise_letter,
+)
 
 
 @pytest.mark.parametrize('partial_call, expected_page_argument', [
@@ -75,6 +80,44 @@ def test_from_database_object_makes_request(
     request_mock.assert_called_once_with(expected_url, json=data, headers=headers)
 
 
+@pytest.mark.parametrize('page_number, expected_url', [
+    ('1', 'http://localhost:9999/precompiled-preview.png?hide_notify=true'),
+    ('2', 'http://localhost:9999/precompiled-preview.png'),
+])
+def test_from_valid_pdf_file_makes_request(mocker, page_number, expected_url):
+    mocker.patch('app.template_previews.extract_page_from_pdf', return_value=b'pdf page')
+    request_mock = mocker.patch(
+        'app.template_previews.requests.post',
+        return_value=Mock(content='a', status_code='b', headers={'c': 'd'})
+    )
+
+    response = TemplatePreview.from_valid_pdf_file(b'pdf file', page_number)
+
+    assert response == ('a', 'b', {'c': 'd'}.items())
+    request_mock.assert_called_once_with(
+        expected_url,
+        data=base64.b64encode(b'pdf page').decode('utf-8'),
+        headers={'Authorization': 'Token my-secret-key'},
+    )
+
+
+def test_from_invalid_pdf_file_makes_request(mocker):
+    mocker.patch('app.template_previews.extract_page_from_pdf', return_value=b'pdf page')
+    request_mock = mocker.patch(
+        'app.template_previews.requests.post',
+        return_value=Mock(content='a', status_code='b', headers={'c': 'd'})
+    )
+
+    response = TemplatePreview.from_invalid_pdf_file(b'pdf file', '1')
+
+    assert response == ('a', 'b', {'c': 'd'}.items())
+    request_mock.assert_called_once_with(
+        'http://localhost:9999/precompiled/overlay.png?page_number=1',
+        data=b'pdf page',
+        headers={'Authorization': 'Token my-secret-key'},
+    )
+
+
 @pytest.mark.parametrize('template_type', [
     'email', 'sms'
 ])
@@ -119,3 +162,15 @@ def test_from_example_template_makes_request(mocker):
               'filename': filename,
               'letter_contact_block': None}
     )
+
+
+def test_sanitise_letter_calls_template_preview_sanitise_endoint_with_file(mocker):
+    request_mock = mocker.patch('app.template_previews.requests.post')
+
+    sanitise_letter('pdf_data')
+
+    request_mock.assert_called_once_with(
+        'http://localhost:9999/precompiled/sanitise',
+        headers={'Authorization': 'Token my-secret-key'},
+        data='pdf_data'
+    )