Upload pre-compiled letter PDF to S3

Pre-compiled letter endpoint uploads PDF contents to S3 directly
instead of creating a letter task to generate PDF using template
preview.

This moves some of the utility functions used by existing letter
celery tasks to app.letters.utils, so that they can be reused by
the API endpoint.

app/celery/letters_pdf_tasks.py

@@ -1,4 +1,3 @@
-from datetime import datetime, timedelta
 import math
 
 from flask import current_app
@@ -9,43 +8,18 @@ from requests import (
 )
 from botocore.exceptions import ClientError as BotoClientError
 
-from notifications_utils.s3 import s3upload
-
 from app import notify_celery
 from app.aws import s3
 from app.config import QueueNames, TaskNames
-from app.variables import Retention
 from app.dao.notifications_dao import (
     get_notification_by_id,
     update_notification_status_by_id,
     dao_update_notification,
     dao_get_notifications_by_references,
 )
+from app.letters.utils import upload_letter_pdf
 from app.models import NOTIFICATION_CREATED
 
-LETTERS_PDF_FILE_LOCATION_STRUCTURE = \
-    '{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
-
-
-def get_letter_pdf_filename(reference, crown):
-    now = datetime.utcnow()
-
-    print_datetime = now
-    if now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'):
-        print_datetime = now + timedelta(days=1)
-
-    upload_file_name = LETTERS_PDF_FILE_LOCATION_STRUCTURE.format(
-        folder=print_datetime.date(),
-        reference=reference,
-        duplex="D",
-        letter_class="2",
-        colour="C",
-        crown="C" if crown else "N",
-        date=now.strftime('%Y%m%d%H%M%S')
-    ).upper()
-
-    return upload_file_name
-
 
 @notify_celery.task(bind=True, name="create-letters-pdf", max_retries=15, default_retry_delay=300)
 @statsd(namespace="tasks")
@@ -59,22 +33,8 @@ def create_letters_pdf(self, notification_id):
             org_id=notification.service.dvla_organisation.id,
             values=notification.personalisation
         )
-        current_app.logger.info("PDF Letter {} reference {} created at {}, {} bytes".format(
-            notification.id, notification.reference, notification.created_at, len(pdf_data)))
 
-        upload_file_name = get_letter_pdf_filename(
-            notification.reference, notification.service.crown)
-
-        s3upload(
-            filedata=pdf_data,
-            region=current_app.config['AWS_REGION'],
-            bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
-            file_location=upload_file_name,
-            tags={Retention.KEY: Retention.ONE_WEEK}
-        )
-
-        current_app.logger.info("Uploaded letters PDF {} to {}".format(
-            upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME']))
+        upload_letter_pdf(notification, pdf_data)
 
         notification.billable_units = billable_units
         dao_update_notification(notification)

app/letters/utils.py

@@ -0,0 +1,50 @@
+from datetime import datetime, timedelta
+
+from flask import current_app
+
+from notifications_utils.s3 import s3upload
+
+from app.variables import Retention
+
+
+LETTERS_PDF_FILE_LOCATION_STRUCTURE = \
+    '{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
+
+
+def get_letter_pdf_filename(reference, crown):
+    now = datetime.utcnow()
+
+    print_datetime = now
+    if now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'):
+        print_datetime = now + timedelta(days=1)
+
+    upload_file_name = LETTERS_PDF_FILE_LOCATION_STRUCTURE.format(
+        folder=print_datetime.date(),
+        reference=reference,
+        duplex="D",
+        letter_class="2",
+        colour="C",
+        crown="C" if crown else "N",
+        date=now.strftime('%Y%m%d%H%M%S')
+    ).upper()
+
+    return upload_file_name
+
+
+def upload_letter_pdf(notification, pdf_data):
+    current_app.logger.info("PDF Letter {} reference {} created at {}, {} bytes".format(
+        notification.id, notification.reference, notification.created_at, len(pdf_data)))
+
+    upload_file_name = get_letter_pdf_filename(
+        notification.reference, notification.service.crown)
+
+    s3upload(
+        filedata=pdf_data,
+        region=current_app.config['AWS_REGION'],
+        bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        file_location=upload_file_name,
+        tags={Retention.KEY: Retention.ONE_WEEK}
+    )
+
+    current_app.logger.info("Uploaded letters PDF {} to {}".format(
+        upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME']))

app/v2/notifications/post_notifications.py

@@ -1,3 +1,4 @@
+import base64
 import functools
 
 from flask import request, jsonify, current_app, abort
@@ -9,11 +10,13 @@ from app.config import QueueNames
 from app.dao.notifications_dao import update_notification_status_by_reference
 from app.dao.templates_dao import dao_create_template
 from app.dao.users_dao import get_user_by_id
+from app.letters.utils import upload_letter_pdf
 from app.models import (
     Template,
     SMS_TYPE,
     EMAIL_TYPE,
     LETTER_TYPE,
+    # PRECOMPILED_LETTER,
     PRIORITY,
     KEY_TYPE_TEST,
     KEY_TYPE_TEAM,
@@ -64,7 +67,9 @@ def post_precompiled_letter_notification():
 
     form = validate(request.get_json(), post_precompiled_letter_request)
 
-    #check_service_has_permission(notification_type, authenticated_service.permissions)
+    # Check both permission to send letters and permission to send pre-compiled PDFs
+    check_service_has_permission(LETTER_TYPE, authenticated_service.permissions)
+    # check_service_has_permission(PRECOMPILED_LETTER, authenticated_service.permissions)
 
     check_rate_limiting(authenticated_service, api_user)
 
@@ -80,7 +85,8 @@ def post_precompiled_letter_notification():
         letter_data=form,
         api_key=api_user,
         template=template,
-        reply_to_text=reply_to
+        reply_to_text=reply_to,
+        precompiled=True
     )
 
     create_resp_partial = functools.partial(
@@ -211,7 +217,7 @@ def process_sms_or_email_notification(*, form, notification_type, api_key, templ
     return notification
 
 
-def process_letter_notification(*, letter_data, api_key, template, reply_to_text):
+def process_letter_notification(*, letter_data, api_key, template, reply_to_text, precompiled=False):
     if api_key.key_type == KEY_TYPE_TEAM:
         raise BadRequestError(message='Cannot send letters with a team api key', status_code=403)
 
@@ -229,10 +235,13 @@ def process_letter_notification(*, letter_data, api_key, template, reply_to_text
                                               reply_to_text=reply_to_text)
 
     if should_send:
-        create_letters_pdf.apply_async(
-            [str(notification.id)],
-            queue=QueueNames.CREATE_LETTERS_PDF
-        )
+        if precompiled:
+            upload_letter_pdf(notification, base64.b64decode(letter_data['content']))
+        else:
+            create_letters_pdf.apply_async(
+                [str(notification.id)],
+                queue=QueueNames.CREATE_LETTERS_PDF
+            )
     elif (api_key.service.research_mode and
           current_app.config['NOTIFY_ENVIRONMENT'] in ['preview', 'development']):
         create_fake_letter_response_file.apply_async(
@@ -279,8 +288,8 @@ def get_precompiled_letter_template(service_id):
         return template
 
     template = Template(
-        name='Pre-compiled PDF letter',
-        created_by=get_user_by_id(api_user.created_by_id),
+        name='Pre-compiled PDF',
+        created_by=get_user_by_id(current_app.config['NOTIFY_USER_ID']),
         service_id=service_id,
         template_type=LETTER_TYPE,
         hidden=True,

tests/app/celery/test_letters_pdf_tasks.py

@@ -16,9 +16,9 @@ from app.celery.letters_pdf_tasks import (
     get_letters_pdf,
     collate_letter_pdfs_for_day,
     group_letters,
-    letter_in_created_state,
-    get_letter_pdf_filename,
+    letter_in_created_state
 )
+from app.letters.utils import get_letter_pdf_filename
 from app.models import Notification, NOTIFICATION_SENDING
 
 from tests.conftest import set_config_values
@@ -106,7 +106,7 @@ def test_get_letters_pdf_calculates_billing_units(
 @freeze_time("2017-12-04 17:31:00")
 def test_create_letters_pdf_calls_s3upload(mocker, sample_letter_notification):
     mocker.patch('app.celery.letters_pdf_tasks.get_letters_pdf', return_value=(b'\x00\x01', '1'))
-    mock_s3 = mocker.patch('app.celery.letters_pdf_tasks.s3upload')
+    mock_s3 = mocker.patch('app.letters.utils.s3upload')
 
     create_letters_pdf(sample_letter_notification.id)
 
@@ -126,7 +126,7 @@ def test_create_letters_pdf_calls_s3upload(mocker, sample_letter_notification):
 
 def test_create_letters_pdf_sets_billable_units(mocker, sample_letter_notification):
     mocker.patch('app.celery.letters_pdf_tasks.get_letters_pdf', return_value=(b'\x00\x01', 1))
-    mocker.patch('app.celery.letters_pdf_tasks.s3upload')
+    mocker.patch('app.letters.utils.s3upload')
 
     create_letters_pdf(sample_letter_notification.id)
     noti = Notification.query.filter(Notification.reference == sample_letter_notification.reference).one()
@@ -150,7 +150,7 @@ def test_create_letters_pdf_handles_request_errors(mocker, sample_letter_notific
 
 def test_create_letters_pdf_handles_s3_errors(mocker, sample_letter_notification):
     mocker.patch('app.celery.letters_pdf_tasks.get_letters_pdf', return_value=(b'\x00\x01', 1))
-    mock_s3 = mocker.patch('app.celery.letters_pdf_tasks.s3upload', side_effect=ClientError({}, 'operation_name'))
+    mock_s3 = mocker.patch('app.letters.utils.s3upload', side_effect=ClientError({}, 'operation_name'))
     mock_retry = mocker.patch('app.celery.letters_pdf_tasks.create_letters_pdf.retry')
 
     create_letters_pdf(sample_letter_notification.id)

tests/app/v2/notifications/test_post_notifications.py

@@ -697,11 +697,11 @@ def test_post_email_notification_with_invalid_reply_to_id_returns_400(client, sa
     assert 'BadRequestError' in resp_json['errors'][0]['error']
 
 
-def test_post_precompiled_letter_notification_returns_201(client, sample_service, mocker):
-    mocker.patch('app.celery.letters_pdf_tasks.create_letters_pdf.apply_async')
+def test_post_precompiled_letter_notification_returns_201(client, sample_service, notify_user, mocker):
+    s3mock = mocker.patch('app.v2.notifications.post_notifications.upload_letter_pdf')
     data = {
         "reference": "letter-reference",
-        "content": "abcdefgh"
+        "content": "bGV0dGVyLWNvbnRlbnQ="
     }
     auth_header = create_authorization_header(service_id=sample_service.id)
     response = client.post(
@@ -711,6 +711,8 @@ def test_post_precompiled_letter_notification_returns_201(client, sample_service
 
     assert response.status_code == 201, response.get_data(as_text=True)
 
+    s3mock.assert_called_once_with(ANY, b'letter-content')
+
     resp_json = json.loads(response.get_data(as_text=True))
     assert resp_json == {
         'content': {'body': None, 'subject': 'Pre-compiled PDF'},