Merge pull request #1737 from alphagov/add_precompiled_letters

Updated API to handle pre-compiled pdfs

app/config.py

@@ -149,6 +149,7 @@ class Config(object):
     CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
     SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
     ORGANISATION_INVITATION_EMAIL_TEMPLATE_ID = '203566f0-d835-47c5-aa06-932439c86573'
+    PRECOMPILED_TEMPLATE_NAME = 'Pre-compiled PDF'
 
     BROKER_URL = 'sqs://'
     BROKER_TRANSPORT_OPTIONS = {

app/letters/utils.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta
 
+import boto3
 from flask import current_app
 
 from notifications_utils.s3 import s3upload
@@ -10,6 +11,8 @@ from app.variables import Retention
 LETTERS_PDF_FILE_LOCATION_STRUCTURE = \
     '{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
 
+PRECOMPILED_BUCKET_PREFIX = '{folder}/NOTIFY.{reference}'
+
 
 def get_letter_pdf_filename(reference, crown):
     now = datetime.utcnow()
@@ -31,6 +34,15 @@ def get_letter_pdf_filename(reference, crown):
     return upload_file_name
 
 
+def get_bucket_prefix_for_notification(notification):
+    upload_file_name = PRECOMPILED_BUCKET_PREFIX.format(
+        folder=notification.created_at.date(),
+        reference=notification.reference
+    ).upper()
+
+    return upload_file_name
+
+
 def upload_letter_pdf(notification, pdf_data):
     current_app.logger.info("PDF Letter {} reference {} created at {}, {} bytes".format(
         notification.id, notification.reference, notification.created_at, len(pdf_data)))
@@ -48,3 +60,23 @@ def upload_letter_pdf(notification, pdf_data):
 
     current_app.logger.info("Uploaded letters PDF {} to {} for notification id {}".format(
         upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME'], notification.id))
+
+
+def get_letter_pdf(notification):
+    bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
+
+    s3 = boto3.resource('s3')
+    bucket = s3.Bucket(bucket_name)
+
+    for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification)):
+        obj = s3.Object(
+            bucket_name=bucket_name,
+            key=item.key
+        )
+        file_content = obj.get()["Body"].read()
+
+    return file_content
+
+
+def is_precompiled_letter(template):
+    return template.hidden and template.name == current_app.config['PRECOMPILED_TEMPLATE_NAME']

app/template/rest.py

@@ -1,4 +1,6 @@
 import base64
+
+import botocore
 from flask import (
     Blueprint,
     current_app,
@@ -6,6 +8,7 @@ from flask import (
     request)
 from requests import post as requests_post
 
+
 from app.dao.notifications_dao import get_notification_by_id
 from app.dao.templates_dao import (
     dao_update_template,
@@ -18,6 +21,7 @@ from app.dao.templates_dao import (
     dao_get_template_by_id)
 from notifications_utils.template import SMSMessageTemplate
 from app.dao.services_dao import dao_fetch_service_by_id
+from app.letters.utils import get_letter_pdf, is_precompiled_letter
 from app.models import SMS_TYPE
 from app.notifications.validators import service_has_permission, check_reply_to
 from app.schemas import (template_schema, template_history_schema)
@@ -185,6 +189,7 @@ def redact_template(template, data):
 
 @template_blueprint.route('/preview/<uuid:notification_id>/<file_type>', methods=['GET'])
 def preview_letter_template_by_notification_id(service_id, notification_id, file_type):
+
     if file_type not in ('pdf', 'png'):
         raise InvalidRequest({'content': ["file_type must be pdf or png"]}, status_code=400)
 
@@ -194,6 +199,30 @@ def preview_letter_template_by_notification_id(service_id, notification_id, file
 
     template = dao_get_template_by_id(notification.template_id)
 
+    if is_precompiled_letter(template):
+
+        try:
+
+            pdf_file = get_letter_pdf(notification)
+
+        except botocore.exceptions.ClientError:
+            current_app.logger.info
+            raise InvalidRequest('Error getting letter file from S3 notification id {}'.format(notification_id),
+                                 status_code=500)
+
+        content = base64.b64encode(pdf_file).decode('utf-8')
+
+        if file_type == 'png':
+
+            url = '{}/precompiled-preview.png{}'.format(
+                current_app.config['TEMPLATE_PREVIEW_API_HOST'],
+                '?page={}'.format(page) if page else ''
+            )
+
+            content = _get_png_preview(url, content, notification.id)
+
+    else:
+
         template_for_letter_print = {
             "id": str(notification.template_id),
             "subject": template.subject,
@@ -210,13 +239,21 @@ def preview_letter_template_by_notification_id(service_id, notification_id, file
             'dvla_org_id': service.dvla_organisation_id,
         }
 
-    resp = requests_post(
+        url = '{}/preview.{}{}'.format(
-        '{}/preview.{}{}'.format(
             current_app.config['TEMPLATE_PREVIEW_API_HOST'],
             file_type,
             '?page={}'.format(page) if page else ''
-        ),
+        )
-        json=data,
+
+        content = _get_png_preview(url, data, notification.id)
+
+    return jsonify({"content": content})
+
+
+def _get_png_preview(url, data, notification_id):
+    resp = requests_post(
+        url,
+        data=data,
         headers={'Authorization': 'Token {}'.format(current_app.config['TEMPLATE_PREVIEW_API_KEY'])}
     )
 
@@ -225,5 +262,4 @@ def preview_letter_template_by_notification_id(service_id, notification_id, file
             'Error generating preview for {}'.format(notification_id), status_code=500
         )
 
-    content = base64.b64encode(resp.content).decode('utf-8')
+    return base64.b64encode(resp.content).decode('utf-8')
-    return jsonify({"content": content})

tests/app/db.py

@@ -130,7 +130,8 @@ def create_template(
         template_name=None,
         subject='Template subject',
         content='Dear Sir/Madam, Hello. Yours Truly, The Government.',
-    reply_to=None
+        reply_to=None,
+        hidden=False
 ):
     data = {
         'name': template_name or '{} Template Name'.format(template_type),
@@ -139,6 +140,7 @@ def create_template(
         'service': service,
         'created_by': service.created_by,
         'reply_to': reply_to,
+        'hidden': hidden
     }
     if template_type != SMS_TYPE:
         data['subject'] = subject

tests/app/letters/test_letter_utils.py

@@ -0,0 +1,39 @@
+import pytest
+from flask import current_app
+
+from app.letters.utils import get_bucket_prefix_for_notification, is_precompiled_letter
+
+
+def test_get_bucket_prefix_for_notification_valid_notification(sample_notification):
+
+    bucket_prefix = get_bucket_prefix_for_notification(sample_notification)
+
+    assert bucket_prefix == '{folder}/NOTIFY.{reference}'.format(
+        folder=sample_notification.created_at.date(),
+        reference=sample_notification.reference
+    ).upper()
+
+
+def test_get_bucket_prefix_for_notification_invalid_notification():
+    with pytest.raises(AttributeError):
+        get_bucket_prefix_for_notification(None)
+
+
+def test_is_precompiled_letter_false(sample_letter_template):
+    assert not is_precompiled_letter(sample_letter_template)
+
+
+def test_is_precompiled_letter_true(sample_letter_template):
+    sample_letter_template.hidden = True
+    sample_letter_template.name = current_app.config['PRECOMPILED_TEMPLATE_NAME']
+    assert is_precompiled_letter(sample_letter_template)
+
+
+def test_is_precompiled_letter_hidden_true_not_name(sample_letter_template):
+    sample_letter_template.hidden = True
+    assert not is_precompiled_letter(sample_letter_template)
+
+
+def test_is_precompiled_letter_name_correct_not_hidden(sample_letter_template):
+    sample_letter_template.name = current_app.config['PRECOMPILED_TEMPLATE_NAME']
+    assert not is_precompiled_letter(sample_letter_template)

tests/app/template/test_rest.py

@@ -4,7 +4,9 @@ import random
 import string
 from datetime import datetime, timedelta
 
+import botocore
 import pytest
+import requests_mock
 from freezegun import freeze_time
 
 from app.models import Template, SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, TemplateHistory
@@ -16,7 +18,7 @@ from tests.app.conftest import (
     sample_template_without_email_permission,
     sample_template_without_letter_permission,
     sample_template_without_sms_permission)
-from tests.app.db import create_service, create_letter_contact, create_template
+from tests.app.db import create_service, create_letter_contact, create_template, create_notification
 from tests.conftest import set_config_values
 
 
@@ -823,7 +825,6 @@ def test_preview_letter_template_by_id_valid_file_type(
         'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
         'TEMPLATE_PREVIEW_API_KEY': 'test-key'
     }):
-        import requests_mock
         with requests_mock.Mocker() as request_mock:
             content = b'\x00\x01'
 
@@ -874,3 +875,210 @@ def test_preview_letter_template_by_id_template_preview_500(
             )
 
             assert resp['message'] == 'Error generating preview for {}'.format(sample_letter_notification.id)
+
+
+def test_preview_letter_template_precompiled_pdf_file_type(
+        notify_api,
+        client,
+        admin_request,
+        sample_service,
+        mocker
+):
+
+    template = create_template(sample_service,
+                               template_type='letter',
+                               template_name='Pre-compiled PDF',
+                               subject='Pre-compiled PDF',
+                               hidden=True)
+
+    notification = create_notification(template)
+
+    with set_config_values(notify_api, {
+        'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
+        'TEMPLATE_PREVIEW_API_KEY': 'test-key'
+    }):
+        with requests_mock.Mocker():
+
+            content = b'\x00\x01'
+
+            mock_get_letter_pdf = mocker.patch('app.template.rest.get_letter_pdf', return_value=content)
+
+            resp = admin_request.get(
+                'template.preview_letter_template_by_notification_id',
+                service_id=notification.service_id,
+                notification_id=notification.id,
+                file_type='pdf'
+            )
+
+            assert mock_get_letter_pdf.called_once_with(notification)
+            assert base64.b64decode(resp['content']) == content
+
+
+def test_preview_letter_template_precompiled_s3_error(
+        notify_api,
+        client,
+        admin_request,
+        sample_service,
+        mocker
+):
+
+    template = create_template(sample_service,
+                               template_type='letter',
+                               template_name='Pre-compiled PDF',
+                               subject='Pre-compiled PDF',
+                               hidden=True)
+
+    notification = create_notification(template)
+
+    with set_config_values(notify_api, {
+        'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
+        'TEMPLATE_PREVIEW_API_KEY': 'test-key'
+    }):
+        with requests_mock.Mocker():
+
+            mocker.patch('app.template.rest.get_letter_pdf',
+                         side_effect=botocore.exceptions.ClientError(
+                             {'Error': {'Code': '403', 'Message': 'Unauthorized'}},
+                             'GetObject'
+                         ))
+
+            admin_request.get(
+                'template.preview_letter_template_by_notification_id',
+                service_id=notification.service_id,
+                notification_id=notification.id,
+                file_type='pdf',
+                _expected_status=500
+            )
+
+
+def test_preview_letter_template_precompiled_png_file_type(
+        notify_api,
+        client,
+        admin_request,
+        sample_service,
+        mocker
+):
+
+    template = create_template(sample_service,
+                               template_type='letter',
+                               template_name='Pre-compiled PDF',
+                               subject='Pre-compiled PDF',
+                               hidden=True)
+
+    notification = create_notification(template)
+
+    with set_config_values(notify_api, {
+        'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
+        'TEMPLATE_PREVIEW_API_KEY': 'test-key'
+    }):
+        with requests_mock.Mocker() as request_mock:
+
+            pdf_content = b'\x00\x01'
+            png_content = b'\x00\x02'
+
+            mock_get_letter_pdf = mocker.patch('app.template.rest.get_letter_pdf', return_value=pdf_content)
+
+            request_mock.post(
+                'http://localhost/notifications-template-preview/precompiled-preview.png',
+                content=png_content,
+                headers={'X-pdf-page-count': '1'},
+                status_code=200
+            )
+
+            resp = admin_request.get(
+                'template.preview_letter_template_by_notification_id',
+                service_id=notification.service_id,
+                notification_id=notification.id,
+                file_type='png'
+            )
+
+            assert mock_get_letter_pdf.called_once_with(notification)
+            assert base64.b64decode(resp['content']) == png_content
+
+
+def test_preview_letter_template_precompiled_png_template_preview_500_error(
+        notify_api,
+        client,
+        admin_request,
+        sample_service,
+        mocker
+):
+
+    template = create_template(sample_service,
+                               template_type='letter',
+                               template_name='Pre-compiled PDF',
+                               subject='Pre-compiled PDF',
+                               hidden=True)
+
+    notification = create_notification(template)
+
+    with set_config_values(notify_api, {
+        'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
+        'TEMPLATE_PREVIEW_API_KEY': 'test-key'
+    }):
+        with requests_mock.Mocker() as request_mock:
+
+            pdf_content = b'\x00\x01'
+            png_content = b'\x00\x02'
+
+            mocker.patch('app.template.rest.get_letter_pdf', return_value=pdf_content)
+
+            request_mock.post(
+                'http://localhost/notifications-template-preview/precompiled-preview.png',
+                content=png_content,
+                headers={'X-pdf-page-count': '1'},
+                status_code=500
+            )
+
+            admin_request.get(
+                'template.preview_letter_template_by_notification_id',
+                service_id=notification.service_id,
+                notification_id=notification.id,
+                file_type='png',
+                _expected_status=500
+
+            )
+
+
+def test_preview_letter_template_precompiled_png_template_preview_400_error(
+        notify_api,
+        client,
+        admin_request,
+        sample_service,
+        mocker
+):
+
+    template = create_template(sample_service,
+                               template_type='letter',
+                               template_name='Pre-compiled PDF',
+                               subject='Pre-compiled PDF',
+                               hidden=True)
+
+    notification = create_notification(template)
+
+    with set_config_values(notify_api, {
+        'TEMPLATE_PREVIEW_API_HOST': 'http://localhost/notifications-template-preview',
+        'TEMPLATE_PREVIEW_API_KEY': 'test-key'
+    }):
+        with requests_mock.Mocker() as request_mock:
+
+            pdf_content = b'\x00\x01'
+            png_content = b'\x00\x02'
+
+            mocker.patch('app.template.rest.get_letter_pdf', return_value=pdf_content)
+
+            request_mock.post(
+                'http://localhost/notifications-template-preview/precompiled-preview.png',
+                content=png_content,
+                headers={'X-pdf-page-count': '1'},
+                status_code=404
+            )
+
+            admin_request.get(
+                'template.preview_letter_template_by_notification_id',
+                service_id=notification.service_id,
+                notification_id=notification.id,
+                file_type='png',
+                _expected_status=500
+
+            )