Merge pull request #1773 from alphagov/ken-preview-precompiled-letters-test-key

Preview precompiled letters sent using an API test key

app/config.py

@@ -321,6 +321,7 @@ class Development(Config):
 
     CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
 
     ADMIN_CLIENT_SECRET = 'dev-notify-secret-key'
@@ -357,6 +358,7 @@ class Test(Development):
 
     CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
 
     # this is overriden in jenkins and on cloudfoundry
@@ -384,6 +386,7 @@ class Preview(Config):
     NOTIFY_ENVIRONMENT = 'preview'
     CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
     FROM_NUMBER = 'preview'
     API_RATE_LIMIT_ENABLED = True
@@ -395,6 +398,7 @@ class Staging(Config):
     NOTIFY_ENVIRONMENT = 'staging'
     CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
     STATSD_ENABLED = True
     FROM_NUMBER = 'stage'
@@ -408,6 +412,7 @@ class Live(Config):
     NOTIFY_ENVIRONMENT = 'live'
     CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'production-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp'
     STATSD_ENABLED = True
     FROM_NUMBER = 'GOVUK'
@@ -428,6 +433,7 @@ class Sandbox(CloudFoundryConfig):
     NOTIFY_ENVIRONMENT = 'sandbox'
     CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
     FROM_NUMBER = 'sandbox'
     REDIS_ENABLED = False

app/letters/utils.py

@@ -5,24 +5,32 @@ from flask import current_app
 
 from notifications_utils.s3 import s3upload
 
+from app.models import KEY_TYPE_TEST
 from app.variables import Retention
 
 
 LETTERS_PDF_FILE_LOCATION_STRUCTURE = \
-    '{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
+    '{folder}NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
 
-PRECOMPILED_BUCKET_PREFIX = '{folder}/NOTIFY.{reference}'
+PRECOMPILED_BUCKET_PREFIX = '{folder}NOTIFY.{reference}'
 
 
-def get_letter_pdf_filename(reference, crown):
+def get_folder_name(_now, is_test_letter):
+    if is_test_letter:
+        folder_name = ''
+    else:
+        print_datetime = _now
+        if _now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'):
+            print_datetime = _now + timedelta(days=1)
+        folder_name = '{}/'.format(print_datetime.date())
+    return folder_name
+
+
+def get_letter_pdf_filename(reference, crown, is_test_letter=False):
     now = datetime.utcnow()
 
-    print_datetime = now
-    if now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'):
-        print_datetime = now + timedelta(days=1)
-
     upload_file_name = LETTERS_PDF_FILE_LOCATION_STRUCTURE.format(
-        folder=print_datetime.date(),
+        folder=get_folder_name(now, is_test_letter),
         reference=reference,
         duplex="D",
         letter_class="2",
@@ -34,41 +42,51 @@ def get_letter_pdf_filename(reference, crown):
     return upload_file_name
 
 
-def get_bucket_prefix_for_notification(notification):
+def get_bucket_prefix_for_notification(notification, is_test_letter=False):
     upload_file_name = PRECOMPILED_BUCKET_PREFIX.format(
-        folder=notification.created_at.date(),
+        folder='' if is_test_letter else
+               '{}/'.format(notification.created_at.date()),
         reference=notification.reference
     ).upper()
 
     return upload_file_name
 
 
-def upload_letter_pdf(notification, pdf_data):
+def upload_letter_pdf(notification, pdf_data, is_test_letter=False):
     current_app.logger.info("PDF Letter {} reference {} created at {}, {} bytes".format(
         notification.id, notification.reference, notification.created_at, len(pdf_data)))
 
     upload_file_name = get_letter_pdf_filename(
-        notification.reference, notification.service.crown)
+        notification.reference, notification.service.crown, is_test_letter)
+
+    if is_test_letter:
+        bucket_name = current_app.config['TEST_LETTERS_BUCKET_NAME']
+    else:
+        bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
 
     s3upload(
         filedata=pdf_data,
         region=current_app.config['AWS_REGION'],
-        bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        bucket_name=bucket_name,
         file_location=upload_file_name,
         tags={Retention.KEY: Retention.ONE_WEEK}
     )
 
     current_app.logger.info("Uploaded letters PDF {} to {} for notification id {}".format(
-        upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME'], notification.id))
+        upload_file_name, bucket_name, notification.id))
 
 
 def get_letter_pdf(notification):
-    bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
+    is_test_letter = notification.key_type == KEY_TYPE_TEST and notification.template.is_precompiled_letter
+    if is_test_letter:
+        bucket_name = current_app.config['TEST_LETTERS_BUCKET_NAME']
+    else:
+        bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
 
     s3 = boto3.resource('s3')
     bucket = s3.Bucket(bucket_name)
 
-    for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification)):
+    for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification, is_test_letter)):
         obj = s3.Object(
             bucket_name=bucket_name,
             key=item.key

app/v2/notifications/post_notifications.py

@@ -265,6 +265,8 @@ def process_letter_notification(*, letter_data, api_key, template, reply_to_text
             queue=QueueNames.RESEARCH_MODE
         )
     else:
+        if precompiled and api_key.key_type == KEY_TYPE_TEST:
+            upload_letter_pdf(notification, letter_content, is_test_letter=True)
         update_notification_status_by_reference(notification.reference, NOTIFICATION_DELIVERED)
 
     return notification

requirements_for_test.txt

@@ -1,5 +1,6 @@
 -r requirements.txt
 flake8==3.5.0
+moto==1.1.25
 pytest==3.4.2
 pytest-env==0.6.2
 pytest-mock==1.7.1

tests/app/celery/test_letters_pdf_tasks.py

@@ -28,25 +28,6 @@ def test_should_have_decorated_tasks_functions():
     assert create_letters_pdf.__wrapped__.__name__ == 'create_letters_pdf'
 
 
-@pytest.mark.parametrize('crown_flag,expected_crown_text', [
-    (True, 'C'),
-    (False, 'N'),
-])
-@freeze_time("2017-12-04 17:29:00")
-def test_get_letter_pdf_filename_returns_correct_filename(
-        notify_api, mocker, crown_flag, expected_crown_text):
-    filename = get_letter_pdf_filename(reference='foo', crown=crown_flag)
-
-    assert filename == '2017-12-04/NOTIFY.FOO.D.2.C.{}.20171204172900.PDF'.format(expected_crown_text)
-
-
-@freeze_time("2017-12-04 17:31:00")
-def test_get_letter_pdf_filename_returns_tomorrows_filename(notify_api, mocker):
-    filename = get_letter_pdf_filename(reference='foo', crown=True)
-
-    assert filename == '2017-12-05/NOTIFY.FOO.D.2.C.C.20171204173100.PDF'
-
-
 @pytest.mark.parametrize('personalisation', [{'name': 'test'}, None])
 def test_get_letters_pdf_calls_notifications_template_preview_service_correctly(
         notify_api, mocker, client, sample_letter_template, personalisation):

tests/app/letters/test_letter_utils.py

@@ -1,6 +1,26 @@
 import pytest
+from datetime import datetime
 
-from app.letters.utils import get_bucket_prefix_for_notification
+import boto3
+from flask import current_app
+from freezegun import freeze_time
+from moto import mock_s3
+
+from app.letters.utils import get_bucket_prefix_for_notification, get_letter_pdf_filename, get_letter_pdf
+from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEST, PRECOMPILED_TEMPLATE_NAME
+
+FROZEN_DATE_TIME = "2018-03-14 17:00:00"
+
+
+@pytest.fixture()
+@freeze_time(FROZEN_DATE_TIME)
+def sample_precompiled_letter_notification_using_test_key(sample_letter_notification):
+    sample_letter_notification.template.hidden = True
+    sample_letter_notification.template.name = PRECOMPILED_TEMPLATE_NAME
+    sample_letter_notification.key_type = KEY_TYPE_TEST
+    sample_letter_notification.reference = 'foo'
+    sample_letter_notification.created_at = datetime.utcnow()
+    return sample_letter_notification
 
 
 def test_get_bucket_prefix_for_notification_valid_notification(sample_notification):
@@ -13,6 +33,70 @@ def test_get_bucket_prefix_for_notification_valid_notification(sample_notificati
     ).upper()
 
 
+@freeze_time(FROZEN_DATE_TIME)
+def test_get_bucket_prefix_for_notification_precompiled_letter_using_test_key(
+    sample_precompiled_letter_notification_using_test_key
+):
+    bucket_prefix = get_bucket_prefix_for_notification(
+        sample_precompiled_letter_notification_using_test_key, is_test_letter=True)
+
+    assert bucket_prefix == 'NOTIFY.{}'.format(
+        sample_precompiled_letter_notification_using_test_key.reference).upper()
+
+
 def test_get_bucket_prefix_for_notification_invalid_notification():
     with pytest.raises(AttributeError):
         get_bucket_prefix_for_notification(None)
+
+
+@pytest.mark.parametrize('crown_flag,expected_crown_text', [
+    (True, 'C'),
+    (False, 'N'),
+])
+@freeze_time("2017-12-04 17:29:00")
+def test_get_letter_pdf_filename_returns_correct_filename(
+        notify_api, mocker, crown_flag, expected_crown_text):
+    filename = get_letter_pdf_filename(reference='foo', crown=crown_flag)
+
+    assert filename == '2017-12-04/NOTIFY.FOO.D.2.C.{}.20171204172900.PDF'.format(expected_crown_text)
+
+
+@freeze_time("2017-12-04 17:29:00")
+def test_get_letter_pdf_filename_returns_correct_filename_for_test_letters(
+        notify_api, mocker):
+    filename = get_letter_pdf_filename(reference='foo', crown='C', is_test_letter=True)
+
+    assert filename == 'NOTIFY.FOO.D.2.C.C.20171204172900.PDF'
+
+
+@freeze_time("2017-12-04 17:31:00")
+def test_get_letter_pdf_filename_returns_tomorrows_filename(notify_api, mocker):
+    filename = get_letter_pdf_filename(reference='foo', crown=True)
+
+    assert filename == '2017-12-05/NOTIFY.FOO.D.2.C.C.20171204173100.PDF'
+
+
+@mock_s3
+@pytest.mark.parametrize('bucket_config_name,filename_format', [
+    ('TEST_LETTERS_BUCKET_NAME', 'NOTIFY.FOO.D.2.C.C.%Y%m%d%H%M%S.PDF'),
+    ('LETTERS_PDF_BUCKET_NAME', '%Y-%m-%d/NOTIFY.FOO.D.2.C.C.%Y%m%d%H%M%S.PDF')
+])
+@freeze_time(FROZEN_DATE_TIME)
+def test_get_letter_pdf_gets_pdf_from_correct_bucket(
+    sample_precompiled_letter_notification_using_test_key,
+    bucket_config_name,
+    filename_format
+):
+    if bucket_config_name == 'LETTERS_PDF_BUCKET_NAME':
+        sample_precompiled_letter_notification_using_test_key.key_type = KEY_TYPE_NORMAL
+
+    bucket_name = current_app.config[bucket_config_name]
+    filename = datetime.utcnow().strftime(filename_format)
+    conn = boto3.resource('s3', region_name='eu-west-1')
+    conn.create_bucket(Bucket=bucket_name)
+    s3 = boto3.client('s3', region_name='eu-west-1')
+    s3.put_object(Bucket=bucket_name, Key=filename, Body=b'pdf_content')
+
+    ret = get_letter_pdf(sample_precompiled_letter_notification_using_test_key)
+
+    assert ret == b'pdf_content'

tests/app/v2/notifications/test_post_letter_notifications.py

@@ -1,4 +1,5 @@
 import uuid
+from unittest.mock import ANY
 
 from flask import json
 from flask import url_for
@@ -29,9 +30,13 @@ test_address = {
 }
 
 
-def letter_request(client, data, service_id, key_type=KEY_TYPE_NORMAL, _expected_status=201):
+def letter_request(client, data, service_id, key_type=KEY_TYPE_NORMAL, _expected_status=201, precompiled=False):
+    if precompiled:
+        url = url_for('v2_notifications.post_precompiled_letter_notification')
+    else:
+        url = url_for('v2_notifications.post_notification', notification_type=LETTER_TYPE)
     resp = client.post(
-        url_for('v2_notifications.post_notification', notification_type=LETTER_TYPE),
+        url,
         data=json.dumps(data),
         headers=[
             ('Content-Type', 'application/json'),
@@ -382,6 +387,30 @@ def test_post_letter_notification_is_delivered_if_in_trial_mode_and_using_test_k
     assert not fake_create_letter_task.called
 
 
+def test_post_letter_notification_is_delivered_and_has_pdf_uploaded_to_test_letters_bucket_using_test_key(
+    client,
+    notify_user,
+    mocker
+):
+    sample_letter_service = create_service(service_permissions=['letter', 'precompiled_letter'])
+    s3mock = mocker.patch('app.v2.notifications.post_notifications.upload_letter_pdf')
+    mocker.patch('app.v2.notifications.post_notifications.pdf_page_count', return_value=1)
+    data = {
+        "reference": "letter-reference",
+        "content": "bGV0dGVyLWNvbnRlbnQ="
+    }
+    letter_request(
+        client,
+        data=data,
+        service_id=str(sample_letter_service.id),
+        key_type=KEY_TYPE_TEST,
+        precompiled=True)
+
+    notification = Notification.query.one()
+    assert notification.status == NOTIFICATION_DELIVERED
+    s3mock.assert_called_once_with(ANY, b'letter-content', is_test_letter=True)
+
+
 def test_post_letter_notification_persists_notification_reply_to_text(
     client, notify_db_session, mocker
 ):