diff --git a/app/config.py b/app/config.py index b1cdd93c2..feaa8d0f9 100644 --- a/app/config.py +++ b/app/config.py @@ -321,6 +321,7 @@ class Development(Config): CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf' + TEST_LETTERS_BUCKET_NAME = 'development-test-letters' DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp' ADMIN_CLIENT_SECRET = 'dev-notify-secret-key' @@ -357,6 +358,7 @@ class Test(Development): CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf' + TEST_LETTERS_BUCKET_NAME = 'test-test-letters' DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp' # this is overriden in jenkins and on cloudfoundry @@ -384,6 +386,7 @@ class Preview(Config): NOTIFY_ENVIRONMENT = 'preview' CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf' + TEST_LETTERS_BUCKET_NAME = 'preview-test-letters' DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp' FROM_NUMBER = 'preview' API_RATE_LIMIT_ENABLED = True @@ -395,6 +398,7 @@ class Staging(Config): NOTIFY_ENVIRONMENT = 'staging' CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload' LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf' + TEST_LETTERS_BUCKET_NAME = 'staging-test-letters' DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp' STATSD_ENABLED = True FROM_NUMBER = 'stage' @@ -408,6 +412,7 @@ class Live(Config): NOTIFY_ENVIRONMENT = 'live' CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload' LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf' + TEST_LETTERS_BUCKET_NAME = 'production-test-letters' DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' STATSD_ENABLED = True FROM_NUMBER = 'GOVUK' @@ -428,6 +433,7 @@ class Sandbox(CloudFoundryConfig): NOTIFY_ENVIRONMENT = 'sandbox' CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload' LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf' + TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters' DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp' FROM_NUMBER = 'sandbox' REDIS_ENABLED = False diff --git a/app/letters/utils.py b/app/letters/utils.py index a7f05c00f..7b6205c68 100644 --- a/app/letters/utils.py +++ b/app/letters/utils.py @@ -5,24 +5,32 @@ from flask import current_app from notifications_utils.s3 import s3upload +from app.models import KEY_TYPE_TEST from app.variables import Retention LETTERS_PDF_FILE_LOCATION_STRUCTURE = \ - '{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf' + '{folder}NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf' -PRECOMPILED_BUCKET_PREFIX = '{folder}/NOTIFY.{reference}' +PRECOMPILED_BUCKET_PREFIX = '{folder}NOTIFY.{reference}' -def get_letter_pdf_filename(reference, crown): +def get_folder_name(_now, is_test_letter): + if is_test_letter: + folder_name = '' + else: + print_datetime = _now + if _now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'): + print_datetime = _now + timedelta(days=1) + folder_name = '{}/'.format(print_datetime.date()) + return folder_name + + +def get_letter_pdf_filename(reference, crown, is_test_letter=False): now = datetime.utcnow() - print_datetime = now - if now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'): - print_datetime = now + timedelta(days=1) - upload_file_name = LETTERS_PDF_FILE_LOCATION_STRUCTURE.format( - folder=print_datetime.date(), + folder=get_folder_name(now, is_test_letter), reference=reference, duplex="D", letter_class="2", @@ -34,41 +42,51 @@ def get_letter_pdf_filename(reference, crown): return upload_file_name -def get_bucket_prefix_for_notification(notification): +def get_bucket_prefix_for_notification(notification, is_test_letter=False): upload_file_name = PRECOMPILED_BUCKET_PREFIX.format( - folder=notification.created_at.date(), + folder='' if is_test_letter else + '{}/'.format(notification.created_at.date()), reference=notification.reference ).upper() return upload_file_name -def upload_letter_pdf(notification, pdf_data): +def upload_letter_pdf(notification, pdf_data, is_test_letter=False): current_app.logger.info("PDF Letter {} reference {} created at {}, {} bytes".format( notification.id, notification.reference, notification.created_at, len(pdf_data))) upload_file_name = get_letter_pdf_filename( - notification.reference, notification.service.crown) + notification.reference, notification.service.crown, is_test_letter) + + if is_test_letter: + bucket_name = current_app.config['TEST_LETTERS_BUCKET_NAME'] + else: + bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME'] s3upload( filedata=pdf_data, region=current_app.config['AWS_REGION'], - bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], + bucket_name=bucket_name, file_location=upload_file_name, tags={Retention.KEY: Retention.ONE_WEEK} ) current_app.logger.info("Uploaded letters PDF {} to {} for notification id {}".format( - upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME'], notification.id)) + upload_file_name, bucket_name, notification.id)) def get_letter_pdf(notification): - bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME'] + is_test_letter = notification.key_type == KEY_TYPE_TEST and notification.template.is_precompiled_letter + if is_test_letter: + bucket_name = current_app.config['TEST_LETTERS_BUCKET_NAME'] + else: + bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME'] s3 = boto3.resource('s3') bucket = s3.Bucket(bucket_name) - for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification)): + for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification, is_test_letter)): obj = s3.Object( bucket_name=bucket_name, key=item.key diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index 7963ee8ce..b2311c081 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -265,6 +265,8 @@ def process_letter_notification(*, letter_data, api_key, template, reply_to_text queue=QueueNames.RESEARCH_MODE ) else: + if precompiled and api_key.key_type == KEY_TYPE_TEST: + upload_letter_pdf(notification, letter_content, is_test_letter=True) update_notification_status_by_reference(notification.reference, NOTIFICATION_DELIVERED) return notification diff --git a/requirements_for_test.txt b/requirements_for_test.txt index f558aab7a..3a1169708 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,5 +1,6 @@ -r requirements.txt flake8==3.5.0 +moto==1.1.25 pytest==3.4.2 pytest-env==0.6.2 pytest-mock==1.7.1 diff --git a/tests/app/celery/test_letters_pdf_tasks.py b/tests/app/celery/test_letters_pdf_tasks.py index 79650c1b5..757be3def 100644 --- a/tests/app/celery/test_letters_pdf_tasks.py +++ b/tests/app/celery/test_letters_pdf_tasks.py @@ -28,25 +28,6 @@ def test_should_have_decorated_tasks_functions(): assert create_letters_pdf.__wrapped__.__name__ == 'create_letters_pdf' -@pytest.mark.parametrize('crown_flag,expected_crown_text', [ - (True, 'C'), - (False, 'N'), -]) -@freeze_time("2017-12-04 17:29:00") -def test_get_letter_pdf_filename_returns_correct_filename( - notify_api, mocker, crown_flag, expected_crown_text): - filename = get_letter_pdf_filename(reference='foo', crown=crown_flag) - - assert filename == '2017-12-04/NOTIFY.FOO.D.2.C.{}.20171204172900.PDF'.format(expected_crown_text) - - -@freeze_time("2017-12-04 17:31:00") -def test_get_letter_pdf_filename_returns_tomorrows_filename(notify_api, mocker): - filename = get_letter_pdf_filename(reference='foo', crown=True) - - assert filename == '2017-12-05/NOTIFY.FOO.D.2.C.C.20171204173100.PDF' - - @pytest.mark.parametrize('personalisation', [{'name': 'test'}, None]) def test_get_letters_pdf_calls_notifications_template_preview_service_correctly( notify_api, mocker, client, sample_letter_template, personalisation): diff --git a/tests/app/letters/test_letter_utils.py b/tests/app/letters/test_letter_utils.py index 244c21f6a..04cb8eea9 100644 --- a/tests/app/letters/test_letter_utils.py +++ b/tests/app/letters/test_letter_utils.py @@ -1,6 +1,26 @@ import pytest +from datetime import datetime -from app.letters.utils import get_bucket_prefix_for_notification +import boto3 +from flask import current_app +from freezegun import freeze_time +from moto import mock_s3 + +from app.letters.utils import get_bucket_prefix_for_notification, get_letter_pdf_filename, get_letter_pdf +from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEST, PRECOMPILED_TEMPLATE_NAME + +FROZEN_DATE_TIME = "2018-03-14 17:00:00" + + +@pytest.fixture() +@freeze_time(FROZEN_DATE_TIME) +def sample_precompiled_letter_notification_using_test_key(sample_letter_notification): + sample_letter_notification.template.hidden = True + sample_letter_notification.template.name = PRECOMPILED_TEMPLATE_NAME + sample_letter_notification.key_type = KEY_TYPE_TEST + sample_letter_notification.reference = 'foo' + sample_letter_notification.created_at = datetime.utcnow() + return sample_letter_notification def test_get_bucket_prefix_for_notification_valid_notification(sample_notification): @@ -13,6 +33,70 @@ def test_get_bucket_prefix_for_notification_valid_notification(sample_notificati ).upper() +@freeze_time(FROZEN_DATE_TIME) +def test_get_bucket_prefix_for_notification_precompiled_letter_using_test_key( + sample_precompiled_letter_notification_using_test_key +): + bucket_prefix = get_bucket_prefix_for_notification( + sample_precompiled_letter_notification_using_test_key, is_test_letter=True) + + assert bucket_prefix == 'NOTIFY.{}'.format( + sample_precompiled_letter_notification_using_test_key.reference).upper() + + def test_get_bucket_prefix_for_notification_invalid_notification(): with pytest.raises(AttributeError): get_bucket_prefix_for_notification(None) + + +@pytest.mark.parametrize('crown_flag,expected_crown_text', [ + (True, 'C'), + (False, 'N'), +]) +@freeze_time("2017-12-04 17:29:00") +def test_get_letter_pdf_filename_returns_correct_filename( + notify_api, mocker, crown_flag, expected_crown_text): + filename = get_letter_pdf_filename(reference='foo', crown=crown_flag) + + assert filename == '2017-12-04/NOTIFY.FOO.D.2.C.{}.20171204172900.PDF'.format(expected_crown_text) + + +@freeze_time("2017-12-04 17:29:00") +def test_get_letter_pdf_filename_returns_correct_filename_for_test_letters( + notify_api, mocker): + filename = get_letter_pdf_filename(reference='foo', crown='C', is_test_letter=True) + + assert filename == 'NOTIFY.FOO.D.2.C.C.20171204172900.PDF' + + +@freeze_time("2017-12-04 17:31:00") +def test_get_letter_pdf_filename_returns_tomorrows_filename(notify_api, mocker): + filename = get_letter_pdf_filename(reference='foo', crown=True) + + assert filename == '2017-12-05/NOTIFY.FOO.D.2.C.C.20171204173100.PDF' + + +@mock_s3 +@pytest.mark.parametrize('bucket_config_name,filename_format', [ + ('TEST_LETTERS_BUCKET_NAME', 'NOTIFY.FOO.D.2.C.C.%Y%m%d%H%M%S.PDF'), + ('LETTERS_PDF_BUCKET_NAME', '%Y-%m-%d/NOTIFY.FOO.D.2.C.C.%Y%m%d%H%M%S.PDF') +]) +@freeze_time(FROZEN_DATE_TIME) +def test_get_letter_pdf_gets_pdf_from_correct_bucket( + sample_precompiled_letter_notification_using_test_key, + bucket_config_name, + filename_format +): + if bucket_config_name == 'LETTERS_PDF_BUCKET_NAME': + sample_precompiled_letter_notification_using_test_key.key_type = KEY_TYPE_NORMAL + + bucket_name = current_app.config[bucket_config_name] + filename = datetime.utcnow().strftime(filename_format) + conn = boto3.resource('s3', region_name='eu-west-1') + conn.create_bucket(Bucket=bucket_name) + s3 = boto3.client('s3', region_name='eu-west-1') + s3.put_object(Bucket=bucket_name, Key=filename, Body=b'pdf_content') + + ret = get_letter_pdf(sample_precompiled_letter_notification_using_test_key) + + assert ret == b'pdf_content' diff --git a/tests/app/v2/notifications/test_post_letter_notifications.py b/tests/app/v2/notifications/test_post_letter_notifications.py index f426e2efa..e145f5e88 100644 --- a/tests/app/v2/notifications/test_post_letter_notifications.py +++ b/tests/app/v2/notifications/test_post_letter_notifications.py @@ -1,4 +1,5 @@ import uuid +from unittest.mock import ANY from flask import json from flask import url_for @@ -29,9 +30,13 @@ test_address = { } -def letter_request(client, data, service_id, key_type=KEY_TYPE_NORMAL, _expected_status=201): +def letter_request(client, data, service_id, key_type=KEY_TYPE_NORMAL, _expected_status=201, precompiled=False): + if precompiled: + url = url_for('v2_notifications.post_precompiled_letter_notification') + else: + url = url_for('v2_notifications.post_notification', notification_type=LETTER_TYPE) resp = client.post( - url_for('v2_notifications.post_notification', notification_type=LETTER_TYPE), + url, data=json.dumps(data), headers=[ ('Content-Type', 'application/json'), @@ -382,6 +387,30 @@ def test_post_letter_notification_is_delivered_if_in_trial_mode_and_using_test_k assert not fake_create_letter_task.called +def test_post_letter_notification_is_delivered_and_has_pdf_uploaded_to_test_letters_bucket_using_test_key( + client, + notify_user, + mocker +): + sample_letter_service = create_service(service_permissions=['letter', 'precompiled_letter']) + s3mock = mocker.patch('app.v2.notifications.post_notifications.upload_letter_pdf') + mocker.patch('app.v2.notifications.post_notifications.pdf_page_count', return_value=1) + data = { + "reference": "letter-reference", + "content": "bGV0dGVyLWNvbnRlbnQ=" + } + letter_request( + client, + data=data, + service_id=str(sample_letter_service.id), + key_type=KEY_TYPE_TEST, + precompiled=True) + + notification = Notification.query.one() + assert notification.status == NOTIFICATION_DELIVERED + s3mock.assert_called_once_with(ANY, b'letter-content', is_test_letter=True) + + def test_post_letter_notification_persists_notification_reply_to_text( client, notify_db_session, mocker ):