From d62b350fcb932e29db6b086e43c938462477c9d0 Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Mon, 4 Dec 2017 16:29:04 +0000 Subject: [PATCH 1/3] Add config for letters pdf s3 bucket different environments --- app/config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/config.py b/app/config.py index 1ba975619..2294016d5 100644 --- a/app/config.py +++ b/app/config.py @@ -314,6 +314,7 @@ class Development(Config): SQLALCHEMY_ECHO = False NOTIFY_EMAIL_DOMAIN = 'notify.tools' CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' + LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf' DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp' NOTIFY_ENVIRONMENT = 'development' NOTIFICATION_QUEUE_PREFIX = 'development' @@ -335,6 +336,7 @@ class Test(Config): DEBUG = True TESTING = True CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' + LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf' DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp' STATSD_ENABLED = True STATSD_HOST = "localhost" @@ -373,6 +375,7 @@ class Preview(Config): NOTIFY_EMAIL_DOMAIN = 'notify.works' NOTIFY_ENVIRONMENT = 'preview' CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' + LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf' DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp' FROM_NUMBER = 'preview' API_RATE_LIMIT_ENABLED = True @@ -383,6 +386,7 @@ class Staging(Config): NOTIFY_EMAIL_DOMAIN = 'staging-notify.works' NOTIFY_ENVIRONMENT = 'staging' CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload' + LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf' DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp' STATSD_ENABLED = True FROM_NUMBER = 'stage' @@ -410,6 +414,7 @@ class Live(Config): NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk' NOTIFY_ENVIRONMENT = 'live' CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload' + LETTERS_PDF_BUCKET_NAME = 'live-letters-pdf' DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp' STATSD_ENABLED = True FROM_NUMBER = 'GOVUK' @@ -429,6 +434,7 @@ class Sandbox(CloudFoundryConfig): NOTIFY_EMAIL_DOMAIN = 'notify.works' NOTIFY_ENVIRONMENT = 'sandbox' CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload' + LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf' DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp' FROM_NUMBER = 'sandbox' REDIS_ENABLED = False From aa39f47d8e3f78f6372cea091bae26f3417dc441 Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Mon, 4 Dec 2017 16:39:32 +0000 Subject: [PATCH 2/3] Added letters pdf S3 upload function - uses s3upload from notifications_utils which will create an s3 bucket if it doesn't already exist --- app/aws/s3.py | 23 +++++++++++++++++++++++ tests/app/aws/test_s3.py | 22 +++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index 927eb0ad1..d5acb69fd 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -5,7 +5,10 @@ from flask import current_app import pytz from boto3 import client, resource +from notifications_utils.s3 import s3upload as utils_s3upload + FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv' +LETTERS_PDF_FILE_LOCATION_STRUCTURE = '{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf' def get_s3_file(bucket_name, file_location): @@ -76,3 +79,23 @@ def remove_transformed_dvla_file(job_id): file_location = '{}-dvla-job.text'.format(job_id) obj = get_s3_object(bucket_name, file_location) return obj.delete() + + +def upload_letters_pdf(reference, crown, filedata): + now = datetime.utcnow() + upload_file_name = LETTERS_PDF_FILE_LOCATION_STRUCTURE.format( + folder=now.date().isoformat(), + reference=reference, + duplex="D", + letter_class="2", + colour="C", + crown="C" if crown else "N", + date=now.strftime('%Y%m%d%H%M%S') + ) + + utils_s3upload( + filedata=filedata, + region=current_app.config['AWS_REGION'], + bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], + file_location=upload_file_name + ) diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index d6e4c1b75..d144a29f0 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -1,5 +1,6 @@ from unittest.mock import call from datetime import datetime, timedelta +import pytest from flask import current_app @@ -9,7 +10,8 @@ from app.aws.s3 import ( get_s3_bucket_objects, get_s3_file, filter_s3_bucket_objects_within_date_range, - remove_transformed_dvla_file + remove_transformed_dvla_file, + upload_letters_pdf ) from tests.app.conftest import datetime_in_past @@ -139,3 +141,21 @@ def test_get_s3_bucket_objects_does_not_return_outside_of_date_range(notify_api, filtered_items = filter_s3_bucket_objects_within_date_range(s3_objects_stub) assert len(filtered_items) == 0 + + +@pytest.mark.parametrize('crown_flag,expected_crown_text', [ + (True, 'C'), + (False, 'N'), +]) +@freeze_time("2017-12-04 15:00:00") +def test_upload_letters_pdf_calls_utils_s3upload_with_correct_args( + notify_api, mocker, crown_flag, expected_crown_text): + s3_upload_mock = mocker.patch('app.aws.s3.utils_s3upload') + upload_letters_pdf(reference='foo', crown=crown_flag, filedata='some_data') + + s3_upload_mock.assert_called_with( + filedata='some_data', + region='eu-west-1', + bucket_name='test-letters-pdf', + file_location='2017-12-04/NOTIFY.foo.D.2.C.{}.20171204150000.pdf'.format(expected_crown_text) + ) From 7b390e74c4733220d35d6056786ba6467c419f83 Mon Sep 17 00:00:00 2001 From: Ken Tsang Date: Tue, 5 Dec 2017 16:03:08 +0000 Subject: [PATCH 3/3] Upper case the filename to meet requirements --- app/aws/s3.py | 2 +- tests/app/aws/test_s3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index d5acb69fd..a64be797c 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -91,7 +91,7 @@ def upload_letters_pdf(reference, crown, filedata): colour="C", crown="C" if crown else "N", date=now.strftime('%Y%m%d%H%M%S') - ) + ).upper() utils_s3upload( filedata=filedata, diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index d144a29f0..e610560db 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -157,5 +157,5 @@ def test_upload_letters_pdf_calls_utils_s3upload_with_correct_args( filedata='some_data', region='eu-west-1', bucket_name='test-letters-pdf', - file_location='2017-12-04/NOTIFY.foo.D.2.C.{}.20171204150000.pdf'.format(expected_crown_text) + file_location='2017-12-04/NOTIFY.FOO.D.2.C.{}.20171204150000.PDF'.format(expected_crown_text) )