diff --git a/app/main/uploader.py b/app/main/uploader.py index 3dfd4d87b..a55eab1ea 100644 --- a/app/main/uploader.py +++ b/app/main/uploader.py @@ -1,28 +1,33 @@ import botocore from boto3 import resource +from flask import current_app - -BUCKET_NAME = 'service-{}-notify' +FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv' def s3upload(upload_id, service_id, filedata, region): s3 = resource('s3') - bucket_name = BUCKET_NAME.format(service_id) + bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME'] contents = filedata['data'] exists = True try: - s3.meta.client.head_bucket(Bucket=bucket_name) + s3.meta.client.head_bucket( + Bucket=bucket_name) except botocore.exceptions.ClientError as e: error_code = int(e.response['Error']['Code']) if error_code == 404: exists = False + else: + current_app.logger.error( + "Unable to create s3 bucket {}".format(bucket_name)) + raise e if not exists: s3.create_bucket(Bucket=bucket_name, CreateBucketConfiguration={'LocationConstraint': region}) - upload_file_name = "{}.csv".format(upload_id) + upload_file_name = FILE_LOCATION_STRUCTURE.format(service_id, upload_id) key = s3.Object(bucket_name, upload_file_name) key.put(Body=contents, ServerSideEncryption='AES256') @@ -31,14 +36,12 @@ def s3download(service_id, upload_id): contents = '' try: s3 = resource('s3') - bucket_name = BUCKET_NAME.format(service_id) - upload_file_name = "{}.csv".format(upload_id) + bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME'] + upload_file_name = FILE_LOCATION_STRUCTURE.format(service_id, upload_id) key = s3.Object(bucket_name, upload_file_name) contents = key.get()['Body'].read().decode('utf-8') except botocore.exceptions.ClientError as e: - err = e.response['Error'] - if err['Code'] == 'NoSuchBucket': - err_msg = '{}:{}'.format(err['BucketName'], err['Message']) - # TODO properly log error - print(err_msg) + current_app.logger.error("Unable to download s3 file {}".format( + FILE_LOCATION_STRUCTURE.format(service_id, upload_id))) + raise e return contents diff --git a/app/notify_client/job_api_client.py b/app/notify_client/job_api_client.py index b24643c8c..071eb4760 100644 --- a/app/notify_client/job_api_client.py +++ b/app/notify_client/job_api_client.py @@ -24,8 +24,6 @@ class JobApiClient(BaseAPIClient): "id": job_id, "template": template_id, "original_file_name": original_file_name, - "bucket_name": "service-{}-notify".format(service_id), - "file_name": "{}.csv".format(job_id), "notification_count": notification_count } diff --git a/config.py b/config.py index d91ff3f83..beec9c652 100644 --- a/config.py +++ b/config.py @@ -35,6 +35,8 @@ class Config(object): SHOW_STYLEGUIDE = True TOKEN_MAX_AGE_SECONDS = 3600 WTF_CSRF_ENABLED = True + CSV_UPLOAD_BUCKET_NAME = '{}-notifications-csv-upload'.format( + os.environ['NOTIFY_ADMIN_ENVIRONMENT'].split('config.')[1].lower()) EMAIL_DOMAIN_REGEXES = [ "gov\.uk", diff --git a/tests/__init__.py b/tests/__init__.py index 27cfd2a84..91ff5a87e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -91,8 +91,6 @@ def job_json(): 'service': 1, 'template': 1, 'original_file_name': 'thisisatest.csv', - 'bucket_name': 'service-1-{}-notify'.format(job_id), - 'file_name': '{}.csv'.format(job_id), 'created_at': created_at, 'notification_count': 1, 'notifications_sent': 1, diff --git a/tests/app/main/notify_client/test_job_client.py b/tests/app/main/notify_client/test_job_client.py index 1eb2be38d..b90b9b122 100644 --- a/tests/app/main/notify_client/test_job_client.py +++ b/tests/app/main/notify_client/test_job_client.py @@ -13,8 +13,6 @@ def test_client_creates_job_data_correctly(mocker): "id": job_id, "template": template_id, "original_file_name": original_file_name, - "bucket_name": "service-{}-notify".format(service_id), - "file_name": "{}.csv".format(job_id), "notification_count": 1 }