Provide s3 credentials for each individual bucket

app/s3_client/__init__.py

@@ -0,0 +1,46 @@
+import os
+
+import botocore
+from boto3 import Session
+from flask import current_app
+
+default_access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+default_secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+
+
+def get_s3_object(bucket_name, filename, access_key=default_access_key, secret_key=default_secret_key):
+    # To inspect contents: obj.get()['Body'].read().decode('utf-8')
+    session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key)
+    s3 = session.resource('s3')
+    obj = s3.Object(bucket_name, filename)
+    return obj
+
+
+def get_s3_metadata(obj):
+    try:
+        return obj.get()['Metadata']
+    except botocore.exceptions.ClientError as client_error:
+        current_app.logger.error(f"Unable to download s3 file {obj.bucket_name}/{obj.key}")
+        raise client_error
+
+
+def set_s3_metadata(obj, **kwargs):
+    copy_from_object_result = obj.copy_from(
+        CopySource=f"{obj.bucket_name}/{obj.key}",
+        ServerSideEncryption='AES256',
+        Metadata={
+            key: str(value) for key, value in kwargs.items()
+        },
+        MetadataDirective='REPLACE',
+    )
+    return copy_from_object_result
+
+
+def get_s3_contents(obj):
+    contents = ''
+    try:
+        contents = obj.get()['Body'].read().decode('utf-8')
+    except botocore.exceptions.ClientError as client_error:
+        current_app.logger.error(f"Unable to download s3 file {obj.bucket_name}/{obj.key}")
+        raise client_error
+    return contents

app/s3_client/s3_csv_client.py

@@ -1,68 +1,52 @@
 import uuid
 
-import botocore
 from flask import current_app
 from notifications_utils.s3 import s3upload as utils_s3upload
 
-from app.s3_client.s3_logo_client import get_s3_object
+from app.s3_client import (
+    get_s3_contents,
+    get_s3_metadata,
+    get_s3_object,
+    set_s3_metadata,
+)
 
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
-def get_csv_location(service_id, upload_id, bucket=None):
+def get_csv_location(service_id, upload_id):
     return (
-        bucket or current_app.config['CSV_UPLOAD_BUCKET_NAME'],
+        current_app.config['CSV_UPLOAD_BUCKET_NAME'],
         FILE_LOCATION_STRUCTURE.format(service_id, upload_id),
+        current_app.config['CSV_UPLOAD_ACCESS_KEY'],
+        current_app.config['CSV_UPLOAD_SECRET_KEY'],
     )
 
 
-def get_csv_upload(service_id, upload_id, bucket=None):
-    return get_s3_object(*get_csv_location(service_id, upload_id, bucket))
+def get_csv_upload(service_id, upload_id):
+    return get_s3_object(*get_csv_location(service_id, upload_id))
 
 
-def s3upload(service_id, filedata, region, bucket=None):
+def s3upload(service_id, filedata, region):
     upload_id = str(uuid.uuid4())
-    bucket_name, file_location = get_csv_location(service_id, upload_id, bucket)
+    bucket_name, file_location, access_key, secret_key = get_csv_location(service_id, upload_id)
     utils_s3upload(
         filedata=filedata['data'],
         region=region,
         bucket_name=bucket_name,
         file_location=file_location,
+        access_key=access_key,
+        secret_key=secret_key,
     )
     return upload_id
 
 
-def s3download(service_id, upload_id, bucket=None):
-    contents = ''
-    try:
-        key = get_csv_upload(service_id, upload_id, bucket)
-        contents = key.get()['Body'].read().decode('utf-8')
-    except botocore.exceptions.ClientError as e:
-        current_app.logger.error("Unable to download s3 file {}".format(
-            FILE_LOCATION_STRUCTURE.format(service_id, upload_id)))
-        raise e
-    return contents
+def s3download(service_id, upload_id):
+    return get_s3_contents(get_csv_upload(service_id, upload_id))
 
 
-def set_metadata_on_csv_upload(service_id, upload_id, bucket=None, **kwargs):
-    copy_from_object_result = get_csv_upload(
-        service_id, upload_id, bucket=bucket
-    ).copy_from(
-        CopySource='{}/{}'.format(*get_csv_location(service_id, upload_id, bucket=bucket)),
-        ServerSideEncryption='AES256',
-        Metadata={
-            key: str(value) for key, value in kwargs.items()
-        },
-        MetadataDirective='REPLACE',
-    )
-    return copy_from_object_result
+def set_metadata_on_csv_upload(service_id, upload_id, **kwargs):
+    return set_s3_metadata(get_csv_upload(service_id, upload_id), **kwargs)
 
 
-def get_csv_metadata(service_id, upload_id, bucket=None):
-    try:
-        key = get_csv_upload(service_id, upload_id, bucket)
-        return key.get()['Metadata']
-    except botocore.exceptions.ClientError as e:
-        current_app.logger.error("Unable to download s3 file {}".format(
-            FILE_LOCATION_STRUCTURE.format(service_id, upload_id)))
-        raise e
+def get_csv_metadata(service_id, upload_id):
+    return get_s3_metadata(get_csv_upload(service_id, upload_id))

app/s3_client/s3_logo_client.py

@@ -1,9 +1,11 @@
 import uuid
 
-from boto3 import resource
+from boto3 import Session
 from flask import current_app
 from notifications_utils.s3 import s3upload as utils_s3upload
 
+from app.s3_client import get_s3_object
+
 TEMP_TAG = 'temp-{user_id}_'
 EMAIL_LOGO_LOCATION_STRUCTURE = '{temp}{unique_id}-{filename}'
 LETTER_PREFIX = 'letters/static/images/letter-template/'
@@ -11,30 +13,33 @@ LETTER_TEMP_TAG = LETTER_PREFIX + TEMP_TAG
 LETTER_TEMP_LOGO_LOCATION = 'letters/static/images/letter-template/temp-{user_id}_{unique_id}-{filename}'
 
 
-def get_s3_object(bucket_name, filename):
-    # To inspect contents: obj.get()['Body'].read().decode('utf-8')
-    s3 = resource('s3')
-    obj = s3.Object(bucket_name, filename)
-    return obj
+def get_logo_location(filename=None):
+    return (
+        current_app.config['LOGO_UPLOAD_BUCKET_NAME'],
+        filename,
+        current_app.config['LOGO_UPLOAD_ACCESS_KEY'],
+        current_app.config['LOGO_UPLOAD_SECRET_KEY'],
+    )
 
 
 def delete_s3_object(filename):
-    bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME']
-    get_s3_object(bucket_name, filename).delete()
+    get_s3_object(*get_logo_location(filename)).delete()
 
 
 def persist_logo(old_name, new_name):
     if old_name == new_name:
         return
-    bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME']
-    get_s3_object(bucket_name, new_name).copy_from(
+    bucket_name, filename, access_key, secret_key = get_logo_location(new_name)
+    get_s3_object(bucket_name, filename, access_key, secret_key).copy_from(
         CopySource='{}/{}'.format(bucket_name, old_name))
     delete_s3_object(old_name)
 
 
 def get_s3_objects_filter_by_prefix(prefix):
     bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME']
-    s3 = resource('s3')
+    session = Session(aws_access_key_id=current_app.config['LOGO_UPLOAD_ACCESS_KEY'],
+                      aws_secret_access_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'])
+    s3 = session.resource('s3')
     return s3.Bucket(bucket_name).objects.filter(Prefix=prefix)
 
 
@@ -58,7 +63,9 @@ def upload_email_logo(filename, filedata, region, user_id):
         region=region,
         bucket_name=bucket_name,
         file_location=upload_file_name,
-        content_type='image/png'
+        content_type='image/png',
+        access_key=current_app.config['LOGO_UPLOAD_ACCESS_KEY'],
+        secret_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'],
     )
 
     return upload_file_name
@@ -76,7 +83,9 @@ def upload_letter_temp_logo(filename, filedata, region, user_id):
         region=region,
         bucket_name=bucket_name,
         file_location=upload_filename,
-        content_type='image/svg+xml'
+        content_type='image/svg+xml',
+        access_key=current_app.config['LOGO_UPLOAD_ACCESS_KEY'],
+        secret_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'],
     )
 
     return upload_filename