diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 927fee6ea..1218e8852 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -2,42 +2,30 @@ import json import os -def find_by_service_name(services, service_name): - for i in range(len(services)): - if services[i]['name'] == service_name: - return services[i] - return None +class CloudfoundryConfig: + def __init__(self): + self.parsed_services = json.loads(os.environ.get('VCAP_SERVICES') or '{}') + buckets = self.parsed_services.get('s3') or [] + self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets} + self._empty_bucket_credentials = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + @property + def redis_url(self): + try: + return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace( + 'redis://', + 'rediss://' + ) + except KeyError: + return os.environ.get('REDIS_URL') + + def s3_credentials(self, service_name): + return self.s3_buckets.get(service_name) or self._empty_bucket_credentials -def extract_cloudfoundry_config(): - vcap_services = json.loads(os.environ['VCAP_SERVICES']) - - # Redis config - os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis', 'rediss') - - # CSV Upload Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}") - if bucket_service: - os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region'] - - # Contact List Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}") - if bucket_service: - os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region'] - - # Logo Upload Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], f"notifications-admin-logo-upload-bucket-{os.environ['DEPLOY_ENV']}") - if bucket_service: - os.environ['LOGO_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['LOGO_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['LOGO_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['LOGO_UPLOAD_REGION'] = bucket_service['credentials']['region'] +cloud_config = CloudfoundryConfig() diff --git a/app/config.py b/app/config.py index e99a79eaa..668d02987 100644 --- a/app/config.py +++ b/app/config.py @@ -1,11 +1,7 @@ import json import os -if os.environ.get('VCAP_SERVICES'): - # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate - # standard environment variables from it - from app.cloudfoundry_config import extract_cloudfoundry_config - extract_cloudfoundry_config() +from app.cloudfoundry_config import cloud_config class Config(object): @@ -60,7 +56,7 @@ class Config(object): AWS_REGION = os.environ.get('AWS_REGION') - REDIS_URL = os.environ.get('REDIS_URL') + REDIS_URL = cloud_config.redis_url REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '1') == '1' # as defined in api db migration 0331_add_broadcast_org.py @@ -93,22 +89,24 @@ class Development(Config): ASSET_PATH = '/static/' # Buckets - CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' # created in gsa sandbox - CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CSV_UPLOAD_REGION = os.environ.get('AWS_REGION') - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'local-contact-list' # created in gsa sandbox - CONTACT_LIST_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CONTACT_LIST_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CONTACT_LIST_UPLOAD_REGION = os.environ.get('AWS_REGION') - LOGO_UPLOAD_BUCKET_NAME = 'local-public-logos-tools' # created in gsa sandbox - LOGO_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - LOGO_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - LOGO_UPLOAD_REGION = os.environ.get('AWS_REGION') - # MOU_BUCKET_NAME = 'local-notify-tools-mou' # not created in gsa sandbox - # TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters' # not created in gsa sandbox - # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = - # 'development-letters-precompiled-originals-backup' # not created in sandbox + CSV_UPLOAD_BUCKET = { + 'bucket': 'local-notifications-csv-upload', + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } + CONTACT_LIST_BUCKET = { + 'bucket': 'local-contact-list', + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } + LOGO_UPLOAD_BUCKET = { + 'bucket': 'local-public-logos-tools', + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } # credential overrides DANGEROUS_SALT = 'dev-notify-salt' @@ -124,14 +122,6 @@ class Test(Development): ASSET_DOMAIN = 'static.example.com' ASSET_PATH = 'https://static.example.com/' - # none of these buckets actually exist - CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'test-contact-list' - LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test' - # MOU_BUCKET_NAME = 'test-mou' - # TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters' - # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup' - API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to' REDIS_URL = 'redis://you-forgot-to-mock-a-redis-call-to' ANTIVIRUS_API_HOST = 'https://test-antivirus' @@ -149,21 +139,12 @@ class Production(Config): DEBUG = False # buckets - CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME') - CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY') - CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY') - CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION') - CONTACT_LIST_UPLOAD_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME') - CONTACT_LIST_UPLOAD_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY') - CONTACT_LIST_UPLOAD_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY') - CONTACT_LIST_UPLOAD_REGION = os.environ.get('CONTACT_LIST_REGION') - LOGO_UPLOAD_BUCKET_NAME = os.environ.get('LOGO_UPLOAD_BUCKET_NAME') - LOGO_UPLOAD_ACCESS_KEY = os.environ.get('LOGO_UPLOAD_ACCESS_KEY') - LOGO_UPLOAD_SECRET_KEY = os.environ.get('LOGO_UPLOAD_SECRET_KEY') - LOGO_UPLOAD_REGION = os.environ.get('LOGO_UPLOAD_REGION') - # MOU_BUCKET_NAME = os.environ.get('MOU_UPLOAD_BUCKET_NAME') - # TRANSIENT_UPLOADED_LETTERS = 'prototype-transient-uploaded-letters' # not created in gsa sandbox - # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'prototype-letters-precompiled-originals-backup' # not in sandbox + CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( + f"notifications-api-csv-upload-bucket-{os.environ['NOTIFY_ENVIRONMENT']}") + CONTACT_LIST_BUCKET = cloud_config.s3_credentials( + f"notifications-api-contact-list-bucket-{os.environ['NOTIFY_ENVIRONMENT']}") + LOGO_UPLOAD_BUCKET = cloud_config.s3_credentials( + f"notifications-admin-logo-upload-bucket-{os.environ['NOTIFY_ENVIRONMENT']}") class Staging(Production): diff --git a/app/models/contact_list.py b/app/models/contact_list.py index 17f97c40b..dfecaf50a 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -44,21 +44,25 @@ class ContactList(JSONModel): contact_list_id=contact_list_id, )) + @staticmethod + def get_bucket_credentials(key): + return current_app.config['CONTACT_LIST_BUCKET'][key] + @staticmethod def get_bucket_name(): - return current_app.config['CONTACT_LIST_UPLOAD_BUCKET_NAME'] + return ContactList.get_bucket_credentials('bucket') @staticmethod def get_access_key(): - return current_app.config['CONTACT_LIST_UPLOAD_ACCESS_KEY'] + return ContactList.get_bucket_credentials('access_key_id') @staticmethod def get_secret_key(): - return current_app.config['CONTACT_LIST_UPLOAD_SECRET_KEY'] + return ContactList.get_bucket_credentials('secret_access_key') @staticmethod def get_region(): - return current_app.config['CONTACT_LIST_UPLOAD_REGION'] + return ContactList.get_bucket_credentials('region') @staticmethod def get_filename(service_id, upload_id): diff --git a/app/s3_client/s3_csv_client.py b/app/s3_client/s3_csv_client.py index 93672d74b..8422a11e8 100644 --- a/app/s3_client/s3_csv_client.py +++ b/app/s3_client/s3_csv_client.py @@ -15,11 +15,11 @@ FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv' def get_csv_location(service_id, upload_id): return ( - current_app.config['CSV_UPLOAD_BUCKET_NAME'], + current_app.config['CSV_UPLOAD_BUCKET']['bucket'], FILE_LOCATION_STRUCTURE.format(service_id, upload_id), - current_app.config['CSV_UPLOAD_ACCESS_KEY'], - current_app.config['CSV_UPLOAD_SECRET_KEY'], - current_app.config['CSV_UPLOAD_REGION'], + current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'], + current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'], + current_app.config['CSV_UPLOAD_BUCKET']['region'], ) diff --git a/app/s3_client/s3_logo_client.py b/app/s3_client/s3_logo_client.py index b7f6a523e..aba076624 100644 --- a/app/s3_client/s3_logo_client.py +++ b/app/s3_client/s3_logo_client.py @@ -15,14 +15,18 @@ LETTER_TEMP_LOGO_LOCATION = 'letters/static/images/letter-template/temp-{user_id def get_logo_location(filename=None): return ( - current_app.config['LOGO_UPLOAD_BUCKET_NAME'], + bucket_creds('bucket'), filename, - current_app.config['LOGO_UPLOAD_ACCESS_KEY'], - current_app.config['LOGO_UPLOAD_SECRET_KEY'], - current_app.config['LOGO_UPLOAD_REGION'], + bucket_creds('access_key_id'), + bucket_creds('secret_access_key'), + bucket_creds('region'), ) +def bucket_creds(key): + return current_app.config['LOGO_UPLOAD_BUCKET'][key] + + def delete_s3_object(filename): get_s3_object(*get_logo_location(filename)).delete() @@ -37,10 +41,10 @@ def persist_logo(old_name, new_name): def get_s3_objects_filter_by_prefix(prefix): - bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME'] - session = Session(aws_access_key_id=current_app.config['LOGO_UPLOAD_ACCESS_KEY'], - aws_secret_access_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'], - region_name=current_app.config['LOGO_UPLOAD_REGION']) + bucket_name = bucket_creds('bucket') + session = Session(aws_access_key_id=bucket_creds('access_key_id'), + aws_secret_access_key=bucket_creds('secret_access_key'), + region_name=bucket_creds('region')) s3 = session.resource('s3') return s3.Bucket(bucket_name).objects.filter(Prefix=prefix) @@ -59,15 +63,15 @@ def upload_email_logo(filename, filedata, user_id): unique_id=str(uuid.uuid4()), filename=filename ) - bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME'] + bucket_name = bucket_creds('bucket') utils_s3upload( filedata=filedata, - region=current_app.config['LOGO_UPLOAD_REGION'], + region=bucket_creds('region'), bucket_name=bucket_name, file_location=upload_file_name, content_type='image/png', - access_key=current_app.config['LOGO_UPLOAD_ACCESS_KEY'], - secret_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'], + access_key=bucket_creds('access_key_id'), + secret_key=bucket_creds('secret_access_key'), ) return upload_file_name @@ -79,15 +83,15 @@ def upload_letter_temp_logo(filename, filedata, user_id): unique_id=str(uuid.uuid4()), filename=filename ) - bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME'] + bucket_name = bucket_creds('bucket') utils_s3upload( filedata=filedata, - region=current_app.config['LOGO_UPLOAD_REGION'], + region=bucket_creds('region'), bucket_name=bucket_name, file_location=upload_filename, content_type='image/svg+xml', - access_key=current_app.config['LOGO_UPLOAD_ACCESS_KEY'], - secret_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'], + access_key=bucket_creds('access_key_id'), + secret_key=bucket_creds('secret_access_key'), ) return upload_filename diff --git a/tests/app/main/views/uploads/test_upload_contact_list.py b/tests/app/main/views/uploads/test_upload_contact_list.py index 1a58c7453..7f69be12f 100644 --- a/tests/app/main/views/uploads/test_upload_contact_list.py +++ b/tests/app/main/views/uploads/test_upload_contact_list.py @@ -208,7 +208,7 @@ def test_upload_csv_file_shows_error_banner( mock_upload.assert_called_once_with( filedata='', region='us-west-2', - bucket_name='test-contact-list', + bucket_name='local-contact-list', file_location=f"service-{SERVICE_ONE_ID}-notify/{fake_uuid}.csv", access_key=default_access_key, secret_key=default_secret_key, diff --git a/tests/app/s3_client/test_s3_csv_client.py b/tests/app/s3_client/test_s3_csv_client.py index 1ffb498b0..b2fc04341 100644 --- a/tests/app/s3_client/test_s3_csv_client.py +++ b/tests/app/s3_client/test_s3_csv_client.py @@ -4,7 +4,7 @@ from app.s3_client.s3_csv_client import set_metadata_on_csv_upload def test_sets_metadata(client_request, mocker): - mocked_s3_object = Mock(bucket_name='test-notifications-csv-upload', key='service-1234-notify/5678.csv') + mocked_s3_object = Mock(bucket_name='local-notifications-csv-upload', key='service-1234-notify/5678.csv') mocked_get_s3_object = mocker.patch( 'app.s3_client.s3_csv_client.get_csv_upload', return_value=mocked_s3_object, @@ -14,7 +14,7 @@ def test_sets_metadata(client_request, mocker): mocked_get_s3_object.assert_called_once_with('1234', '5678') mocked_s3_object.copy_from.assert_called_once_with( - CopySource='test-notifications-csv-upload/service-1234-notify/5678.csv', + CopySource='local-notifications-csv-upload/service-1234-notify/5678.csv', Metadata={'baz': 'True', 'foo': 'bar'}, MetadataDirective='REPLACE', ServerSideEncryption='AES256', diff --git a/tests/app/s3_client/test_s3_logo_client.py b/tests/app/s3_client/test_s3_logo_client.py index 49f2062cb..4011b23c6 100644 --- a/tests/app/s3_client/test_s3_logo_client.py +++ b/tests/app/s3_client/test_s3_logo_client.py @@ -21,6 +21,12 @@ from app.s3_client.s3_logo_client import ( ) bucket = 'test_bucket' +bucket_credentials = { + 'bucket': bucket, + 'access_key_id': default_access_key, + 'secret_access_key': default_secret_key, + 'region': default_region +} data = {'data': 'some_data'} filename = 'test.png' svg_filename = 'test.svg' @@ -45,7 +51,7 @@ def letter_upload_filename(fake_uuid): def test_upload_email_logo_calls_correct_args(client_request, mocker, fake_uuid, upload_filename): mocker.patch('uuid.uuid4', return_value=upload_id) - mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET_NAME': bucket}) + mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET': bucket_credentials}) mocked_s3_upload = mocker.patch('app.s3_client.s3_logo_client.utils_s3upload') upload_email_logo(filename=filename, user_id=fake_uuid, filedata=data) @@ -63,7 +69,7 @@ def test_upload_email_logo_calls_correct_args(client_request, mocker, fake_uuid, def test_upload_letter_temp_logo_calls_correct_args(mocker, fake_uuid, letter_upload_filename): mocker.patch('uuid.uuid4', return_value=upload_id) - mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET_NAME': bucket}) + mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET': bucket_credentials}) mocked_s3_upload = mocker.patch('app.s3_client.s3_logo_client.utils_s3upload') new_filename = upload_letter_temp_logo(filename=svg_filename, user_id=fake_uuid, filedata=data) @@ -81,7 +87,7 @@ def test_upload_letter_temp_logo_calls_correct_args(mocker, fake_uuid, letter_up def test_persist_logo(client_request, mocker, fake_uuid, upload_filename): - mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET_NAME': bucket}) + mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET': bucket_credentials}) mocked_get_s3_object = mocker.patch('app.s3_client.s3_logo_client.get_s3_object') mocked_delete_s3_object = mocker.patch('app.s3_client.s3_logo_client.delete_s3_object') diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index f2df67971..79ad640e2 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -3,7 +3,14 @@ import os import pytest -from app.cloudfoundry_config import extract_cloudfoundry_config +from app.cloudfoundry_config import CloudfoundryConfig + +bucket_credentials = { + 'access_key_id': 'contact-list-access', + 'bucket': 'contact-list-bucket', + 'region': 'us-gov-west-1', + 'secret_access_key': 'contact-list-secret' +} @pytest.fixture @@ -26,22 +33,39 @@ def vcap_services(): }, { 'name': 'notifications-api-contact-list-bucket-test', - 'credentials': { - 'access_key_id': 'contact-list-access', - 'bucket': 'contact-list-bucket', - 'region': 'us-gov-west-1', - 'secret_access_key': 'contact-list-secret' - } + 'credentials': bucket_credentials } ], } -def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services): - os.environ['DEPLOY_ENV'] = 'test' +def test_redis_url(vcap_services): os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) - extract_cloudfoundry_config() - assert os.environ['REDIS_URL'] == 'rediss://xxx:6379' - assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket' - assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket' + assert CloudfoundryConfig().redis_url == 'rediss://xxx:6379' + + +def test_redis_url_falls_back_to_REDIS_URL(): + expected = 'rediss://yyy:6379' + os.environ['REDIS_URL'] = expected + os.environ['VCAP_SERVICES'] = "" + + assert CloudfoundryConfig().redis_url == expected + + +def test_s3_bucket_credentials(vcap_services): + os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) + + assert CloudfoundryConfig().s3_credentials('notifications-api-contact-list-bucket-test') == bucket_credentials + + +def test_s3_bucket_credentials_falls_back_to_empty_creds(): + os.environ['VCAP_SERVICES'] = "" + expected = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + assert CloudfoundryConfig().s3_credentials('bucket') == expected diff --git a/tests/app/test_config.py b/tests/app/test_config.py deleted file mode 100644 index 152dd5f35..000000000 --- a/tests/app/test_config.py +++ /dev/null @@ -1,53 +0,0 @@ -import importlib -import os -from unittest import mock - -import pytest - -from app import config - - -def cf_conf(): - os.environ['REDIS_URL'] = 'rediss://xxx:6379' - - -@pytest.fixture -def reload_config(os_environ): - """ - Reset config, by simply re-running config.py from a fresh environment - """ - old_env = os.environ.copy() - os.environ.clear() - - yield - - os.environ.clear() - for k, v in old_env.items(): - os.environ[k] = v - importlib.reload(config) - - -def test_load_cloudfoundry_config_if_available(reload_config): - os.environ['REDIS_URL'] = 'some uri' - os.environ['VCAP_SERVICES'] = 'some json blob' - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf): - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert os.environ['REDIS_URL'] == 'rediss://xxx:6379' - assert config.Config.REDIS_URL == 'rediss://xxx:6379' - - -def test_load_config_if_cloudfoundry_not_available(reload_config): - os.environ['REDIS_URL'] = 'redis://xxx:6379' - os.environ.pop('VCAP_SERVICES', None) - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config: - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert not cf_config.called - - assert os.environ['REDIS_URL'] == 'redis://xxx:6379' - assert config.Config.REDIS_URL == 'redis://xxx:6379'