Refactor method of retrieving credentials out of VCAP_SERVICES

This commit is contained in:
Ryan Ahearn
2022-10-20 08:03:32 -04:00
parent d87d673b85
commit 921c872bd1
10 changed files with 132 additions and 178 deletions

View File

@@ -2,42 +2,30 @@ import json
import os
def find_by_service_name(services, service_name):
for i in range(len(services)):
if services[i]['name'] == service_name:
return services[i]
return None
class CloudfoundryConfig:
def __init__(self):
self.parsed_services = json.loads(os.environ.get('VCAP_SERVICES') or '{}')
buckets = self.parsed_services.get('s3') or []
self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets}
self._empty_bucket_credentials = {
'bucket': '',
'access_key_id': '',
'secret_access_key': '',
'region': ''
}
@property
def redis_url(self):
try:
return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace(
'redis://',
'rediss://'
)
except KeyError:
return os.environ.get('REDIS_URL')
def s3_credentials(self, service_name):
return self.s3_buckets.get(service_name) or self._empty_bucket_credentials
def extract_cloudfoundry_config():
vcap_services = json.loads(os.environ['VCAP_SERVICES'])
# Redis config
os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis', 'rediss')
# CSV Upload Bucket Name
bucket_service = find_by_service_name(
vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}")
if bucket_service:
os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket']
os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region']
# Contact List Bucket Name
bucket_service = find_by_service_name(
vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}")
if bucket_service:
os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket']
os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region']
# Logo Upload Bucket Name
bucket_service = find_by_service_name(
vcap_services['s3'], f"notifications-admin-logo-upload-bucket-{os.environ['DEPLOY_ENV']}")
if bucket_service:
os.environ['LOGO_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket']
os.environ['LOGO_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
os.environ['LOGO_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
os.environ['LOGO_UPLOAD_REGION'] = bucket_service['credentials']['region']
cloud_config = CloudfoundryConfig()

View File

@@ -1,11 +1,7 @@
import json
import os
if os.environ.get('VCAP_SERVICES'):
# on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
# standard environment variables from it
from app.cloudfoundry_config import extract_cloudfoundry_config
extract_cloudfoundry_config()
from app.cloudfoundry_config import cloud_config
class Config(object):
@@ -60,7 +56,7 @@ class Config(object):
AWS_REGION = os.environ.get('AWS_REGION')
REDIS_URL = os.environ.get('REDIS_URL')
REDIS_URL = cloud_config.redis_url
REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '1') == '1'
# as defined in api db migration 0331_add_broadcast_org.py
@@ -93,22 +89,24 @@ class Development(Config):
ASSET_PATH = '/static/'
# Buckets
CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' # created in gsa sandbox
CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
CSV_UPLOAD_REGION = os.environ.get('AWS_REGION')
CONTACT_LIST_UPLOAD_BUCKET_NAME = 'local-contact-list' # created in gsa sandbox
CONTACT_LIST_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
CONTACT_LIST_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
CONTACT_LIST_UPLOAD_REGION = os.environ.get('AWS_REGION')
LOGO_UPLOAD_BUCKET_NAME = 'local-public-logos-tools' # created in gsa sandbox
LOGO_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
LOGO_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
LOGO_UPLOAD_REGION = os.environ.get('AWS_REGION')
# MOU_BUCKET_NAME = 'local-notify-tools-mou' # not created in gsa sandbox
# TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters' # not created in gsa sandbox
# PRECOMPILED_ORIGINALS_BACKUP_LETTERS =
# 'development-letters-precompiled-originals-backup' # not created in sandbox
CSV_UPLOAD_BUCKET = {
'bucket': 'local-notifications-csv-upload',
'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'),
'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'),
'region': os.environ.get('AWS_REGION')
}
CONTACT_LIST_BUCKET = {
'bucket': 'local-contact-list',
'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'),
'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'),
'region': os.environ.get('AWS_REGION')
}
LOGO_UPLOAD_BUCKET = {
'bucket': 'local-public-logos-tools',
'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'),
'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'),
'region': os.environ.get('AWS_REGION')
}
# credential overrides
DANGEROUS_SALT = 'dev-notify-salt'
@@ -124,14 +122,6 @@ class Test(Development):
ASSET_DOMAIN = 'static.example.com'
ASSET_PATH = 'https://static.example.com/'
# none of these buckets actually exist
CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
CONTACT_LIST_UPLOAD_BUCKET_NAME = 'test-contact-list'
LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test'
# MOU_BUCKET_NAME = 'test-mou'
# TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
# PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup'
API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to'
REDIS_URL = 'redis://you-forgot-to-mock-a-redis-call-to'
ANTIVIRUS_API_HOST = 'https://test-antivirus'
@@ -149,21 +139,12 @@ class Production(Config):
DEBUG = False
# buckets
CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME')
CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY')
CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY')
CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION')
CONTACT_LIST_UPLOAD_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME')
CONTACT_LIST_UPLOAD_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY')
CONTACT_LIST_UPLOAD_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY')
CONTACT_LIST_UPLOAD_REGION = os.environ.get('CONTACT_LIST_REGION')
LOGO_UPLOAD_BUCKET_NAME = os.environ.get('LOGO_UPLOAD_BUCKET_NAME')
LOGO_UPLOAD_ACCESS_KEY = os.environ.get('LOGO_UPLOAD_ACCESS_KEY')
LOGO_UPLOAD_SECRET_KEY = os.environ.get('LOGO_UPLOAD_SECRET_KEY')
LOGO_UPLOAD_REGION = os.environ.get('LOGO_UPLOAD_REGION')
# MOU_BUCKET_NAME = os.environ.get('MOU_UPLOAD_BUCKET_NAME')
# TRANSIENT_UPLOADED_LETTERS = 'prototype-transient-uploaded-letters' # not created in gsa sandbox
# PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'prototype-letters-precompiled-originals-backup' # not in sandbox
CSV_UPLOAD_BUCKET = cloud_config.s3_credentials(
f"notifications-api-csv-upload-bucket-{os.environ['NOTIFY_ENVIRONMENT']}")
CONTACT_LIST_BUCKET = cloud_config.s3_credentials(
f"notifications-api-contact-list-bucket-{os.environ['NOTIFY_ENVIRONMENT']}")
LOGO_UPLOAD_BUCKET = cloud_config.s3_credentials(
f"notifications-admin-logo-upload-bucket-{os.environ['NOTIFY_ENVIRONMENT']}")
class Staging(Production):

View File

@@ -44,21 +44,25 @@ class ContactList(JSONModel):
contact_list_id=contact_list_id,
))
@staticmethod
def get_bucket_credentials(key):
return current_app.config['CONTACT_LIST_BUCKET'][key]
@staticmethod
def get_bucket_name():
return current_app.config['CONTACT_LIST_UPLOAD_BUCKET_NAME']
return ContactList.get_bucket_credentials('bucket')
@staticmethod
def get_access_key():
return current_app.config['CONTACT_LIST_UPLOAD_ACCESS_KEY']
return ContactList.get_bucket_credentials('access_key_id')
@staticmethod
def get_secret_key():
return current_app.config['CONTACT_LIST_UPLOAD_SECRET_KEY']
return ContactList.get_bucket_credentials('secret_access_key')
@staticmethod
def get_region():
return current_app.config['CONTACT_LIST_UPLOAD_REGION']
return ContactList.get_bucket_credentials('region')
@staticmethod
def get_filename(service_id, upload_id):

View File

@@ -15,11 +15,11 @@ FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
def get_csv_location(service_id, upload_id):
return (
current_app.config['CSV_UPLOAD_BUCKET_NAME'],
current_app.config['CSV_UPLOAD_BUCKET']['bucket'],
FILE_LOCATION_STRUCTURE.format(service_id, upload_id),
current_app.config['CSV_UPLOAD_ACCESS_KEY'],
current_app.config['CSV_UPLOAD_SECRET_KEY'],
current_app.config['CSV_UPLOAD_REGION'],
current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'],
current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'],
current_app.config['CSV_UPLOAD_BUCKET']['region'],
)

View File

@@ -15,14 +15,18 @@ LETTER_TEMP_LOGO_LOCATION = 'letters/static/images/letter-template/temp-{user_id
def get_logo_location(filename=None):
return (
current_app.config['LOGO_UPLOAD_BUCKET_NAME'],
bucket_creds('bucket'),
filename,
current_app.config['LOGO_UPLOAD_ACCESS_KEY'],
current_app.config['LOGO_UPLOAD_SECRET_KEY'],
current_app.config['LOGO_UPLOAD_REGION'],
bucket_creds('access_key_id'),
bucket_creds('secret_access_key'),
bucket_creds('region'),
)
def bucket_creds(key):
return current_app.config['LOGO_UPLOAD_BUCKET'][key]
def delete_s3_object(filename):
get_s3_object(*get_logo_location(filename)).delete()
@@ -37,10 +41,10 @@ def persist_logo(old_name, new_name):
def get_s3_objects_filter_by_prefix(prefix):
bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME']
session = Session(aws_access_key_id=current_app.config['LOGO_UPLOAD_ACCESS_KEY'],
aws_secret_access_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'],
region_name=current_app.config['LOGO_UPLOAD_REGION'])
bucket_name = bucket_creds('bucket')
session = Session(aws_access_key_id=bucket_creds('access_key_id'),
aws_secret_access_key=bucket_creds('secret_access_key'),
region_name=bucket_creds('region'))
s3 = session.resource('s3')
return s3.Bucket(bucket_name).objects.filter(Prefix=prefix)
@@ -59,15 +63,15 @@ def upload_email_logo(filename, filedata, user_id):
unique_id=str(uuid.uuid4()),
filename=filename
)
bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME']
bucket_name = bucket_creds('bucket')
utils_s3upload(
filedata=filedata,
region=current_app.config['LOGO_UPLOAD_REGION'],
region=bucket_creds('region'),
bucket_name=bucket_name,
file_location=upload_file_name,
content_type='image/png',
access_key=current_app.config['LOGO_UPLOAD_ACCESS_KEY'],
secret_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'],
access_key=bucket_creds('access_key_id'),
secret_key=bucket_creds('secret_access_key'),
)
return upload_file_name
@@ -79,15 +83,15 @@ def upload_letter_temp_logo(filename, filedata, user_id):
unique_id=str(uuid.uuid4()),
filename=filename
)
bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME']
bucket_name = bucket_creds('bucket')
utils_s3upload(
filedata=filedata,
region=current_app.config['LOGO_UPLOAD_REGION'],
region=bucket_creds('region'),
bucket_name=bucket_name,
file_location=upload_filename,
content_type='image/svg+xml',
access_key=current_app.config['LOGO_UPLOAD_ACCESS_KEY'],
secret_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'],
access_key=bucket_creds('access_key_id'),
secret_key=bucket_creds('secret_access_key'),
)
return upload_filename

View File

@@ -208,7 +208,7 @@ def test_upload_csv_file_shows_error_banner(
mock_upload.assert_called_once_with(
filedata='',
region='us-west-2',
bucket_name='test-contact-list',
bucket_name='local-contact-list',
file_location=f"service-{SERVICE_ONE_ID}-notify/{fake_uuid}.csv",
access_key=default_access_key,
secret_key=default_secret_key,

View File

@@ -4,7 +4,7 @@ from app.s3_client.s3_csv_client import set_metadata_on_csv_upload
def test_sets_metadata(client_request, mocker):
mocked_s3_object = Mock(bucket_name='test-notifications-csv-upload', key='service-1234-notify/5678.csv')
mocked_s3_object = Mock(bucket_name='local-notifications-csv-upload', key='service-1234-notify/5678.csv')
mocked_get_s3_object = mocker.patch(
'app.s3_client.s3_csv_client.get_csv_upload',
return_value=mocked_s3_object,
@@ -14,7 +14,7 @@ def test_sets_metadata(client_request, mocker):
mocked_get_s3_object.assert_called_once_with('1234', '5678')
mocked_s3_object.copy_from.assert_called_once_with(
CopySource='test-notifications-csv-upload/service-1234-notify/5678.csv',
CopySource='local-notifications-csv-upload/service-1234-notify/5678.csv',
Metadata={'baz': 'True', 'foo': 'bar'},
MetadataDirective='REPLACE',
ServerSideEncryption='AES256',

View File

@@ -21,6 +21,12 @@ from app.s3_client.s3_logo_client import (
)
bucket = 'test_bucket'
bucket_credentials = {
'bucket': bucket,
'access_key_id': default_access_key,
'secret_access_key': default_secret_key,
'region': default_region
}
data = {'data': 'some_data'}
filename = 'test.png'
svg_filename = 'test.svg'
@@ -45,7 +51,7 @@ def letter_upload_filename(fake_uuid):
def test_upload_email_logo_calls_correct_args(client_request, mocker, fake_uuid, upload_filename):
mocker.patch('uuid.uuid4', return_value=upload_id)
mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET_NAME': bucket})
mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET': bucket_credentials})
mocked_s3_upload = mocker.patch('app.s3_client.s3_logo_client.utils_s3upload')
upload_email_logo(filename=filename, user_id=fake_uuid, filedata=data)
@@ -63,7 +69,7 @@ def test_upload_email_logo_calls_correct_args(client_request, mocker, fake_uuid,
def test_upload_letter_temp_logo_calls_correct_args(mocker, fake_uuid, letter_upload_filename):
mocker.patch('uuid.uuid4', return_value=upload_id)
mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET_NAME': bucket})
mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET': bucket_credentials})
mocked_s3_upload = mocker.patch('app.s3_client.s3_logo_client.utils_s3upload')
new_filename = upload_letter_temp_logo(filename=svg_filename, user_id=fake_uuid, filedata=data)
@@ -81,7 +87,7 @@ def test_upload_letter_temp_logo_calls_correct_args(mocker, fake_uuid, letter_up
def test_persist_logo(client_request, mocker, fake_uuid, upload_filename):
mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET_NAME': bucket})
mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET': bucket_credentials})
mocked_get_s3_object = mocker.patch('app.s3_client.s3_logo_client.get_s3_object')
mocked_delete_s3_object = mocker.patch('app.s3_client.s3_logo_client.delete_s3_object')

View File

@@ -3,7 +3,14 @@ import os
import pytest
from app.cloudfoundry_config import extract_cloudfoundry_config
from app.cloudfoundry_config import CloudfoundryConfig
bucket_credentials = {
'access_key_id': 'contact-list-access',
'bucket': 'contact-list-bucket',
'region': 'us-gov-west-1',
'secret_access_key': 'contact-list-secret'
}
@pytest.fixture
@@ -26,22 +33,39 @@ def vcap_services():
},
{
'name': 'notifications-api-contact-list-bucket-test',
'credentials': {
'access_key_id': 'contact-list-access',
'bucket': 'contact-list-bucket',
'region': 'us-gov-west-1',
'secret_access_key': 'contact-list-secret'
}
'credentials': bucket_credentials
}
],
}
def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services):
os.environ['DEPLOY_ENV'] = 'test'
def test_redis_url(vcap_services):
os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
extract_cloudfoundry_config()
assert os.environ['REDIS_URL'] == 'rediss://xxx:6379'
assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket'
assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket'
assert CloudfoundryConfig().redis_url == 'rediss://xxx:6379'
def test_redis_url_falls_back_to_REDIS_URL():
expected = 'rediss://yyy:6379'
os.environ['REDIS_URL'] = expected
os.environ['VCAP_SERVICES'] = ""
assert CloudfoundryConfig().redis_url == expected
def test_s3_bucket_credentials(vcap_services):
os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
assert CloudfoundryConfig().s3_credentials('notifications-api-contact-list-bucket-test') == bucket_credentials
def test_s3_bucket_credentials_falls_back_to_empty_creds():
os.environ['VCAP_SERVICES'] = ""
expected = {
'bucket': '',
'access_key_id': '',
'secret_access_key': '',
'region': ''
}
assert CloudfoundryConfig().s3_credentials('bucket') == expected

View File

@@ -1,53 +0,0 @@
import importlib
import os
from unittest import mock
import pytest
from app import config
def cf_conf():
os.environ['REDIS_URL'] = 'rediss://xxx:6379'
@pytest.fixture
def reload_config(os_environ):
"""
Reset config, by simply re-running config.py from a fresh environment
"""
old_env = os.environ.copy()
os.environ.clear()
yield
os.environ.clear()
for k, v in old_env.items():
os.environ[k] = v
importlib.reload(config)
def test_load_cloudfoundry_config_if_available(reload_config):
os.environ['REDIS_URL'] = 'some uri'
os.environ['VCAP_SERVICES'] = 'some json blob'
with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf):
# reload config so that its module level code (ie: all of it) is re-instantiated
importlib.reload(config)
assert os.environ['REDIS_URL'] == 'rediss://xxx:6379'
assert config.Config.REDIS_URL == 'rediss://xxx:6379'
def test_load_config_if_cloudfoundry_not_available(reload_config):
os.environ['REDIS_URL'] = 'redis://xxx:6379'
os.environ.pop('VCAP_SERVICES', None)
with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config:
# reload config so that its module level code (ie: all of it) is re-instantiated
importlib.reload(config)
assert not cf_config.called
assert os.environ['REDIS_URL'] == 'redis://xxx:6379'
assert config.Config.REDIS_URL == 'redis://xxx:6379'