From 41a52daca0059328973402e3ca2ddadfacc0042f Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 31 Oct 2022 15:37:12 -0400 Subject: [PATCH] Clean up bucket settings --- app/aws/s3.py | 16 +++---- app/cloudfoundry_config.py | 61 +++++++++++---------------- app/config.py | 51 ++++++++++------------ manifest.yml | 7 +-- tests/app/test_cloudfoundry_config.py | 51 ++++++++++++++++------ tests/app/test_config.py | 56 ------------------------ 6 files changed, 93 insertions(+), 149 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index 7312f4cbe..7a705c544 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -41,21 +41,21 @@ def file_exists( def get_job_location(service_id, job_id): return ( - current_app.config['CSV_UPLOAD_BUCKET_NAME'], + current_app.config['CSV_UPLOAD_BUCKET']['bucket'], FILE_LOCATION_STRUCTURE.format(service_id, job_id), - current_app.config['CSV_UPLOAD_ACCESS_KEY'], - current_app.config['CSV_UPLOAD_SECRET_KEY'], - current_app.config['CSV_UPLOAD_REGION'], + current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'], + current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'], + current_app.config['CSV_UPLOAD_BUCKET']['region'], ) def get_contact_list_location(service_id, contact_list_id): return ( - current_app.config['CONTACT_LIST_BUCKET_NAME'], + current_app.config['CONTACT_LIST_BUCKET']['bucket'], FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id), - current_app.config['CONTACT_LIST_ACCESS_KEY'], - current_app.config['CONTACT_LIST_SECRET_KEY'], - current_app.config['CONTACT_LIST_REGION'], + current_app.config['CONTACT_LIST_BUCKET']['access_key_id'], + current_app.config['CONTACT_LIST_BUCKET']['secret_access_key'], + current_app.config['CONTACT_LIST_BUCKET']['region'], ) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 533f3d6af..1218e8852 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -2,41 +2,30 @@ import json import os -def find_by_service_name(services, service_name): - for i in range(len(services)): - if services[i]['name'] == service_name: - return services[i] - return None +class CloudfoundryConfig: + def __init__(self): + self.parsed_services = json.loads(os.environ.get('VCAP_SERVICES') or '{}') + buckets = self.parsed_services.get('s3') or [] + self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets} + self._empty_bucket_credentials = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + @property + def redis_url(self): + try: + return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace( + 'redis://', + 'rediss://' + ) + except KeyError: + return os.environ.get('REDIS_URL') + + def s3_credentials(self, service_name): + return self.s3_buckets.get(service_name) or self._empty_bucket_credentials -def extract_cloudfoundry_config(): - vcap_services = json.loads(os.environ['VCAP_SERVICES']) - - # Postgres config - os.environ['SQLALCHEMY_DATABASE_URI'] = \ - vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres', 'postgresql') - # Redis config - os.environ['REDIS_URL'] = \ - vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://', 'rediss://') - - # CSV Upload Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], - f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}" - ) - if bucket_service: - os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region'] - - # Contact List Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], - f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}" - ) - if bucket_service: - os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region'] +cloud_config = CloudfoundryConfig() diff --git a/app/config.py b/app/config.py index 86f0be77b..723143c4f 100644 --- a/app/config.py +++ b/app/config.py @@ -5,12 +5,7 @@ from datetime import timedelta from celery.schedules import crontab from kombu import Exchange, Queue -if os.environ.get('VCAP_SERVICES'): - # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate - # standard environment variables from it - from app.cloudfoundry_config import extract_cloudfoundry_config - - extract_cloudfoundry_config() +from app.cloudfoundry_config import cloud_config class QueueNames(object): @@ -77,7 +72,8 @@ class Config(object): # Credentials # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API - ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID') + # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match + ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID', 'notify-admin') INTERNAL_CLIENT_API_KEYS = json.loads( os.environ.get( 'INTERNAL_CLIENT_API_KEYS', @@ -100,7 +96,7 @@ class Config(object): SQLALCHEMY_STATEMENT_TIMEOUT = 1200 PAGE_SIZE = 50 API_PAGE_SIZE = 250 - REDIS_URL = os.environ.get('REDIS_URL') + REDIS_URL = cloud_config.redis_url REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '0') == '1' EXPIRE_CACHE_TEN_MINUTES = 600 EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60 @@ -350,27 +346,28 @@ class Config(object): DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token') +def _default_s3_credentials(bucket_name): + return { + 'bucket': bucket_name, + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } + + class Development(Config): DEBUG = True SQLALCHEMY_ECHO = False DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] # Buckets - CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' - CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CSV_UPLOAD_REGION = os.environ.get('AWS_REGION', 'us-west-2') - CONTACT_LIST_BUCKET_NAME = 'local-contact-list' - CONTACT_LIST_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CONTACT_LIST_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CONTACT_LIST_REGION = os.environ.get('AWS_REGION', 'us-west-2') + CSV_UPLOAD_BUCKET = _default_s3_credentials('local-notifications-csv-upload') + CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list') # credential overrides DANGEROUS_SALT = 'dev-notify-salt' SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development - # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match - ADMIN_CLIENT_ID = 'notify-admin' - INTERNAL_CLIENT_API_KEYS = {ADMIN_CLIENT_ID: ['dev-notify-secret-key']} + INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']} class Test(Development): @@ -390,8 +387,8 @@ class Test(Development): '10d1b9c9-0072-4fa9-ae1c-595e333841da', ] - CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' - CONTACT_LIST_BUCKET_NAME = 'test-contact-list' + CSV_UPLOAD_BUCKET = _default_s3_credentials('test-notifications-csv-upload') + CONTACT_LIST_BUCKET = _default_s3_credentials('test-contact-list') # this is overriden in CI SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI') @@ -406,14 +403,10 @@ class Test(Development): class Production(Config): # buckets - CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME') - CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY') - CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY') - CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION') - CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME') - CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY') - CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY') - CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION') + CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( + f"notifications-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}") + CONTACT_LIST_BUCKET = cloud_config.s3_credentials( + f"notifications-api-contact-list-bucket-{Config.NOTIFY_ENVIRONMENT}") FROM_NUMBER = 'US Notify' CRONITOR_ENABLED = True diff --git a/manifest.yml b/manifest.yml index c07573fc6..77ee86a1c 100644 --- a/manifest.yml +++ b/manifest.yml @@ -32,16 +32,13 @@ applications: NOTIFY_LOG_PATH: /home/vcap/logs/app.log FLASK_APP: application.py FLASK_ENV: production - DEPLOY_ENV: ((env)) - NOTIFY_ENVIRONMENT: live + NOTIFY_ENVIRONMENT: ((env)) API_HOST_NAME: https://notifications-api.app.cloud.gov ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov - STATSD_HOST: localhost # Credentials variables INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}' - ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET)) DANGEROUS_SALT: ((DANGEROUS_SALT)) SECRET_KEY: ((SECRET_KEY)) AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID)) @@ -49,5 +46,3 @@ applications: AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 - - DVLA_EMAIL_ADDRESSES: [] diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index 1d5db39d7..80fad2648 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -3,7 +3,14 @@ import os import pytest -from app.cloudfoundry_config import extract_cloudfoundry_config +from app.cloudfoundry_config import CloudfoundryConfig + +bucket_credentials = { + 'access_key_id': 'csv-access', + 'bucket': 'csv-upload-bucket', + 'region': 'us-gov-west-1', + 'secret_access_key': 'csv-secret' +} @pytest.fixture @@ -22,12 +29,7 @@ def vcap_services(): 's3': [ { 'name': 'notifications-api-csv-upload-bucket-test', - 'credentials': { - 'access_key_id': 'csv-access', - 'bucket': 'csv-upload-bucket', - 'region': 'us-gov-west-1', - 'secret_access_key': 'csv-secret' - } + 'credentials': bucket_credentials }, { 'name': 'notifications-api-contact-list-bucket-test', @@ -43,12 +45,33 @@ def vcap_services(): } -def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services): - os.environ['DEPLOY_ENV'] = 'test' +def test_redis_url(vcap_services): os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) - extract_cloudfoundry_config() - assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri' - assert os.environ['REDIS_URL'] == 'rediss://xxx:6379' - assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket' - assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket' + assert CloudfoundryConfig().redis_url == 'rediss://xxx:6379' + + +def test_redis_url_falls_back_to_REDIS_URL(): + expected = 'redis://yyy:6379' + os.environ['REDIS_URL'] = expected + os.environ['VCAP_SERVICES'] = "" + + assert CloudfoundryConfig().redis_url == expected + + +def test_s3_bucket_credentials(vcap_services): + os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) + + assert CloudfoundryConfig().s3_credentials('notifications-api-csv-upload-bucket-test') == bucket_credentials + + +def test_s3_bucket_credentials_falls_back_to_empty_creds(): + os.environ['VCAP_SERVICES'] = "" + expected = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + assert CloudfoundryConfig().s3_credentials('bucket') == expected diff --git a/tests/app/test_config.py b/tests/app/test_config.py index 17bb96bb6..1f77278d1 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -1,62 +1,6 @@ -import importlib -import os -from unittest import mock - -import pytest - -from app import config from app.config import QueueNames -def cf_conf(): - os.environ['ADMIN_BASE_URL'] = 'cf' - - -@pytest.fixture -def reload_config(): - """ - Reset config, by simply re-running config.py from a fresh environment - """ - old_env = os.environ.copy() - - yield - - os.environ.clear() - for k, v in old_env.items(): - os.environ[k] = v - - importlib.reload(config) - - -def test_load_cloudfoundry_config_if_available(reload_config): - os.environ['ADMIN_BASE_URL'] = 'env' - os.environ['VCAP_SERVICES'] = 'some json blob' - os.environ['VCAP_APPLICATION'] = 'some json blob' - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config: - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert cf_config.called - - assert os.environ['ADMIN_BASE_URL'] == 'cf' - assert config.Config.ADMIN_BASE_URL == 'cf' - - -def test_load_config_if_cloudfoundry_not_available(reload_config): - os.environ['ADMIN_BASE_URL'] = 'env' - os.environ.pop('VCAP_SERVICES', None) - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config: - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert not cf_config.called - - assert os.environ['ADMIN_BASE_URL'] == 'env' - assert config.Config.ADMIN_BASE_URL == 'env' - - def test_queue_names_all_queues_correct(): # Need to ensure that all_queues() only returns queue names used in API queues = QueueNames.all_queues()