Clean up bucket settings

app/aws/s3.py

@@ -41,21 +41,21 @@ def file_exists(
 
 def get_job_location(service_id, job_id):
     return (
-        current_app.config['CSV_UPLOAD_BUCKET_NAME'],
+        current_app.config['CSV_UPLOAD_BUCKET']['bucket'],
         FILE_LOCATION_STRUCTURE.format(service_id, job_id),
-        current_app.config['CSV_UPLOAD_ACCESS_KEY'],
+        current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'],
-        current_app.config['CSV_UPLOAD_SECRET_KEY'],
+        current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'],
-        current_app.config['CSV_UPLOAD_REGION'],
+        current_app.config['CSV_UPLOAD_BUCKET']['region'],
     )
 
 
 def get_contact_list_location(service_id, contact_list_id):
     return (
-        current_app.config['CONTACT_LIST_BUCKET_NAME'],
+        current_app.config['CONTACT_LIST_BUCKET']['bucket'],
         FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id),
-        current_app.config['CONTACT_LIST_ACCESS_KEY'],
+        current_app.config['CONTACT_LIST_BUCKET']['access_key_id'],
-        current_app.config['CONTACT_LIST_SECRET_KEY'],
+        current_app.config['CONTACT_LIST_BUCKET']['secret_access_key'],
-        current_app.config['CONTACT_LIST_REGION'],
+        current_app.config['CONTACT_LIST_BUCKET']['region'],
     )
 
 

app/cloudfoundry_config.py

@@ -2,41 +2,30 @@ import json
 import os
 
 
-def find_by_service_name(services, service_name):
+class CloudfoundryConfig:
-    for i in range(len(services)):
+    def __init__(self):
-        if services[i]['name'] == service_name:
+        self.parsed_services = json.loads(os.environ.get('VCAP_SERVICES') or '{}')
-            return services[i]
+        buckets = self.parsed_services.get('s3') or []
-    return None
+        self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets}
+        self._empty_bucket_credentials = {
+            'bucket': '',
+            'access_key_id': '',
+            'secret_access_key': '',
+            'region': ''
+        }
 
-
+    @property
-def extract_cloudfoundry_config():
+    def redis_url(self):
-    vcap_services = json.loads(os.environ['VCAP_SERVICES'])
+        try:
-
+            return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace(
-    # Postgres config
+                'redis://',
-    os.environ['SQLALCHEMY_DATABASE_URI'] = \
+                'rediss://'
-        vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres', 'postgresql')
-    # Redis config
-    os.environ['REDIS_URL'] = \
-        vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://', 'rediss://')
-
-    # CSV Upload Bucket Name
-    bucket_service = find_by_service_name(
-        vcap_services['s3'],
-        f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}"
             )
-    if bucket_service:
+        except KeyError:
-        os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket']
+            return os.environ.get('REDIS_URL')
-        os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
-        os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
-        os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region']
 
-    # Contact List Bucket Name
+    def s3_credentials(self, service_name):
-    bucket_service = find_by_service_name(
+        return self.s3_buckets.get(service_name) or self._empty_bucket_credentials
-        vcap_services['s3'],
+
-        f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}"
+
-    )
+cloud_config = CloudfoundryConfig()
-    if bucket_service:
-        os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket']
-        os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
-        os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
-        os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region']

app/config.py

@@ -5,12 +5,7 @@ from datetime import timedelta
 from celery.schedules import crontab
 from kombu import Exchange, Queue
 
-if os.environ.get('VCAP_SERVICES'):
+from app.cloudfoundry_config import cloud_config
-    # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
-    # standard environment variables from it
-    from app.cloudfoundry_config import extract_cloudfoundry_config
-
-    extract_cloudfoundry_config()
 
 
 class QueueNames(object):
@@ -77,7 +72,8 @@ class Config(object):
 
     # Credentials
     # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
-    ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID')
+    # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
+    ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID', 'notify-admin')
     INTERNAL_CLIENT_API_KEYS = json.loads(
         os.environ.get(
             'INTERNAL_CLIENT_API_KEYS',
@@ -100,7 +96,7 @@ class Config(object):
     SQLALCHEMY_STATEMENT_TIMEOUT = 1200
     PAGE_SIZE = 50
     API_PAGE_SIZE = 250
-    REDIS_URL = os.environ.get('REDIS_URL')
+    REDIS_URL = cloud_config.redis_url
     REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '0') == '1'
     EXPIRE_CACHE_TEN_MINUTES = 600
     EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
@@ -350,27 +346,28 @@ class Config(object):
     DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token')
 
 
+def _default_s3_credentials(bucket_name):
+    return {
+        'bucket': bucket_name,
+        'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'),
+        'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'),
+        'region': os.environ.get('AWS_REGION')
+    }
+
+
 class Development(Config):
     DEBUG = True
     SQLALCHEMY_ECHO = False
     DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com']
 
     # Buckets
-    CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
+    CSV_UPLOAD_BUCKET = _default_s3_credentials('local-notifications-csv-upload')
-    CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
+    CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list')
-    CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
-    CSV_UPLOAD_REGION = os.environ.get('AWS_REGION', 'us-west-2')
-    CONTACT_LIST_BUCKET_NAME = 'local-contact-list'
-    CONTACT_LIST_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
-    CONTACT_LIST_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
-    CONTACT_LIST_REGION = os.environ.get('AWS_REGION', 'us-west-2')
 
     # credential overrides
     DANGEROUS_SALT = 'dev-notify-salt'
     SECRET_KEY = 'dev-notify-secret-key'  # nosec B105 - this is only used in development
-    # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
+    INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']}
-    ADMIN_CLIENT_ID = 'notify-admin'
-    INTERNAL_CLIENT_API_KEYS = {ADMIN_CLIENT_ID: ['dev-notify-secret-key']}
 
 
 class Test(Development):
@@ -390,8 +387,8 @@ class Test(Development):
         '10d1b9c9-0072-4fa9-ae1c-595e333841da',
     ]
 
-    CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
+    CSV_UPLOAD_BUCKET = _default_s3_credentials('test-notifications-csv-upload')
-    CONTACT_LIST_BUCKET_NAME = 'test-contact-list'
+    CONTACT_LIST_BUCKET = _default_s3_credentials('test-contact-list')
 
     # this is overriden in CI
     SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI')
@@ -406,14 +403,10 @@ class Test(Development):
 
 class Production(Config):
     # buckets
-    CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME')
+    CSV_UPLOAD_BUCKET = cloud_config.s3_credentials(
-    CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY')
+        f"notifications-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}")
-    CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY')
+    CONTACT_LIST_BUCKET = cloud_config.s3_credentials(
-    CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION')
+        f"notifications-api-contact-list-bucket-{Config.NOTIFY_ENVIRONMENT}")
-    CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME')
-    CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY')
-    CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY')
-    CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION')
 
     FROM_NUMBER = 'US Notify'
     CRONITOR_ENABLED = True

manifest.yml

@@ -32,16 +32,13 @@ applications:
       NOTIFY_LOG_PATH: /home/vcap/logs/app.log
       FLASK_APP: application.py
       FLASK_ENV: production
-      DEPLOY_ENV: ((env))
 
-      NOTIFY_ENVIRONMENT: live
+      NOTIFY_ENVIRONMENT: ((env))
       API_HOST_NAME: https://notifications-api.app.cloud.gov
       ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov
-      STATSD_HOST: localhost
 
       # Credentials variables
       INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}'
-      ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET))
       DANGEROUS_SALT: ((DANGEROUS_SALT))
       SECRET_KEY: ((SECRET_KEY))
       AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID))
@@ -49,5 +46,3 @@ applications:
       AWS_REGION: us-west-2
       AWS_PINPOINT_REGION: us-west-2
       AWS_US_TOLL_FREE_NUMBER: +18446120782
-
-      DVLA_EMAIL_ADDRESSES: []

tests/app/test_cloudfoundry_config.py

@@ -3,7 +3,14 @@ import os
 
 import pytest
 
-from app.cloudfoundry_config import extract_cloudfoundry_config
+from app.cloudfoundry_config import CloudfoundryConfig
+
+bucket_credentials = {
+    'access_key_id': 'csv-access',
+    'bucket': 'csv-upload-bucket',
+    'region': 'us-gov-west-1',
+    'secret_access_key': 'csv-secret'
+}
 
 
 @pytest.fixture
@@ -22,12 +29,7 @@ def vcap_services():
         's3': [
             {
                 'name': 'notifications-api-csv-upload-bucket-test',
-                'credentials': {
+                'credentials': bucket_credentials
-                    'access_key_id': 'csv-access',
-                    'bucket': 'csv-upload-bucket',
-                    'region': 'us-gov-west-1',
-                    'secret_access_key': 'csv-secret'
-                }
             },
             {
                 'name': 'notifications-api-contact-list-bucket-test',
@@ -43,12 +45,33 @@ def vcap_services():
     }
 
 
-def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services):
+def test_redis_url(vcap_services):
-    os.environ['DEPLOY_ENV'] = 'test'
     os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
-    extract_cloudfoundry_config()
 
-    assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri'
+    assert CloudfoundryConfig().redis_url == 'rediss://xxx:6379'
-    assert os.environ['REDIS_URL'] == 'rediss://xxx:6379'
+
-    assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket'
+
-    assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket'
+def test_redis_url_falls_back_to_REDIS_URL():
+    expected = 'redis://yyy:6379'
+    os.environ['REDIS_URL'] = expected
+    os.environ['VCAP_SERVICES'] = ""
+
+    assert CloudfoundryConfig().redis_url == expected
+
+
+def test_s3_bucket_credentials(vcap_services):
+    os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
+
+    assert CloudfoundryConfig().s3_credentials('notifications-api-csv-upload-bucket-test') == bucket_credentials
+
+
+def test_s3_bucket_credentials_falls_back_to_empty_creds():
+    os.environ['VCAP_SERVICES'] = ""
+    expected = {
+        'bucket': '',
+        'access_key_id': '',
+        'secret_access_key': '',
+        'region': ''
+    }
+
+    assert CloudfoundryConfig().s3_credentials('bucket') == expected

tests/app/test_config.py

@@ -1,62 +1,6 @@
-import importlib
-import os
-from unittest import mock
-
-import pytest
-
-from app import config
 from app.config import QueueNames
 
 
-def cf_conf():
-    os.environ['ADMIN_BASE_URL'] = 'cf'
-
-
-@pytest.fixture
-def reload_config():
-    """
-    Reset config, by simply re-running config.py from a fresh environment
-    """
-    old_env = os.environ.copy()
-
-    yield
-
-    os.environ.clear()
-    for k, v in old_env.items():
-        os.environ[k] = v
-
-    importlib.reload(config)
-
-
-def test_load_cloudfoundry_config_if_available(reload_config):
-    os.environ['ADMIN_BASE_URL'] = 'env'
-    os.environ['VCAP_SERVICES'] = 'some json blob'
-    os.environ['VCAP_APPLICATION'] = 'some json blob'
-
-    with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config:
-        # reload config so that its module level code (ie: all of it) is re-instantiated
-        importlib.reload(config)
-
-    assert cf_config.called
-
-    assert os.environ['ADMIN_BASE_URL'] == 'cf'
-    assert config.Config.ADMIN_BASE_URL == 'cf'
-
-
-def test_load_config_if_cloudfoundry_not_available(reload_config):
-    os.environ['ADMIN_BASE_URL'] = 'env'
-    os.environ.pop('VCAP_SERVICES', None)
-
-    with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config:
-        # reload config so that its module level code (ie: all of it) is re-instantiated
-        importlib.reload(config)
-
-    assert not cf_config.called
-
-    assert os.environ['ADMIN_BASE_URL'] == 'env'
-    assert config.Config.ADMIN_BASE_URL == 'env'
-
-
 def test_queue_names_all_queues_correct():
     # Need to ensure that all_queues() only returns queue names used in API
     queues = QueueNames.all_queues()