Clean up bucket settings

2026-01-30 06:21:50 -05:00 · 2022-10-31 15:37:12 -04:00
parent 7aafdd7bac
commit 41a52daca0
6 changed files with 93 additions and 149 deletions
--- a/app/aws/s3.py
+++ b/app/aws/s3.py
@@ -41,21 +41,21 @@ def file_exists(

 def get_job_location(service_id, job_id):
    return (
-        current_app.config['CSV_UPLOAD_BUCKET_NAME'],
+        current_app.config['CSV_UPLOAD_BUCKET']['bucket'],
        FILE_LOCATION_STRUCTURE.format(service_id, job_id),
-        current_app.config['CSV_UPLOAD_ACCESS_KEY'],
-        current_app.config['CSV_UPLOAD_SECRET_KEY'],
-        current_app.config['CSV_UPLOAD_REGION'],
+        current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'],
+        current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'],
+        current_app.config['CSV_UPLOAD_BUCKET']['region'],
    )


 def get_contact_list_location(service_id, contact_list_id):
    return (
-        current_app.config['CONTACT_LIST_BUCKET_NAME'],
+        current_app.config['CONTACT_LIST_BUCKET']['bucket'],
        FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id),
-        current_app.config['CONTACT_LIST_ACCESS_KEY'],
-        current_app.config['CONTACT_LIST_SECRET_KEY'],
-        current_app.config['CONTACT_LIST_REGION'],
+        current_app.config['CONTACT_LIST_BUCKET']['access_key_id'],
+        current_app.config['CONTACT_LIST_BUCKET']['secret_access_key'],
+        current_app.config['CONTACT_LIST_BUCKET']['region'],
    )


--- a/app/cloudfoundry_config.py
+++ b/app/cloudfoundry_config.py
@@ -2,41 +2,30 @@ import json
 import os


-def find_by_service_name(services, service_name):
-    for i in range(len(services)):
-        if services[i]['name'] == service_name:
-            return services[i]
-    return None
+class CloudfoundryConfig:
+    def __init__(self):
+        self.parsed_services = json.loads(os.environ.get('VCAP_SERVICES') or '{}')
+        buckets = self.parsed_services.get('s3') or []
+        self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets}
+        self._empty_bucket_credentials = {
+            'bucket': '',
+            'access_key_id': '',
+            'secret_access_key': '',
+            'region': ''
+        }
+
+    @property
+    def redis_url(self):
+        try:
+            return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace(
+                'redis://',
+                'rediss://'
+            )
+        except KeyError:
+            return os.environ.get('REDIS_URL')
+
+    def s3_credentials(self, service_name):
+        return self.s3_buckets.get(service_name) or self._empty_bucket_credentials


-def extract_cloudfoundry_config():
-    vcap_services = json.loads(os.environ['VCAP_SERVICES'])
-
-    # Postgres config
-    os.environ['SQLALCHEMY_DATABASE_URI'] = \
-        vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres', 'postgresql')
-    # Redis config
-    os.environ['REDIS_URL'] = \
-        vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://', 'rediss://')
-
-    # CSV Upload Bucket Name
-    bucket_service = find_by_service_name(
-        vcap_services['s3'],
-        f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}"
-    )
-    if bucket_service:
-        os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket']
-        os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
-        os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
-        os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region']
-
-    # Contact List Bucket Name
-    bucket_service = find_by_service_name(
-        vcap_services['s3'],
-        f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}"
-    )
-    if bucket_service:
-        os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket']
-        os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
-        os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
-        os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region']
+cloud_config = CloudfoundryConfig()
--- a/app/config.py
+++ b/app/config.py
@@ -5,12 +5,7 @@ from datetime import timedelta
 from celery.schedules import crontab
 from kombu import Exchange, Queue

-if os.environ.get('VCAP_SERVICES'):
-    # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
-    # standard environment variables from it
-    from app.cloudfoundry_config import extract_cloudfoundry_config
-
-    extract_cloudfoundry_config()
+from app.cloudfoundry_config import cloud_config


 class QueueNames(object):
@@ -77,7 +72,8 @@ class Config(object):

    # Credentials
    # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
-    ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID')
+    # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
+    ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID', 'notify-admin')
    INTERNAL_CLIENT_API_KEYS = json.loads(
        os.environ.get(
            'INTERNAL_CLIENT_API_KEYS',
@@ -100,7 +96,7 @@ class Config(object):
    SQLALCHEMY_STATEMENT_TIMEOUT = 1200
    PAGE_SIZE = 50
    API_PAGE_SIZE = 250
-    REDIS_URL = os.environ.get('REDIS_URL')
+    REDIS_URL = cloud_config.redis_url
    REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '0') == '1'
    EXPIRE_CACHE_TEN_MINUTES = 600
    EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
@@ -350,27 +346,28 @@ class Config(object):
    DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token')


+def _default_s3_credentials(bucket_name):
+    return {
+        'bucket': bucket_name,
+        'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'),
+        'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'),
+        'region': os.environ.get('AWS_REGION')
+    }
+
+
 class Development(Config):
    DEBUG = True
    SQLALCHEMY_ECHO = False
    DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com']

    # Buckets
-    CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
-    CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
-    CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
-    CSV_UPLOAD_REGION = os.environ.get('AWS_REGION', 'us-west-2')
-    CONTACT_LIST_BUCKET_NAME = 'local-contact-list'
-    CONTACT_LIST_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
-    CONTACT_LIST_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
-    CONTACT_LIST_REGION = os.environ.get('AWS_REGION', 'us-west-2')
+    CSV_UPLOAD_BUCKET = _default_s3_credentials('local-notifications-csv-upload')
+    CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list')

    # credential overrides
    DANGEROUS_SALT = 'dev-notify-salt'
    SECRET_KEY = 'dev-notify-secret-key'  # nosec B105 - this is only used in development
-    # ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
-    ADMIN_CLIENT_ID = 'notify-admin'
-    INTERNAL_CLIENT_API_KEYS = {ADMIN_CLIENT_ID: ['dev-notify-secret-key']}
+    INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']}


 class Test(Development):
@@ -390,8 +387,8 @@ class Test(Development):
        '10d1b9c9-0072-4fa9-ae1c-595e333841da',
    ]

-    CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
-    CONTACT_LIST_BUCKET_NAME = 'test-contact-list'
+    CSV_UPLOAD_BUCKET = _default_s3_credentials('test-notifications-csv-upload')
+    CONTACT_LIST_BUCKET = _default_s3_credentials('test-contact-list')

    # this is overriden in CI
    SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI')
@@ -406,14 +403,10 @@ class Test(Development):

 class Production(Config):
    # buckets
-    CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME')
-    CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY')
-    CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY')
-    CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION')
-    CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME')
-    CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY')
-    CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY')
-    CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION')
+    CSV_UPLOAD_BUCKET = cloud_config.s3_credentials(
+        f"notifications-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}")
+    CONTACT_LIST_BUCKET = cloud_config.s3_credentials(
+        f"notifications-api-contact-list-bucket-{Config.NOTIFY_ENVIRONMENT}")

    FROM_NUMBER = 'US Notify'
    CRONITOR_ENABLED = True