darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-20 15:31:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

Clean up bucket settings

Browse Source
This commit is contained in:
Ryan Ahearn
2022-10-31 15:37:12 -04:00
parent 7aafdd7bac
commit 41a52daca0
6 changed files with 93 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/aws/s3.py
View File

@@ -41,21 +41,21 @@ def file_exists(
def get_job_location(service_id, job_id):
return (
current_app.config['CSV_UPLOAD_BUCKET_NAME'],
current_app.config['CSV_UPLOAD_BUCKET']['bucket'],
FILE_LOCATION_STRUCTURE.format(service_id, job_id),
current_app.config['CSV_UPLOAD_ACCESS_KEY'],
current_app.config['CSV_UPLOAD_SECRET_KEY'],
current_app.config['CSV_UPLOAD_REGION'],
current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'],
current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'],
current_app.config['CSV_UPLOAD_BUCKET']['region'],
)
def get_contact_list_location(service_id, contact_list_id):
return (
current_app.config['CONTACT_LIST_BUCKET_NAME'],
current_app.config['CONTACT_LIST_BUCKET']['bucket'],
FILE_LOCATION_STRUCTURE.format(service_id, contact_list_id),
current_app.config['CONTACT_LIST_ACCESS_KEY'],
current_app.config['CONTACT_LIST_SECRET_KEY'],
current_app.config['CONTACT_LIST_REGION'],
current_app.config['CONTACT_LIST_BUCKET']['access_key_id'],
current_app.config['CONTACT_LIST_BUCKET']['secret_access_key'],
current_app.config['CONTACT_LIST_BUCKET']['region'],
)

61
app/cloudfoundry_config.py
View File

@@ -2,41 +2,30 @@ import json
import os
def find_by_service_name(services, service_name):
for i in range(len(services)):
if services[i]['name'] == service_name:
return services[i]
return None
class CloudfoundryConfig:
def __init__(self):
self.parsed_services = json.loads(os.environ.get('VCAP_SERVICES') or '{}')
buckets = self.parsed_services.get('s3') or []
self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets}
self._empty_bucket_credentials = {
'bucket': '',
'access_key_id': '',
'secret_access_key': '',
'region': ''
}
@property
def redis_url(self):
try:
return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace(
'redis://',
'rediss://'
)
except KeyError:
return os.environ.get('REDIS_URL')
def s3_credentials(self, service_name):
return self.s3_buckets.get(service_name) or self._empty_bucket_credentials
def extract_cloudfoundry_config():
vcap_services = json.loads(os.environ['VCAP_SERVICES'])
# Postgres config
os.environ['SQLALCHEMY_DATABASE_URI'] = \
vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres', 'postgresql')
# Redis config
os.environ['REDIS_URL'] = \
vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://', 'rediss://')
# CSV Upload Bucket Name
bucket_service = find_by_service_name(
vcap_services['s3'],
f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}"
)
if bucket_service:
os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket']
os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region']
# Contact List Bucket Name
bucket_service = find_by_service_name(
vcap_services['s3'],
f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}"
)
if bucket_service:
os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket']
os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id']
os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key']
os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region']
cloud_config = CloudfoundryConfig()

51
app/config.py
View File

@@ -5,12 +5,7 @@ from datetime import timedelta
from celery.schedules import crontab
from kombu import Exchange, Queue
if os.environ.get('VCAP_SERVICES'):
# on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
# standard environment variables from it
from app.cloudfoundry_config import extract_cloudfoundry_config
extract_cloudfoundry_config()
from app.cloudfoundry_config import cloud_config
class QueueNames(object):
@@ -77,7 +72,8 @@ class Config(object):
# Credentials
# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API
ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID')
# ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
ADMIN_CLIENT_ID = os.environ.get('ADMIN_CLIENT_ID', 'notify-admin')
INTERNAL_CLIENT_API_KEYS = json.loads(
os.environ.get(
'INTERNAL_CLIENT_API_KEYS',
@@ -100,7 +96,7 @@ class Config(object):
SQLALCHEMY_STATEMENT_TIMEOUT = 1200
PAGE_SIZE = 50
API_PAGE_SIZE = 250
REDIS_URL = os.environ.get('REDIS_URL')
REDIS_URL = cloud_config.redis_url
REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '0') == '1'
EXPIRE_CACHE_TEN_MINUTES = 600
EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
@@ -350,27 +346,28 @@ class Config(object):
DOCUMENT_DOWNLOAD_API_KEY = os.environ.get('DOCUMENT_DOWNLOAD_API_KEY', 'auth-token')
def _default_s3_credentials(bucket_name):
return {
'bucket': bucket_name,
'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'),
'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'),
'region': os.environ.get('AWS_REGION')
}
class Development(Config):
DEBUG = True
SQLALCHEMY_ECHO = False
DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com']
# Buckets
CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
CSV_UPLOAD_REGION = os.environ.get('AWS_REGION', 'us-west-2')
CONTACT_LIST_BUCKET_NAME = 'local-contact-list'
CONTACT_LIST_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID')
CONTACT_LIST_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
CONTACT_LIST_REGION = os.environ.get('AWS_REGION', 'us-west-2')
CSV_UPLOAD_BUCKET = _default_s3_credentials('local-notifications-csv-upload')
CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list')
# credential overrides
DANGEROUS_SALT = 'dev-notify-salt'
SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development
# ADMIN_CLIENT_ID is called ADMIN_CLIENT_USER_NAME in api repo, they should match
ADMIN_CLIENT_ID = 'notify-admin'
INTERNAL_CLIENT_API_KEYS = {ADMIN_CLIENT_ID: ['dev-notify-secret-key']}
INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']}
class Test(Development):
@@ -390,8 +387,8 @@ class Test(Development):
'10d1b9c9-0072-4fa9-ae1c-595e333841da',
]
CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
CONTACT_LIST_BUCKET_NAME = 'test-contact-list'
CSV_UPLOAD_BUCKET = _default_s3_credentials('test-notifications-csv-upload')
CONTACT_LIST_BUCKET = _default_s3_credentials('test-contact-list')
# this is overriden in CI
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI')
@@ -406,14 +403,10 @@ class Test(Development):
class Production(Config):
# buckets
CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME')
CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY')
CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY')
CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION')
CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME')
CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY')
CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY')
CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION')
CSV_UPLOAD_BUCKET = cloud_config.s3_credentials(
f"notifications-api-csv-upload-bucket-{Config.NOTIFY_ENVIRONMENT}")
CONTACT_LIST_BUCKET = cloud_config.s3_credentials(
f"notifications-api-contact-list-bucket-{Config.NOTIFY_ENVIRONMENT}")
FROM_NUMBER = 'US Notify'
CRONITOR_ENABLED = True

7
manifest.yml
View File

@@ -32,16 +32,13 @@ applications:
NOTIFY_LOG_PATH: /home/vcap/logs/app.log
FLASK_APP: application.py
FLASK_ENV: production
DEPLOY_ENV: ((env))
NOTIFY_ENVIRONMENT: live
NOTIFY_ENVIRONMENT: ((env))
API_HOST_NAME: https://notifications-api.app.cloud.gov
ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov
STATSD_HOST: localhost
# Credentials variables
INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}'
ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET))
DANGEROUS_SALT: ((DANGEROUS_SALT))
SECRET_KEY: ((SECRET_KEY))
AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID))
@@ -49,5 +46,3 @@ applications:
AWS_REGION: us-west-2
AWS_PINPOINT_REGION: us-west-2
AWS_US_TOLL_FREE_NUMBER: +18446120782
DVLA_EMAIL_ADDRESSES: []

51
tests/app/test_cloudfoundry_config.py
View File

@@ -3,7 +3,14 @@ import os
import pytest
from app.cloudfoundry_config import extract_cloudfoundry_config
from app.cloudfoundry_config import CloudfoundryConfig
bucket_credentials = {
'access_key_id': 'csv-access',
'bucket': 'csv-upload-bucket',
'region': 'us-gov-west-1',
'secret_access_key': 'csv-secret'
}
@pytest.fixture
@@ -22,12 +29,7 @@ def vcap_services():
's3': [
{
'name': 'notifications-api-csv-upload-bucket-test',
'credentials': {
'access_key_id': 'csv-access',
'bucket': 'csv-upload-bucket',
'region': 'us-gov-west-1',
'secret_access_key': 'csv-secret'
}
'credentials': bucket_credentials
},
{
'name': 'notifications-api-contact-list-bucket-test',
@@ -43,12 +45,33 @@ def vcap_services():
}
def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services):
os.environ['DEPLOY_ENV'] = 'test'
def test_redis_url(vcap_services):
os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
extract_cloudfoundry_config()
assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri'
assert os.environ['REDIS_URL'] == 'rediss://xxx:6379'
assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket'
assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket'
assert CloudfoundryConfig().redis_url == 'rediss://xxx:6379'
def test_redis_url_falls_back_to_REDIS_URL():
expected = 'redis://yyy:6379'
os.environ['REDIS_URL'] = expected
os.environ['VCAP_SERVICES'] = ""
assert CloudfoundryConfig().redis_url == expected
def test_s3_bucket_credentials(vcap_services):
os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
assert CloudfoundryConfig().s3_credentials('notifications-api-csv-upload-bucket-test') == bucket_credentials
def test_s3_bucket_credentials_falls_back_to_empty_creds():
os.environ['VCAP_SERVICES'] = ""
expected = {
'bucket': '',
'access_key_id': '',
'secret_access_key': '',
'region': ''
}
assert CloudfoundryConfig().s3_credentials('bucket') == expected

56
tests/app/test_config.py
View File

@@ -1,62 +1,6 @@
import importlib
import os
from unittest import mock
import pytest
from app import config
from app.config import QueueNames
def cf_conf():
os.environ['ADMIN_BASE_URL'] = 'cf'
@pytest.fixture
def reload_config():
"""
Reset config, by simply re-running config.py from a fresh environment
"""
old_env = os.environ.copy()
yield
os.environ.clear()
for k, v in old_env.items():
os.environ[k] = v
importlib.reload(config)
def test_load_cloudfoundry_config_if_available(reload_config):
os.environ['ADMIN_BASE_URL'] = 'env'
os.environ['VCAP_SERVICES'] = 'some json blob'
os.environ['VCAP_APPLICATION'] = 'some json blob'
with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config:
# reload config so that its module level code (ie: all of it) is re-instantiated
importlib.reload(config)
assert cf_config.called
assert os.environ['ADMIN_BASE_URL'] == 'cf'
assert config.Config.ADMIN_BASE_URL == 'cf'
def test_load_config_if_cloudfoundry_not_available(reload_config):
os.environ['ADMIN_BASE_URL'] = 'env'
os.environ.pop('VCAP_SERVICES', None)
with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config:
# reload config so that its module level code (ie: all of it) is re-instantiated
importlib.reload(config)
assert not cf_config.called
assert os.environ['ADMIN_BASE_URL'] == 'env'
assert config.Config.ADMIN_BASE_URL == 'env'
def test_queue_names_all_queues_correct():
# Need to ensure that all_queues() only returns queue names used in API
queues = QueueNames.all_queues()