From d87d673b8583d1143322832efe0255f4f06fdee2 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 18 Oct 2022 16:16:13 -0400 Subject: [PATCH 1/3] Consolidate and simplify environment variables and config.py --- .github/workflows/checks.yml | 13 +- .github/workflows/daily_checks.yml | 14 +- app/config.py | 233 ++++++++++------------------- manifest.yml | 5 +- sample.env | 22 +-- 5 files changed, 79 insertions(+), 208 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 518bee75c..722ff8eb2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -10,20 +10,9 @@ env: FLASK_APP: application.py FLASK_ENV: development WERKZEUG_DEBUG_PIN: off - REDIS_URL: "redis://you-forgot-to-mock-a-redis-call-to" - REDIS_ENABLED: True - ANTIVIRUS_ENABLED: 0 + REDIS_ENABLED: 0 NODE_VERSION: 16.15.1 - ADMIN_CLIENT_ID: notify-admin - ADMIN_CLIENT_USERNAME: notify-admin - ADMIN_CLIENT_SECRET: dev-notify-secret-key - GOVUK_ALERTS_CLIENT_ID: govuk-alerts - ADMIN_BASE_URL: http://localhost:6012 - API_HOST_NAME: http://localhost:6011 - DEV_API_HOST_NAME: http://localhost:6011 AWS_REGION: us-west-2 - BASIC_AUTH_USERNAME: curiousabout - BASIC_AUTH_PASSWORD: the10xnotifybeta jobs: build: diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index 6cc18b884..0f93d0731 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -14,21 +14,9 @@ env: FLASK_APP: application.py FLASK_ENV: development WERKZEUG_DEBUG_PIN: off - REDIS_URL: redis://adminredis:6379/0 - DEV_REDIS_URL: redis://adminredis:6379/0 - REDIS_ENABLED: False - ANTIVIRUS_ENABLED: 0 + REDIS_ENABLED: 0 NODE_VERSION: 16.15.1 - ADMIN_CLIENT_ID: notify-admin - ADMIN_CLIENT_USERNAME: notify-admin - ADMIN_CLIENT_SECRET: dev-notify-secret-key - GOVUK_ALERTS_CLIENT_ID: govuk-alerts - ADMIN_BASE_URL: http://localhost:6012 - API_HOST_NAME: http://localhost:6011 - DEV_API_HOST_NAME: http://localhost:6011 AWS_REGION: us-west-2 - BASIC_AUTH_USERNAME: curiousabout - BASIC_AUTH_PASSWORD: the10xnotifybeta jobs: dependency-audits: diff --git a/app/config.py b/app/config.py index 95521c451..e99a79eaa 100644 --- a/app/config.py +++ b/app/config.py @@ -9,39 +9,43 @@ if os.environ.get('VCAP_SERVICES'): class Config(object): + NOTIFY_APP_NAME = 'admin' + NOTIFY_ENVIRONMENT = os.environ.get('NOTIFY_ENVIRONMENT', 'development') + API_HOST_NAME = os.environ.get('API_HOST_NAME', 'localhost') + ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL', 'http://localhost:6012') + HEADER_COLOUR = '#81878b' # mix(govuk-colour("dark-grey"), govuk-colour("mid-grey")) + LOGO_CDN_DOMAIN = 'static-logos.notifications.service.gov.uk' # TODO use our own CDN + ASSETS_DEBUG = False + + # Credentials ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET') ADMIN_CLIENT_USER_NAME = os.environ.get('ADMIN_CLIENT_USERNAME') - API_HOST_NAME = os.environ.get('API_HOST_NAME', 'localhost') SECRET_KEY = os.environ.get('SECRET_KEY') DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT') - ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY') + # ZENDESK_API_KEY = os.environ.get('ZENDESK_API_KEY') + ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1') + ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2') + BASIC_AUTH_USERNAME = os.environ.get('BASIC_AUTH_USERNAME') + BASIC_AUTH_PASSWORD = os.environ.get('BASIC_AUTH_PASSWORD') - # if we're not on cloudfoundry, we can get to this app from localhost. but on cloudfoundry its different - ADMIN_BASE_URL = os.environ.get('ADMIN_BASE_URL', 'http://localhost:6012') - - TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013') + TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:9999') TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key') + ANTIVIRUS_API_HOST = os.environ.get('ANTIVIRUS_API_HOST', 'http://localhost:6016') + ANTIVIRUS_API_KEY = os.environ.get('ANTIVIRUS_API_KEY', 'test-key') # Logging - DEBUG = False + NOTIFY_LOG_LEVEL = os.environ.get('NOTIFY_LOG_LEVEL', 'INFO') NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log') - ANTIVIRUS_API_HOST = os.environ.get('ANTIVIRUS_API_HOST') - ANTIVIRUS_API_KEY = os.environ.get('ANTIVIRUS_API_KEY') - - ASSETS_DEBUG = False - AWS_REGION = os.environ.get('AWS_REGION') DEFAULT_SERVICE_LIMIT = 50 EMAIL_EXPIRY_SECONDS = 3600 # 1 hour INVITATION_EXPIRY_SECONDS = 3600 * 24 * 2 # 2 days - also set on api EMAIL_2FA_EXPIRY_SECONDS = 1800 # 30 Minutes - HEADER_COLOUR = '#81878b' # mix(govuk-colour("dark-grey"), govuk-colour("mid-grey")) - HTTP_PROTOCOL = 'http' - NOTIFY_APP_NAME = 'admin' - NOTIFY_LOG_LEVEL = os.environ.get('NOTIFY_LOG_LEVEL', 'INFO') PERMANENT_SESSION_LIFETIME = 20 * 60 * 60 # 20 hours SEND_FILE_MAX_AGE_DEFAULT = 365 * 24 * 60 * 60 # 1 year + REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45 + ACTIVITY_STATS_LIMIT_DAYS = 7 SESSION_COOKIE_HTTPONLY = True SESSION_COOKIE_NAME = 'notify_admin_session' SESSION_COOKIE_SECURE = True @@ -51,39 +55,13 @@ class Config(object): SESSION_REFRESH_EACH_REQUEST = False WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None - CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' - CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CSV_UPLOAD_REGION = os.environ.get('AWS_REGION') - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'local-contact-list' - CONTACT_LIST_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CONTACT_LIST_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CONTACT_LIST_UPLOAD_REGION = os.environ.get('AWS_REGION') - ACTIVITY_STATS_LIMIT_DAYS = 7 - - REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45 - - NOTIFY_ENVIRONMENT = 'development' - LOGO_UPLOAD_BUCKET_NAME = 'public-logos-local' - LOGO_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - LOGO_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - LOGO_UPLOAD_REGION = os.environ.get('AWS_REGION') - # MOU_BUCKET_NAME = 'local-mou' - # TRANSIENT_UPLOADED_LETTERS = 'local-transient-uploaded-letters' - ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1') - ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2') CHECK_PROXY_HEADER = False - ANTIVIRUS_ENABLED = True + ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1' + + AWS_REGION = os.environ.get('AWS_REGION') REDIS_URL = os.environ.get('REDIS_URL') - REDIS_ENABLED = True - - BASIC_AUTH_USERNAME = os.environ.get('BASIC_AUTH_USERNAME') - BASIC_AUTH_PASSWORD = os.environ.get('BASIC_AUTH_PASSWORD') - BASIC_AUTH_FORCE = True - - ASSET_DOMAIN = '' - ASSET_PATH = '/static/' + REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '1') == '1' # as defined in api db migration 0331_add_broadcast_org.py BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df' @@ -106,167 +84,106 @@ class Config(object): class Development(Config): - ADMIN_BASE_URL = 'http://localhost:6012' BASIC_AUTH_FORCE = False - NOTIFY_LOG_PATH = 'application.log' DEBUG = True SESSION_COOKIE_SECURE = False SESSION_PROTECTION = None + HTTP_PROTOCOL = 'http' + ASSET_DOMAIN = '' + ASSET_PATH = '/static/' # Buckets CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' # created in gsa sandbox + CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') + CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') + CSV_UPLOAD_REGION = os.environ.get('AWS_REGION') CONTACT_LIST_UPLOAD_BUCKET_NAME = 'local-contact-list' # created in gsa sandbox + CONTACT_LIST_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') + CONTACT_LIST_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') + CONTACT_LIST_UPLOAD_REGION = os.environ.get('AWS_REGION') LOGO_UPLOAD_BUCKET_NAME = 'local-public-logos-tools' # created in gsa sandbox - # MOU_BUCKET_NAME = 'local-notify-tools-mou' # created in gsa sandbox + LOGO_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') + LOGO_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') + LOGO_UPLOAD_REGION = os.environ.get('AWS_REGION') + # MOU_BUCKET_NAME = 'local-notify-tools-mou' # not created in gsa sandbox # TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters' # not created in gsa sandbox # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = # 'development-letters-precompiled-originals-backup' # not created in sandbox - ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET') - # check for local compose orchestration variable - API_HOST_NAME = os.environ.get('DEV_API_HOST_NAME', 'http://dev:6011') + # credential overrides DANGEROUS_SALT = 'dev-notify-salt' SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - only used in development - ANTIVIRUS_API_HOST = 'http://localhost:6016' - ANTIVIRUS_API_KEY = 'test-key' - ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1' - - ASSET_PATH = '/static/' - LOGO_CDN_DOMAIN = 'static-logos.notify.tools' # replace with our own CDN + # ADMIN_CLIENT_USER_NAME is called ADMIN_CLIENT_ID in api repo, they should match + ADMIN_CLIENT_USER_NAME = 'notify-admin' + ADMIN_CLIENT_SECRET = 'dev-notify-secret-key' # nosec B105 - only used in development class Test(Development): - BASIC_AUTH_FORCE = False - DEBUG = True TESTING = True WTF_CSRF_ENABLED = False - CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'test-contact-list' - LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test' - LOGO_CDN_DOMAIN = 'static-logos.test.com' - # MOU_BUCKET_NAME = 'test-mou' - # TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters' - # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup' - NOTIFY_ENVIRONMENT = 'test' - API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to' - REDIS_URL = 'redis://you-forgot-to-mock-a-redis-call-to' - TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999' - ANTIVIRUS_API_HOST = 'https://test-antivirus' - ANTIVIRUS_API_KEY = 'test-antivirus-secret' - ANTIVIRUS_ENABLED = True - ASSET_DOMAIN = 'static.example.com' ASSET_PATH = 'https://static.example.com/' + # none of these buckets actually exist + CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' + CONTACT_LIST_UPLOAD_BUCKET_NAME = 'test-contact-list' + LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test' + # MOU_BUCKET_NAME = 'test-mou' + # TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters' + # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup' -class Scanning(Test): - BASIC_AUTH_FORCE = False - API_HOST_NAME = 'https://notifications-api.app.cloud.gov/' - NOTIFY_ENVIRONMENT = 'scanning' - ASSET_DOMAIN = '' - ASSET_PATH = '/static/' + API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to' + REDIS_URL = 'redis://you-forgot-to-mock-a-redis-call-to' + ANTIVIRUS_API_HOST = 'https://test-antivirus' + ANTIVIRUS_API_KEY = 'test-antivirus-secret' + ANTIVIRUS_ENABLED = True + LOGO_CDN_DOMAIN = 'static-logos.test.com' -class Preview(Config): - BASIC_AUTH_FORCE = True - HTTP_PROTOCOL = 'https' - HEADER_COLOUR = '#F499BE' # $baby-pink - CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'preview-contact-list' - LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview' - LOGO_CDN_DOMAIN = 'static-logos.notify.works' - # MOU_BUCKET_NAME = 'notify.works-mou' - # TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters' - # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'preview-letters-precompiled-originals-backup' - NOTIFY_ENVIRONMENT = 'preview' - CHECK_PROXY_HEADER = False - ASSET_DOMAIN = 'static.notify.works' - ASSET_PATH = 'https://static.notify.works/' - - # On preview, extend the validation timeout to allow more leniency when running functional tests - REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 120 - - -class Staging(Config): - BASIC_AUTH_FORCE = True - HTTP_PROTOCOL = 'https' - HEADER_COLOUR = '#6F72AF' # $mauve - CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload' - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'staging-contact-list' - LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging' - LOGO_CDN_DOMAIN = 'static-logos.staging-notify.works' - # MOU_BUCKET_NAME = 'staging-notify.works-mou' - # TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters' - # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'staging-letters-precompiled-originals-backup' - NOTIFY_ENVIRONMENT = 'staging' - CHECK_PROXY_HEADER = False - ASSET_DOMAIN = 'static.staging-notify.works' - ASSET_PATH = 'https://static.staging-notify.works/' - - -class Live(Config): - BASIC_AUTH_FORCE = True +class Production(Config): HEADER_COLOUR = '#005EA5' # $govuk-blue HTTP_PROTOCOL = 'https' + BASIC_AUTH_FORCE = True + ASSET_DOMAIN = '' # TODO use a CDN + ASSET_PATH = '/static/' # TODO use a CDN + DEBUG = False + # buckets - CSV_UPLOAD_BUCKET_NAME = os.environ.get( - 'CSV_UPLOAD_BUCKET_NAME', 'notifications-prototype-csv-upload') # created in gsa sandbox + CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME') CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY') CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY') CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION') - CONTACT_LIST_UPLOAD_BUCKET_NAME = os.environ.get( - 'CONTACT_LIST_BUCKET_NAME', 'notifications-prototype-contact-list-upload') # created in gsa sandbox + CONTACT_LIST_UPLOAD_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME') CONTACT_LIST_UPLOAD_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY') CONTACT_LIST_UPLOAD_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY') CONTACT_LIST_UPLOAD_REGION = os.environ.get('CONTACT_LIST_REGION') - LOGO_UPLOAD_BUCKET_NAME = os.environ.get( - 'LOGO_UPLOAD_BUCKET_NAME', 'notifications-prototype-logo-upload') # created in gsa sandbox + LOGO_UPLOAD_BUCKET_NAME = os.environ.get('LOGO_UPLOAD_BUCKET_NAME') LOGO_UPLOAD_ACCESS_KEY = os.environ.get('LOGO_UPLOAD_ACCESS_KEY') LOGO_UPLOAD_SECRET_KEY = os.environ.get('LOGO_UPLOAD_SECRET_KEY') LOGO_UPLOAD_REGION = os.environ.get('LOGO_UPLOAD_REGION') - # MOU_BUCKET_NAME = os.environ.get( - # 'MOU_UPLOAD_BUCKET_NAME', 'notifications-prototype-mou') # created in gsa sandbox + # MOU_BUCKET_NAME = os.environ.get('MOU_UPLOAD_BUCKET_NAME') # TRANSIENT_UPLOADED_LETTERS = 'prototype-transient-uploaded-letters' # not created in gsa sandbox # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'prototype-letters-precompiled-originals-backup' # not in sandbox - NOTIFY_ENVIRONMENT = 'live' - CHECK_PROXY_HEADER = False - # ASSET_DOMAIN = 'static.notifications.service.gov.uk' - # ASSET_PATH = 'https://static.notifications.service.gov.uk/' - ASSET_DOMAIN = '' # TODO use a CDN - ASSET_PATH = '/static/' # TODO use a CDN - LOGO_CDN_DOMAIN = 'static-logos.notifications.service.gov.uk' # TODO use our own CDN - ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET') - ADMIN_CLIENT_USER_NAME = os.environ.get('ADMIN_CLIENT_USERNAME') - API_HOST_NAME = os.environ.get('API_HOST_NAME') - DANGEROUS_SALT = os.environ.get('DANGEROUS_SALT') - SECRET_KEY = os.environ.get('SECRET_KEY') - ANTIVIRUS_API_HOST = 'http://localhost:6016' - ANTIVIRUS_API_KEY = 'test-key' - ANTIVIRUS_ENABLED = False +class Staging(Production): + BASIC_AUTH_FORCE = True + HEADER_COLOUR = '#6F72AF' # $mauve -class CloudFoundryConfig(Config): - pass - - -# CloudFoundry sandbox -class Sandbox(CloudFoundryConfig): - HTTP_PROTOCOL = 'https' - HEADER_COLOUR = '#F499BE' # $baby-pink - CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload' - LOGO_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-logo-upload' - NOTIFY_ENVIRONMENT = 'sandbox' +class Scanning(Production): + BASIC_AUTH_FORCE = False + HTTP_PROTOCOL = 'http' + API_HOST_NAME = 'https://notifications-api.app.cloud.gov/' + SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - only used in development + ADMIN_CLIENT_USER_NAME = 'notify-admin' + ADMIN_CLIENT_SECRET = 'dev-notify-secret-key' # nosec B105 - only used in development configs = { 'development': Development, 'test': Test, 'scanning': Scanning, - 'preview': Preview, 'staging': Staging, - 'live': Live, - 'production': Live, - 'sandbox': Sandbox + 'production': Production } diff --git a/manifest.yml b/manifest.yml index d1fabb8aa..36b4eaf62 100644 --- a/manifest.yml +++ b/manifest.yml @@ -10,9 +10,6 @@ applications: - route: notifications-admin.app.cloud.gov services: - # - logit-ssl-syslog-drain - # - notify-prometheus - # - notify-splunk - notifications-admin-redis-((env)) - notifications-api-csv-upload-bucket-((env)) - notifications-api-contact-list-bucket-((env)) @@ -27,7 +24,7 @@ applications: DEPLOY_ENV: ((env)) REDIS_ENABLED: ((REDIS_ENABLED)) - NOTIFY_ENVIRONMENT: live + NOTIFY_ENVIRONMENT: ((env)) # Credentials variables ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET)) diff --git a/sample.env b/sample.env index 6fdce3b14..663dd35e6 100644 --- a/sample.env +++ b/sample.env @@ -5,34 +5,14 @@ FLASK_APP=application.py FLASK_ENV=development WERKZEUG_DEBUG_PIN=off -REDIS_URL=redis://adminredis:6379/0 -DEV_REDIS_URL=redis://adminredis:6379/0 -REDIS_ENABLED=True - -SQLALCHEMY_DATABASE_URI=postgres://postgres:chummy@db:5432/notification_api - ANTIVIRUS_ENABLED=0 NODE_VERSION=16.15.1 -# ADMIN_CLIENT_USERNAME is called ADMIN_CLIENT_ID in api repo, they should match -ADMIN_CLIENT_USERNAME=notify-admin -ADMIN_CLIENT_SECRET=dev-notify-secret-key -GOVUK_ALERTS_CLIENT_ID=govuk-alerts - -# URL of admin app -ADMIN_BASE_URL=http://admin:6012 - # URL of api app (on AWS this is the internal api endpoint) API_HOST_NAME=http://dev:6011 -DEV_API_HOST_NAME=http://dev:6011 +REDIS_URL=redis://adminredis:6379/0 # AWS AWS_REGION=us-west-2 AWS_ACCESS_KEY_ID="don't write secrets to the sample file" AWS_SECRET_ACCESS_KEY="don't write secrets to the sample file" - -# beta protection -BASIC_AUTH_USERNAME=username -BASIC_AUTH_PASSWORD=password - - From 921c872bd1c149ea419f729ae3e450e631367267 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Thu, 20 Oct 2022 08:03:32 -0400 Subject: [PATCH 2/3] Refactor method of retrieving credentials out of VCAP_SERVICES --- app/cloudfoundry_config.py | 62 +++++++--------- app/config.py | 71 +++++++------------ app/models/contact_list.py | 12 ++-- app/s3_client/s3_csv_client.py | 8 +-- app/s3_client/s3_logo_client.py | 36 +++++----- .../views/uploads/test_upload_contact_list.py | 2 +- tests/app/s3_client/test_s3_csv_client.py | 4 +- tests/app/s3_client/test_s3_logo_client.py | 12 +++- tests/app/test_cloudfoundry_config.py | 50 +++++++++---- tests/app/test_config.py | 53 -------------- 10 files changed, 132 insertions(+), 178 deletions(-) delete mode 100644 tests/app/test_config.py diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 927fee6ea..1218e8852 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -2,42 +2,30 @@ import json import os -def find_by_service_name(services, service_name): - for i in range(len(services)): - if services[i]['name'] == service_name: - return services[i] - return None +class CloudfoundryConfig: + def __init__(self): + self.parsed_services = json.loads(os.environ.get('VCAP_SERVICES') or '{}') + buckets = self.parsed_services.get('s3') or [] + self.s3_buckets = {bucket['name']: bucket['credentials'] for bucket in buckets} + self._empty_bucket_credentials = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + @property + def redis_url(self): + try: + return self.parsed_services['aws-elasticache-redis'][0]['credentials']['uri'].replace( + 'redis://', + 'rediss://' + ) + except KeyError: + return os.environ.get('REDIS_URL') + + def s3_credentials(self, service_name): + return self.s3_buckets.get(service_name) or self._empty_bucket_credentials -def extract_cloudfoundry_config(): - vcap_services = json.loads(os.environ['VCAP_SERVICES']) - - # Redis config - os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis', 'rediss') - - # CSV Upload Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}") - if bucket_service: - os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CSV_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region'] - - # Contact List Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}") - if bucket_service: - os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['CONTACT_LIST_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['CONTACT_LIST_REGION'] = bucket_service['credentials']['region'] - - # Logo Upload Bucket Name - bucket_service = find_by_service_name( - vcap_services['s3'], f"notifications-admin-logo-upload-bucket-{os.environ['DEPLOY_ENV']}") - if bucket_service: - os.environ['LOGO_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket'] - os.environ['LOGO_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] - os.environ['LOGO_UPLOAD_SECRET_KEY'] = bucket_service['credentials']['secret_access_key'] - os.environ['LOGO_UPLOAD_REGION'] = bucket_service['credentials']['region'] +cloud_config = CloudfoundryConfig() diff --git a/app/config.py b/app/config.py index e99a79eaa..668d02987 100644 --- a/app/config.py +++ b/app/config.py @@ -1,11 +1,7 @@ import json import os -if os.environ.get('VCAP_SERVICES'): - # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate - # standard environment variables from it - from app.cloudfoundry_config import extract_cloudfoundry_config - extract_cloudfoundry_config() +from app.cloudfoundry_config import cloud_config class Config(object): @@ -60,7 +56,7 @@ class Config(object): AWS_REGION = os.environ.get('AWS_REGION') - REDIS_URL = os.environ.get('REDIS_URL') + REDIS_URL = cloud_config.redis_url REDIS_ENABLED = os.environ.get('REDIS_ENABLED', '1') == '1' # as defined in api db migration 0331_add_broadcast_org.py @@ -93,22 +89,24 @@ class Development(Config): ASSET_PATH = '/static/' # Buckets - CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' # created in gsa sandbox - CSV_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CSV_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CSV_UPLOAD_REGION = os.environ.get('AWS_REGION') - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'local-contact-list' # created in gsa sandbox - CONTACT_LIST_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - CONTACT_LIST_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - CONTACT_LIST_UPLOAD_REGION = os.environ.get('AWS_REGION') - LOGO_UPLOAD_BUCKET_NAME = 'local-public-logos-tools' # created in gsa sandbox - LOGO_UPLOAD_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY_ID') - LOGO_UPLOAD_SECRET_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') - LOGO_UPLOAD_REGION = os.environ.get('AWS_REGION') - # MOU_BUCKET_NAME = 'local-notify-tools-mou' # not created in gsa sandbox - # TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters' # not created in gsa sandbox - # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = - # 'development-letters-precompiled-originals-backup' # not created in sandbox + CSV_UPLOAD_BUCKET = { + 'bucket': 'local-notifications-csv-upload', + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } + CONTACT_LIST_BUCKET = { + 'bucket': 'local-contact-list', + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } + LOGO_UPLOAD_BUCKET = { + 'bucket': 'local-public-logos-tools', + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } # credential overrides DANGEROUS_SALT = 'dev-notify-salt' @@ -124,14 +122,6 @@ class Test(Development): ASSET_DOMAIN = 'static.example.com' ASSET_PATH = 'https://static.example.com/' - # none of these buckets actually exist - CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'test-contact-list' - LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test' - # MOU_BUCKET_NAME = 'test-mou' - # TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters' - # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup' - API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to' REDIS_URL = 'redis://you-forgot-to-mock-a-redis-call-to' ANTIVIRUS_API_HOST = 'https://test-antivirus' @@ -149,21 +139,12 @@ class Production(Config): DEBUG = False # buckets - CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME') - CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY') - CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY') - CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION') - CONTACT_LIST_UPLOAD_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME') - CONTACT_LIST_UPLOAD_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY') - CONTACT_LIST_UPLOAD_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY') - CONTACT_LIST_UPLOAD_REGION = os.environ.get('CONTACT_LIST_REGION') - LOGO_UPLOAD_BUCKET_NAME = os.environ.get('LOGO_UPLOAD_BUCKET_NAME') - LOGO_UPLOAD_ACCESS_KEY = os.environ.get('LOGO_UPLOAD_ACCESS_KEY') - LOGO_UPLOAD_SECRET_KEY = os.environ.get('LOGO_UPLOAD_SECRET_KEY') - LOGO_UPLOAD_REGION = os.environ.get('LOGO_UPLOAD_REGION') - # MOU_BUCKET_NAME = os.environ.get('MOU_UPLOAD_BUCKET_NAME') - # TRANSIENT_UPLOADED_LETTERS = 'prototype-transient-uploaded-letters' # not created in gsa sandbox - # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'prototype-letters-precompiled-originals-backup' # not in sandbox + CSV_UPLOAD_BUCKET = cloud_config.s3_credentials( + f"notifications-api-csv-upload-bucket-{os.environ['NOTIFY_ENVIRONMENT']}") + CONTACT_LIST_BUCKET = cloud_config.s3_credentials( + f"notifications-api-contact-list-bucket-{os.environ['NOTIFY_ENVIRONMENT']}") + LOGO_UPLOAD_BUCKET = cloud_config.s3_credentials( + f"notifications-admin-logo-upload-bucket-{os.environ['NOTIFY_ENVIRONMENT']}") class Staging(Production): diff --git a/app/models/contact_list.py b/app/models/contact_list.py index 17f97c40b..dfecaf50a 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -44,21 +44,25 @@ class ContactList(JSONModel): contact_list_id=contact_list_id, )) + @staticmethod + def get_bucket_credentials(key): + return current_app.config['CONTACT_LIST_BUCKET'][key] + @staticmethod def get_bucket_name(): - return current_app.config['CONTACT_LIST_UPLOAD_BUCKET_NAME'] + return ContactList.get_bucket_credentials('bucket') @staticmethod def get_access_key(): - return current_app.config['CONTACT_LIST_UPLOAD_ACCESS_KEY'] + return ContactList.get_bucket_credentials('access_key_id') @staticmethod def get_secret_key(): - return current_app.config['CONTACT_LIST_UPLOAD_SECRET_KEY'] + return ContactList.get_bucket_credentials('secret_access_key') @staticmethod def get_region(): - return current_app.config['CONTACT_LIST_UPLOAD_REGION'] + return ContactList.get_bucket_credentials('region') @staticmethod def get_filename(service_id, upload_id): diff --git a/app/s3_client/s3_csv_client.py b/app/s3_client/s3_csv_client.py index 93672d74b..8422a11e8 100644 --- a/app/s3_client/s3_csv_client.py +++ b/app/s3_client/s3_csv_client.py @@ -15,11 +15,11 @@ FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv' def get_csv_location(service_id, upload_id): return ( - current_app.config['CSV_UPLOAD_BUCKET_NAME'], + current_app.config['CSV_UPLOAD_BUCKET']['bucket'], FILE_LOCATION_STRUCTURE.format(service_id, upload_id), - current_app.config['CSV_UPLOAD_ACCESS_KEY'], - current_app.config['CSV_UPLOAD_SECRET_KEY'], - current_app.config['CSV_UPLOAD_REGION'], + current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'], + current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'], + current_app.config['CSV_UPLOAD_BUCKET']['region'], ) diff --git a/app/s3_client/s3_logo_client.py b/app/s3_client/s3_logo_client.py index b7f6a523e..aba076624 100644 --- a/app/s3_client/s3_logo_client.py +++ b/app/s3_client/s3_logo_client.py @@ -15,14 +15,18 @@ LETTER_TEMP_LOGO_LOCATION = 'letters/static/images/letter-template/temp-{user_id def get_logo_location(filename=None): return ( - current_app.config['LOGO_UPLOAD_BUCKET_NAME'], + bucket_creds('bucket'), filename, - current_app.config['LOGO_UPLOAD_ACCESS_KEY'], - current_app.config['LOGO_UPLOAD_SECRET_KEY'], - current_app.config['LOGO_UPLOAD_REGION'], + bucket_creds('access_key_id'), + bucket_creds('secret_access_key'), + bucket_creds('region'), ) +def bucket_creds(key): + return current_app.config['LOGO_UPLOAD_BUCKET'][key] + + def delete_s3_object(filename): get_s3_object(*get_logo_location(filename)).delete() @@ -37,10 +41,10 @@ def persist_logo(old_name, new_name): def get_s3_objects_filter_by_prefix(prefix): - bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME'] - session = Session(aws_access_key_id=current_app.config['LOGO_UPLOAD_ACCESS_KEY'], - aws_secret_access_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'], - region_name=current_app.config['LOGO_UPLOAD_REGION']) + bucket_name = bucket_creds('bucket') + session = Session(aws_access_key_id=bucket_creds('access_key_id'), + aws_secret_access_key=bucket_creds('secret_access_key'), + region_name=bucket_creds('region')) s3 = session.resource('s3') return s3.Bucket(bucket_name).objects.filter(Prefix=prefix) @@ -59,15 +63,15 @@ def upload_email_logo(filename, filedata, user_id): unique_id=str(uuid.uuid4()), filename=filename ) - bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME'] + bucket_name = bucket_creds('bucket') utils_s3upload( filedata=filedata, - region=current_app.config['LOGO_UPLOAD_REGION'], + region=bucket_creds('region'), bucket_name=bucket_name, file_location=upload_file_name, content_type='image/png', - access_key=current_app.config['LOGO_UPLOAD_ACCESS_KEY'], - secret_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'], + access_key=bucket_creds('access_key_id'), + secret_key=bucket_creds('secret_access_key'), ) return upload_file_name @@ -79,15 +83,15 @@ def upload_letter_temp_logo(filename, filedata, user_id): unique_id=str(uuid.uuid4()), filename=filename ) - bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME'] + bucket_name = bucket_creds('bucket') utils_s3upload( filedata=filedata, - region=current_app.config['LOGO_UPLOAD_REGION'], + region=bucket_creds('region'), bucket_name=bucket_name, file_location=upload_filename, content_type='image/svg+xml', - access_key=current_app.config['LOGO_UPLOAD_ACCESS_KEY'], - secret_key=current_app.config['LOGO_UPLOAD_SECRET_KEY'], + access_key=bucket_creds('access_key_id'), + secret_key=bucket_creds('secret_access_key'), ) return upload_filename diff --git a/tests/app/main/views/uploads/test_upload_contact_list.py b/tests/app/main/views/uploads/test_upload_contact_list.py index 1a58c7453..7f69be12f 100644 --- a/tests/app/main/views/uploads/test_upload_contact_list.py +++ b/tests/app/main/views/uploads/test_upload_contact_list.py @@ -208,7 +208,7 @@ def test_upload_csv_file_shows_error_banner( mock_upload.assert_called_once_with( filedata='', region='us-west-2', - bucket_name='test-contact-list', + bucket_name='local-contact-list', file_location=f"service-{SERVICE_ONE_ID}-notify/{fake_uuid}.csv", access_key=default_access_key, secret_key=default_secret_key, diff --git a/tests/app/s3_client/test_s3_csv_client.py b/tests/app/s3_client/test_s3_csv_client.py index 1ffb498b0..b2fc04341 100644 --- a/tests/app/s3_client/test_s3_csv_client.py +++ b/tests/app/s3_client/test_s3_csv_client.py @@ -4,7 +4,7 @@ from app.s3_client.s3_csv_client import set_metadata_on_csv_upload def test_sets_metadata(client_request, mocker): - mocked_s3_object = Mock(bucket_name='test-notifications-csv-upload', key='service-1234-notify/5678.csv') + mocked_s3_object = Mock(bucket_name='local-notifications-csv-upload', key='service-1234-notify/5678.csv') mocked_get_s3_object = mocker.patch( 'app.s3_client.s3_csv_client.get_csv_upload', return_value=mocked_s3_object, @@ -14,7 +14,7 @@ def test_sets_metadata(client_request, mocker): mocked_get_s3_object.assert_called_once_with('1234', '5678') mocked_s3_object.copy_from.assert_called_once_with( - CopySource='test-notifications-csv-upload/service-1234-notify/5678.csv', + CopySource='local-notifications-csv-upload/service-1234-notify/5678.csv', Metadata={'baz': 'True', 'foo': 'bar'}, MetadataDirective='REPLACE', ServerSideEncryption='AES256', diff --git a/tests/app/s3_client/test_s3_logo_client.py b/tests/app/s3_client/test_s3_logo_client.py index 49f2062cb..4011b23c6 100644 --- a/tests/app/s3_client/test_s3_logo_client.py +++ b/tests/app/s3_client/test_s3_logo_client.py @@ -21,6 +21,12 @@ from app.s3_client.s3_logo_client import ( ) bucket = 'test_bucket' +bucket_credentials = { + 'bucket': bucket, + 'access_key_id': default_access_key, + 'secret_access_key': default_secret_key, + 'region': default_region +} data = {'data': 'some_data'} filename = 'test.png' svg_filename = 'test.svg' @@ -45,7 +51,7 @@ def letter_upload_filename(fake_uuid): def test_upload_email_logo_calls_correct_args(client_request, mocker, fake_uuid, upload_filename): mocker.patch('uuid.uuid4', return_value=upload_id) - mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET_NAME': bucket}) + mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET': bucket_credentials}) mocked_s3_upload = mocker.patch('app.s3_client.s3_logo_client.utils_s3upload') upload_email_logo(filename=filename, user_id=fake_uuid, filedata=data) @@ -63,7 +69,7 @@ def test_upload_email_logo_calls_correct_args(client_request, mocker, fake_uuid, def test_upload_letter_temp_logo_calls_correct_args(mocker, fake_uuid, letter_upload_filename): mocker.patch('uuid.uuid4', return_value=upload_id) - mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET_NAME': bucket}) + mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET': bucket_credentials}) mocked_s3_upload = mocker.patch('app.s3_client.s3_logo_client.utils_s3upload') new_filename = upload_letter_temp_logo(filename=svg_filename, user_id=fake_uuid, filedata=data) @@ -81,7 +87,7 @@ def test_upload_letter_temp_logo_calls_correct_args(mocker, fake_uuid, letter_up def test_persist_logo(client_request, mocker, fake_uuid, upload_filename): - mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET_NAME': bucket}) + mocker.patch.dict('flask.current_app.config', {'LOGO_UPLOAD_BUCKET': bucket_credentials}) mocked_get_s3_object = mocker.patch('app.s3_client.s3_logo_client.get_s3_object') mocked_delete_s3_object = mocker.patch('app.s3_client.s3_logo_client.delete_s3_object') diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index f2df67971..79ad640e2 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -3,7 +3,14 @@ import os import pytest -from app.cloudfoundry_config import extract_cloudfoundry_config +from app.cloudfoundry_config import CloudfoundryConfig + +bucket_credentials = { + 'access_key_id': 'contact-list-access', + 'bucket': 'contact-list-bucket', + 'region': 'us-gov-west-1', + 'secret_access_key': 'contact-list-secret' +} @pytest.fixture @@ -26,22 +33,39 @@ def vcap_services(): }, { 'name': 'notifications-api-contact-list-bucket-test', - 'credentials': { - 'access_key_id': 'contact-list-access', - 'bucket': 'contact-list-bucket', - 'region': 'us-gov-west-1', - 'secret_access_key': 'contact-list-secret' - } + 'credentials': bucket_credentials } ], } -def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services): - os.environ['DEPLOY_ENV'] = 'test' +def test_redis_url(vcap_services): os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) - extract_cloudfoundry_config() - assert os.environ['REDIS_URL'] == 'rediss://xxx:6379' - assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket' - assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket' + assert CloudfoundryConfig().redis_url == 'rediss://xxx:6379' + + +def test_redis_url_falls_back_to_REDIS_URL(): + expected = 'rediss://yyy:6379' + os.environ['REDIS_URL'] = expected + os.environ['VCAP_SERVICES'] = "" + + assert CloudfoundryConfig().redis_url == expected + + +def test_s3_bucket_credentials(vcap_services): + os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) + + assert CloudfoundryConfig().s3_credentials('notifications-api-contact-list-bucket-test') == bucket_credentials + + +def test_s3_bucket_credentials_falls_back_to_empty_creds(): + os.environ['VCAP_SERVICES'] = "" + expected = { + 'bucket': '', + 'access_key_id': '', + 'secret_access_key': '', + 'region': '' + } + + assert CloudfoundryConfig().s3_credentials('bucket') == expected diff --git a/tests/app/test_config.py b/tests/app/test_config.py deleted file mode 100644 index 152dd5f35..000000000 --- a/tests/app/test_config.py +++ /dev/null @@ -1,53 +0,0 @@ -import importlib -import os -from unittest import mock - -import pytest - -from app import config - - -def cf_conf(): - os.environ['REDIS_URL'] = 'rediss://xxx:6379' - - -@pytest.fixture -def reload_config(os_environ): - """ - Reset config, by simply re-running config.py from a fresh environment - """ - old_env = os.environ.copy() - os.environ.clear() - - yield - - os.environ.clear() - for k, v in old_env.items(): - os.environ[k] = v - importlib.reload(config) - - -def test_load_cloudfoundry_config_if_available(reload_config): - os.environ['REDIS_URL'] = 'some uri' - os.environ['VCAP_SERVICES'] = 'some json blob' - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf): - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert os.environ['REDIS_URL'] == 'rediss://xxx:6379' - assert config.Config.REDIS_URL == 'rediss://xxx:6379' - - -def test_load_config_if_cloudfoundry_not_available(reload_config): - os.environ['REDIS_URL'] = 'redis://xxx:6379' - os.environ.pop('VCAP_SERVICES', None) - - with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config: - # reload config so that its module level code (ie: all of it) is re-instantiated - importlib.reload(config) - - assert not cf_config.called - - assert os.environ['REDIS_URL'] == 'redis://xxx:6379' - assert config.Config.REDIS_URL == 'redis://xxx:6379' From d98e25180f1ac241a6a5492adb048e41cf5542f5 Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Wed, 26 Oct 2022 13:19:07 +0000 Subject: [PATCH 3/3] Use nonexistant buckets in testing to verify mocks --- app/config.py | 35 +++++++++---------- .../views/uploads/test_upload_contact_list.py | 2 +- tests/app/s3_client/test_s3_csv_client.py | 4 +-- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/app/config.py b/app/config.py index 668d02987..0b461e235 100644 --- a/app/config.py +++ b/app/config.py @@ -79,6 +79,15 @@ class Config(object): } +def _default_s3_credentials(bucket_name): + return { + 'bucket': bucket_name, + 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), + 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'region': os.environ.get('AWS_REGION') + } + + class Development(Config): BASIC_AUTH_FORCE = False DEBUG = True @@ -89,24 +98,9 @@ class Development(Config): ASSET_PATH = '/static/' # Buckets - CSV_UPLOAD_BUCKET = { - 'bucket': 'local-notifications-csv-upload', - 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), - 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), - 'region': os.environ.get('AWS_REGION') - } - CONTACT_LIST_BUCKET = { - 'bucket': 'local-contact-list', - 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), - 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), - 'region': os.environ.get('AWS_REGION') - } - LOGO_UPLOAD_BUCKET = { - 'bucket': 'local-public-logos-tools', - 'access_key_id': os.environ.get('AWS_ACCESS_KEY_ID'), - 'secret_access_key': os.environ.get('AWS_SECRET_ACCESS_KEY'), - 'region': os.environ.get('AWS_REGION') - } + CSV_UPLOAD_BUCKET = _default_s3_credentials('local-notifications-csv-upload') + CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list') + LOGO_UPLOAD_BUCKET = _default_s3_credentials('local-public-logos-tools') # credential overrides DANGEROUS_SALT = 'dev-notify-salt' @@ -129,6 +123,11 @@ class Test(Development): ANTIVIRUS_ENABLED = True LOGO_CDN_DOMAIN = 'static-logos.test.com' + # Buckets + CSV_UPLOAD_BUCKET = _default_s3_credentials('test-csv-upload') + CONTACT_LIST_BUCKET = _default_s3_credentials('test-contact-list') + LOGO_UPLOAD_BUCKET = _default_s3_credentials('test-logo-upload') + class Production(Config): HEADER_COLOUR = '#005EA5' # $govuk-blue diff --git a/tests/app/main/views/uploads/test_upload_contact_list.py b/tests/app/main/views/uploads/test_upload_contact_list.py index 7f69be12f..1a58c7453 100644 --- a/tests/app/main/views/uploads/test_upload_contact_list.py +++ b/tests/app/main/views/uploads/test_upload_contact_list.py @@ -208,7 +208,7 @@ def test_upload_csv_file_shows_error_banner( mock_upload.assert_called_once_with( filedata='', region='us-west-2', - bucket_name='local-contact-list', + bucket_name='test-contact-list', file_location=f"service-{SERVICE_ONE_ID}-notify/{fake_uuid}.csv", access_key=default_access_key, secret_key=default_secret_key, diff --git a/tests/app/s3_client/test_s3_csv_client.py b/tests/app/s3_client/test_s3_csv_client.py index b2fc04341..4a099d73d 100644 --- a/tests/app/s3_client/test_s3_csv_client.py +++ b/tests/app/s3_client/test_s3_csv_client.py @@ -4,7 +4,7 @@ from app.s3_client.s3_csv_client import set_metadata_on_csv_upload def test_sets_metadata(client_request, mocker): - mocked_s3_object = Mock(bucket_name='local-notifications-csv-upload', key='service-1234-notify/5678.csv') + mocked_s3_object = Mock(bucket_name='test-csv-upload', key='service-1234-notify/5678.csv') mocked_get_s3_object = mocker.patch( 'app.s3_client.s3_csv_client.get_csv_upload', return_value=mocked_s3_object, @@ -14,7 +14,7 @@ def test_sets_metadata(client_request, mocker): mocked_get_s3_object.assert_called_once_with('1234', '5678') mocked_s3_object.copy_from.assert_called_once_with( - CopySource='local-notifications-csv-upload/service-1234-notify/5678.csv', + CopySource='test-csv-upload/service-1234-notify/5678.csv', Metadata={'baz': 'True', 'foo': 'bar'}, MetadataDirective='REPLACE', ServerSideEncryption='AES256',