From 271cf7643da35357fd78de98986474857a67a98d Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Mon, 19 Sep 2022 11:19:38 -0400 Subject: [PATCH] Connect to s3 buckets created by notifications-api --- app/cloudfoundry_config.py | 21 ++++++++++ app/config.py | 39 ++++++++++--------- manifest.yml | 2 + .../s3_client/test_s3_letter_upload_client.py | 2 + tests/app/test_cloudfoundry_config.py | 17 ++++++++ tests/app/test_config.py | 22 +++-------- 6 files changed, 68 insertions(+), 35 deletions(-) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index f493738a5..4716603f7 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -2,6 +2,27 @@ import json import os +def find_by_service_name(services, service_name): + for i in range(len(services)): + if services[i]['name'] == service_name: + return services[i] + return None + + def extract_cloudfoundry_config(): vcap_services = json.loads(os.environ['VCAP_SERVICES']) + + # Redis config os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis', 'rediss') + + # CSV Upload Bucket Name + csv_bucket_service = find_by_service_name( + vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}") + if csv_bucket_service: + os.environ['CSV_UPLOAD_BUCKET_NAME'] = csv_bucket_service['credentials']['bucket'] + + # Contact List Bucket Name + contact_bucket_service = find_by_service_name( + vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}") + if contact_bucket_service: + os.environ['CONTACT_LIST_BUCKET_NAME'] = contact_bucket_service['credentials']['bucket'] diff --git a/app/config.py b/app/config.py index f3f034c85..6c0f74c40 100644 --- a/app/config.py +++ b/app/config.py @@ -1,13 +1,11 @@ import json import os -if os.environ.get('VCAP_APPLICATION'): - # on cloudfoundry, config is a json blob in VCAP_APPLICATION - unpack it, and populate +if os.environ.get('VCAP_SERVICES'): + # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate # standard environment variables from it - # from app.cloudfoundry_config import extract_cloudfoundry_config - # extract_cloudfoundry_config() - vcap_services = json.loads(os.environ['VCAP_SERVICES']) - os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace("redis", "rediss") + from app.cloudfoundry_config import extract_cloudfoundry_config + extract_cloudfoundry_config() class Config(object): @@ -62,7 +60,7 @@ class Config(object): NOTIFY_ENVIRONMENT = 'development' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-local' MOU_BUCKET_NAME = 'local-mou' - TRANSIENT_UPLOADED_LETTERS = 'local-transient-uploaded-letters' + # TRANSIENT_UPLOADED_LETTERS = 'local-transient-uploaded-letters' ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1') ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2') CHECK_PROXY_HEADER = False @@ -111,8 +109,9 @@ class Development(Config): CONTACT_LIST_UPLOAD_BUCKET_NAME = 'local-contact-list' # created in gsa sandbox LOGO_UPLOAD_BUCKET_NAME = 'local-public-logos-tools' # created in gsa sandbox MOU_BUCKET_NAME = 'local-notify-tools-mou' # created in gsa sandbox - TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters' # not created in gsa sandbox - PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'development-letters-precompiled-originals-backup' # not created in sandbox + # TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters' # not created in gsa sandbox + # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = + # 'development-letters-precompiled-originals-backup' # not created in sandbox ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET') # check for local compose orchestration variable @@ -137,8 +136,8 @@ class Test(Development): LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test' LOGO_CDN_DOMAIN = 'static-logos.test.com' MOU_BUCKET_NAME = 'test-mou' - TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters' - PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup' + # TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters' + # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup' NOTIFY_ENVIRONMENT = 'test' API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to' REDIS_URL = 'redis://you-forgot-to-mock-a-redis-call-to' @@ -168,8 +167,8 @@ class Preview(Config): LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview' LOGO_CDN_DOMAIN = 'static-logos.notify.works' MOU_BUCKET_NAME = 'notify.works-mou' - TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters' - PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'preview-letters-precompiled-originals-backup' + # TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters' + # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'preview-letters-precompiled-originals-backup' NOTIFY_ENVIRONMENT = 'preview' CHECK_PROXY_HEADER = False ASSET_DOMAIN = 'static.notify.works' @@ -188,8 +187,8 @@ class Staging(Config): LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging' LOGO_CDN_DOMAIN = 'static-logos.staging-notify.works' MOU_BUCKET_NAME = 'staging-notify.works-mou' - TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters' - PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'staging-letters-precompiled-originals-backup' + # TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters' + # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'staging-letters-precompiled-originals-backup' NOTIFY_ENVIRONMENT = 'staging' CHECK_PROXY_HEADER = False ASSET_DOMAIN = 'static.staging-notify.works' @@ -201,12 +200,14 @@ class Live(Config): HEADER_COLOUR = '#005EA5' # $govuk-blue HTTP_PROTOCOL = 'https' # buckets - CSV_UPLOAD_BUCKET_NAME = 'notifications-prototype-csv-upload' # created in gsa sandbox - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'notifications-prototype-contact-list-upload' # created in gsa sandbox + CSV_UPLOAD_BUCKET_NAME = os.environ.get( + 'CSV_UPLOAD_BUCKET_NAME', 'notifications-prototype-csv-upload') # created in gsa sandbox + CONTACT_LIST_UPLOAD_BUCKET_NAME = os.environ.get( + 'CONTACT_LIST_BUCKET_NAME', 'notifications-prototype-contact-list-upload') # created in gsa sandbox LOGO_UPLOAD_BUCKET_NAME = 'notifications-prototype-logo-upload' # created in gsa sandbox MOU_BUCKET_NAME = 'notifications-prototype-mou' # created in gsa sandbox - TRANSIENT_UPLOADED_LETTERS = 'prototype-transient-uploaded-letters' # not created in gsa sandbox - PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'prototype-letters-precompiled-originals-backup' # not in sandbox + # TRANSIENT_UPLOADED_LETTERS = 'prototype-transient-uploaded-letters' # not created in gsa sandbox + # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'prototype-letters-precompiled-originals-backup' # not in sandbox NOTIFY_ENVIRONMENT = 'live' CHECK_PROXY_HEADER = False diff --git a/manifest.yml b/manifest.yml index b0e658864..af997762d 100644 --- a/manifest.yml +++ b/manifest.yml @@ -14,6 +14,8 @@ applications: # - notify-prometheus # - notify-splunk - notifications-admin-redis-((env)) + - notifications-api-csv-upload-bucket-((env)) + - notifications-api-contact-list-bucket-((env)) env: NOTIFY_APP_NAME: admin diff --git a/tests/app/s3_client/test_s3_letter_upload_client.py b/tests/app/s3_client/test_s3_letter_upload_client.py index e65327572..5934576e6 100644 --- a/tests/app/s3_client/test_s3_letter_upload_client.py +++ b/tests/app/s3_client/test_s3_letter_upload_client.py @@ -15,6 +15,8 @@ from app.s3_client.s3_letter_upload_client import ( upload_letter_to_s3, ) +pytest.skip(reason="Skipping letter-related features", allow_module_level=True) + def test_backup_original_letter_to_s3(mocker, notify_admin): s3_mock = mocker.patch('app.s3_client.s3_letter_upload_client.utils_s3upload') diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index c7ac6f3e9..0ab3bad8c 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -14,11 +14,28 @@ def vcap_services(): 'uri': 'redis://xxx:6379' } }], + 's3': [ + { + 'name': 'notifications-api-csv-upload-bucket-test', + 'credentials': { + 'bucket': 'csv-upload-bucket' + } + }, + { + 'name': 'notifications-api-contact-list-bucket-test', + 'credentials': { + 'bucket': 'contact-list-bucket' + } + } + ], } def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services): + os.environ['DEPLOY_ENV'] = 'test' os.environ['VCAP_SERVICES'] = json.dumps(vcap_services) extract_cloudfoundry_config() assert os.environ['REDIS_URL'] == 'rediss://xxx:6379' + assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket' + assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket' diff --git a/tests/app/test_config.py b/tests/app/test_config.py index 8bfa429eb..152dd5f35 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -1,5 +1,4 @@ import importlib -import json import os from unittest import mock @@ -9,7 +8,7 @@ from app import config def cf_conf(): - os.environ['API_HOST_NAME'] = 'cf' + os.environ['REDIS_URL'] = 'rediss://xxx:6379' @pytest.fixture @@ -29,16 +28,8 @@ def reload_config(os_environ): def test_load_cloudfoundry_config_if_available(reload_config): - os.environ['API_HOST_NAME'] = 'env' os.environ['REDIS_URL'] = 'some uri' - os.environ['VCAP_APPLICATION'] = 'some json blob' - os.environ['VCAP_SERVICES'] = json.dumps({ - 'aws-elasticache-redis': [{ - 'credentials': { - 'uri': 'redis://xxx:6379' - } - }], - }) + os.environ['VCAP_SERVICES'] = 'some json blob' with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf): # reload config so that its module level code (ie: all of it) is re-instantiated @@ -49,9 +40,8 @@ def test_load_cloudfoundry_config_if_available(reload_config): def test_load_config_if_cloudfoundry_not_available(reload_config): - os.environ['API_HOST_NAME'] = 'env' - - os.environ.pop('VCAP_APPLICATION', None) + os.environ['REDIS_URL'] = 'redis://xxx:6379' + os.environ.pop('VCAP_SERVICES', None) with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config: # reload config so that its module level code (ie: all of it) is re-instantiated @@ -59,5 +49,5 @@ def test_load_config_if_cloudfoundry_not_available(reload_config): assert not cf_config.called - assert os.environ['API_HOST_NAME'] == 'env' - assert config.Config.API_HOST_NAME == 'env' + assert os.environ['REDIS_URL'] == 'redis://xxx:6379' + assert config.Config.REDIS_URL == 'redis://xxx:6379'