Connect to s3 buckets created by notifications-api

app/cloudfoundry_config.py

@@ -2,6 +2,27 @@ import json
 import os
 
 
+def find_by_service_name(services, service_name):
+    for i in range(len(services)):
+        if services[i]['name'] == service_name:
+            return services[i]
+    return None
+
+
 def extract_cloudfoundry_config():
     vcap_services = json.loads(os.environ['VCAP_SERVICES'])
+
+    # Redis config
     os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis', 'rediss')
+
+    # CSV Upload Bucket Name
+    csv_bucket_service = find_by_service_name(
+        vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}")
+    if csv_bucket_service:
+        os.environ['CSV_UPLOAD_BUCKET_NAME'] = csv_bucket_service['credentials']['bucket']
+
+    # Contact List Bucket Name
+    contact_bucket_service = find_by_service_name(
+        vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}")
+    if contact_bucket_service:
+        os.environ['CONTACT_LIST_BUCKET_NAME'] = contact_bucket_service['credentials']['bucket']

app/config.py

@@ -1,13 +1,11 @@
 import json
 import os
 
-if os.environ.get('VCAP_APPLICATION'):
+if os.environ.get('VCAP_SERVICES'):
-    # on cloudfoundry, config is a json blob in VCAP_APPLICATION - unpack it, and populate
+    # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
     # standard environment variables from it
-    # from app.cloudfoundry_config import extract_cloudfoundry_config
+    from app.cloudfoundry_config import extract_cloudfoundry_config
-    # extract_cloudfoundry_config()
+    extract_cloudfoundry_config()
-    vcap_services = json.loads(os.environ['VCAP_SERVICES'])
-    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace("redis", "rediss")
 
 
 class Config(object):
@@ -62,7 +60,7 @@ class Config(object):
     NOTIFY_ENVIRONMENT = 'development'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-local'
     MOU_BUCKET_NAME = 'local-mou'
-    TRANSIENT_UPLOADED_LETTERS = 'local-transient-uploaded-letters'
+    # TRANSIENT_UPLOADED_LETTERS = 'local-transient-uploaded-letters'
     ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1')
     ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2')
     CHECK_PROXY_HEADER = False
@@ -111,8 +109,9 @@ class Development(Config):
     CONTACT_LIST_UPLOAD_BUCKET_NAME = 'local-contact-list'  # created in gsa sandbox
     LOGO_UPLOAD_BUCKET_NAME = 'local-public-logos-tools'  # created in gsa sandbox
     MOU_BUCKET_NAME = 'local-notify-tools-mou'  # created in gsa sandbox
-    TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'  # not created in gsa sandbox
+    # TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'  # not created in gsa sandbox
-    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'development-letters-precompiled-originals-backup'  # not created in sandbox
+    # PRECOMPILED_ORIGINALS_BACKUP_LETTERS =
+    # 'development-letters-precompiled-originals-backup'  # not created in sandbox
 
     ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET')
     # check for local compose orchestration variable
@@ -137,8 +136,8 @@ class Test(Development):
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test'
     LOGO_CDN_DOMAIN = 'static-logos.test.com'
     MOU_BUCKET_NAME = 'test-mou'
-    TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
+    # TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
-    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup'
+    # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup'
     NOTIFY_ENVIRONMENT = 'test'
     API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to'
     REDIS_URL = 'redis://you-forgot-to-mock-a-redis-call-to'
@@ -168,8 +167,8 @@ class Preview(Config):
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview'
     LOGO_CDN_DOMAIN = 'static-logos.notify.works'
     MOU_BUCKET_NAME = 'notify.works-mou'
-    TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
+    # TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
-    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'preview-letters-precompiled-originals-backup'
+    # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'preview-letters-precompiled-originals-backup'
     NOTIFY_ENVIRONMENT = 'preview'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.notify.works'
@@ -188,8 +187,8 @@ class Staging(Config):
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging'
     LOGO_CDN_DOMAIN = 'static-logos.staging-notify.works'
     MOU_BUCKET_NAME = 'staging-notify.works-mou'
-    TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
+    # TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
-    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'staging-letters-precompiled-originals-backup'
+    # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'staging-letters-precompiled-originals-backup'
     NOTIFY_ENVIRONMENT = 'staging'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.staging-notify.works'
@@ -201,12 +200,14 @@ class Live(Config):
     HEADER_COLOUR = '#005EA5'  # $govuk-blue
     HTTP_PROTOCOL = 'https'
     # buckets
-    CSV_UPLOAD_BUCKET_NAME = 'notifications-prototype-csv-upload'  # created in gsa sandbox
+    CSV_UPLOAD_BUCKET_NAME = os.environ.get(
-    CONTACT_LIST_UPLOAD_BUCKET_NAME = 'notifications-prototype-contact-list-upload'  # created in gsa sandbox
+        'CSV_UPLOAD_BUCKET_NAME', 'notifications-prototype-csv-upload')  # created in gsa sandbox
+    CONTACT_LIST_UPLOAD_BUCKET_NAME = os.environ.get(
+        'CONTACT_LIST_BUCKET_NAME', 'notifications-prototype-contact-list-upload')  # created in gsa sandbox
     LOGO_UPLOAD_BUCKET_NAME = 'notifications-prototype-logo-upload'  # created in gsa sandbox
     MOU_BUCKET_NAME = 'notifications-prototype-mou'  # created in gsa sandbox
-    TRANSIENT_UPLOADED_LETTERS = 'prototype-transient-uploaded-letters'  # not created in gsa sandbox
+    # TRANSIENT_UPLOADED_LETTERS = 'prototype-transient-uploaded-letters'  # not created in gsa sandbox
-    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'prototype-letters-precompiled-originals-backup'  # not in sandbox
+    # PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'prototype-letters-precompiled-originals-backup'  # not in sandbox
 
     NOTIFY_ENVIRONMENT = 'live'
     CHECK_PROXY_HEADER = False

manifest.yml

@@ -14,6 +14,8 @@ applications:
       # - notify-prometheus
       # - notify-splunk
       - notifications-admin-redis-((env))
+      - notifications-api-csv-upload-bucket-((env))
+      - notifications-api-contact-list-bucket-((env))
 
     env:
       NOTIFY_APP_NAME: admin

tests/app/s3_client/test_s3_letter_upload_client.py

@@ -15,6 +15,8 @@ from app.s3_client.s3_letter_upload_client import (
     upload_letter_to_s3,
 )
 
+pytest.skip(reason="Skipping letter-related features", allow_module_level=True)
+
 
 def test_backup_original_letter_to_s3(mocker, notify_admin):
     s3_mock = mocker.patch('app.s3_client.s3_letter_upload_client.utils_s3upload')

tests/app/test_cloudfoundry_config.py

@@ -14,11 +14,28 @@ def vcap_services():
                 'uri': 'redis://xxx:6379'
             }
         }],
+        's3': [
+            {
+                'name': 'notifications-api-csv-upload-bucket-test',
+                'credentials': {
+                    'bucket': 'csv-upload-bucket'
+                }
+            },
+            {
+                'name': 'notifications-api-contact-list-bucket-test',
+                'credentials': {
+                    'bucket': 'contact-list-bucket'
+                }
+            }
+        ],
     }
 
 
 def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services):
+    os.environ['DEPLOY_ENV'] = 'test'
     os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
     extract_cloudfoundry_config()
 
     assert os.environ['REDIS_URL'] == 'rediss://xxx:6379'
+    assert os.environ['CSV_UPLOAD_BUCKET_NAME'] == 'csv-upload-bucket'
+    assert os.environ['CONTACT_LIST_BUCKET_NAME'] == 'contact-list-bucket'

tests/app/test_config.py

@@ -1,5 +1,4 @@
 import importlib
-import json
 import os
 from unittest import mock
 
@@ -9,7 +8,7 @@ from app import config
 
 
 def cf_conf():
-    os.environ['API_HOST_NAME'] = 'cf'
+    os.environ['REDIS_URL'] = 'rediss://xxx:6379'
 
 
 @pytest.fixture
@@ -29,16 +28,8 @@ def reload_config(os_environ):
 
 
 def test_load_cloudfoundry_config_if_available(reload_config):
-    os.environ['API_HOST_NAME'] = 'env'
     os.environ['REDIS_URL'] = 'some uri'
-    os.environ['VCAP_APPLICATION'] = 'some json blob'
+    os.environ['VCAP_SERVICES'] = 'some json blob'
-    os.environ['VCAP_SERVICES'] = json.dumps({
-        'aws-elasticache-redis': [{
-            'credentials': {
-                'uri': 'redis://xxx:6379'
-            }
-        }],
-    })
 
     with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf):
         # reload config so that its module level code (ie: all of it) is re-instantiated
@@ -49,9 +40,8 @@ def test_load_cloudfoundry_config_if_available(reload_config):
 
 
 def test_load_config_if_cloudfoundry_not_available(reload_config):
-    os.environ['API_HOST_NAME'] = 'env'
+    os.environ['REDIS_URL'] = 'redis://xxx:6379'
-
+    os.environ.pop('VCAP_SERVICES', None)
-    os.environ.pop('VCAP_APPLICATION', None)
 
     with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config:
         # reload config so that its module level code (ie: all of it) is re-instantiated
@@ -59,5 +49,5 @@ def test_load_config_if_cloudfoundry_not_available(reload_config):
 
     assert not cf_config.called
 
-    assert os.environ['API_HOST_NAME'] == 'env'
+    assert os.environ['REDIS_URL'] == 'redis://xxx:6379'
-    assert config.Config.API_HOST_NAME == 'env'
+    assert config.Config.REDIS_URL == 'redis://xxx:6379'