Merge pull request #1543 from alphagov/generate-manifest-env

Replaces user-provided services with manifest environment variables

Makefile

@@ -27,6 +27,8 @@ $(eval export CF_HOME)
 
 CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml
 
+NOTIFY_CREDENTIALS ?= ~/.notify-credentials
+
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -188,13 +190,23 @@ cf-login: ## Log in to Cloud Foundry
 	@echo "Logging in to Cloud Foundry on ${CF_API}"
 	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
 
+.PHONY: generate-manifest
+generate-manifest:
+	$(if ${CF_APP},,$(error Must specify CF_APP))
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
+	$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
+
+	@./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \
+	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg)
+
 .PHONY: cf-deploy
 cf-deploy: ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	$(if ${CF_APP},,$(error Must specify CF_APP))
 	@cf app --guid ${CF_APP} || exit 1
 	cf rename ${CF_APP} ${CF_APP}-rollback
-	cf push ${CF_APP} -f ${CF_MANIFEST_FILE}
+	cf push ${CF_APP} -f <(make -s generate-manifest)
 	cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP}
 	cf stop ${CF_APP}-rollback
 	# sleep for 10 seconds to try and make sure that all worker threads (either web api or celery) have finished before we delete
@@ -211,7 +223,7 @@ cf-deploy-api-db-migration:
 	cf unbind-service notify-api-db-migration notify-db
 	cf unbind-service notify-api-db-migration notify-config
 	cf unbind-service notify-api-db-migration notify-aws
-	cf push notify-api-db-migration -f manifest-api-${CF_SPACE}.yml
+	cf push notify-api-db-migration -f -f <(make -s CF_APP=api generate-manifest)
 	cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
 
 .PHONY: cf-check-api-db-migration-task
@@ -230,7 +242,7 @@ cf-rollback: ## Rollbacks the app to the previous release
 cf-push:
 	$(if ${CF_APP},,$(error Must specify CF_APP))
 	cf target -o ${CF_ORG} -s ${CF_SPACE}
-	cf push ${CF_APP} -f ${CF_MANIFEST_FILE}
+	cf push ${CF_APP} -f <(make -s generate-manifest)
 
 .PHONY: check-if-migrations-to-run
 check-if-migrations-to-run:

app/cloudfoundry_config.py

@@ -19,70 +19,3 @@ def set_config_env_vars(vcap_services):
     vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
     os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
     os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'
-
-    # Notify common config
-    for s in vcap_services['user-provided']:
-        if s['name'] == 'notify-config':
-            extract_notify_config(s)
-        elif s['name'] == 'notify-aws':
-            extract_notify_aws_config(s)
-        elif s['name'] == 'hosted-graphite':
-            extract_hosted_graphite_config(s)
-        elif s['name'] == 'mmg':
-            extract_mmg_config(s)
-        elif s['name'] == 'firetext':
-            extract_firetext_config(s)
-        elif s['name'] == 'redis':
-            extract_redis_config(s)
-        elif s['name'] == 'performance-platform':
-            extract_performance_platform_config(s)
-        elif s['name'] == 'notify-template-preview':
-            extract_template_preview_config(s)
-
-
-def extract_notify_config(notify_config):
-    os.environ['ADMIN_BASE_URL'] = notify_config['credentials']['admin_base_url']
-    os.environ['API_HOST_NAME'] = notify_config['credentials']['api_host_name']
-    os.environ['ADMIN_CLIENT_SECRET'] = notify_config['credentials']['admin_client_secret']
-    os.environ['SECRET_KEY'] = notify_config['credentials']['secret_key']
-    os.environ['DANGEROUS_SALT'] = notify_config['credentials']['dangerous_salt']
-    os.environ['SMS_INBOUND_WHITELIST'] = json.dumps(notify_config['credentials']['allow_ip_inbound_sms'])
-    os.environ['FIRETEXT_INBOUND_SMS_AUTH'] = json.dumps(notify_config['credentials']['firetext_inbound_sms_auth'])
-    os.environ['MMG_INBOUND_SMS_AUTH'] = json.dumps(notify_config['credentials']['mmg_inbound_sms_auth'])
-    os.environ['MMG_INBOUND_SMS_USERNAME'] = json.dumps(notify_config['credentials']['mmg_inbound_sms_username'])
-    os.environ['ROUTE_SECRET_KEY_1'] = notify_config['credentials']['route_secret_key_1']
-    os.environ['ROUTE_SECRET_KEY_2'] = notify_config['credentials']['route_secret_key_2']
-
-
-def extract_performance_platform_config(performance_platform_config):
-    os.environ['PERFORMANCE_PLATFORM_ENDPOINTS'] = json.dumps(performance_platform_config['credentials'])
-
-
-def extract_notify_aws_config(aws_config):
-    os.environ['NOTIFICATION_QUEUE_PREFIX'] = aws_config['credentials']['sqs_queue_prefix']
-    os.environ['AWS_ACCESS_KEY_ID'] = aws_config['credentials']['aws_access_key_id']
-    os.environ['AWS_SECRET_ACCESS_KEY'] = aws_config['credentials']['aws_secret_access_key']
-
-
-def extract_hosted_graphite_config(hosted_graphite_config):
-    os.environ['STATSD_PREFIX'] = hosted_graphite_config['credentials']['statsd_prefix']
-
-
-def extract_mmg_config(mmg_config):
-    os.environ['MMG_URL'] = mmg_config['credentials']['api_url']
-    os.environ['MMG_API_KEY'] = mmg_config['credentials']['api_key']
-
-
-def extract_firetext_config(firetext_config):
-    os.environ['FIRETEXT_API_KEY'] = firetext_config['credentials']['api_key']
-    os.environ['LOADTESTING_API_KEY'] = firetext_config['credentials']['loadtesting_api_key']
-
-
-def extract_redis_config(redis_config):
-    os.environ['REDIS_ENABLED'] = redis_config['credentials']['redis_enabled']
-    os.environ['REDIS_URL'] = redis_config['credentials']['redis_url']
-
-
-def extract_template_preview_config(template_preview_config):
-    os.environ['TEMPLATE_PREVIEW_API_HOST'] = template_preview_config['credentials']['api_host']
-    os.environ['TEMPLATE_PREVIEW_API_KEY'] = template_preview_config['credentials']['api_key']

app/config.py

@@ -103,7 +103,6 @@ class Config(object):
     # Performance platform
     PERFORMANCE_PLATFORM_ENABLED = False
     PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/'
-    PERFORMANCE_PLATFORM_TOKEN = os.getenv('PERFORMANCE_PLATFORM_TOKEN')
 
     # Logging
     DEBUG = False

manifest-api-base.yml

@@ -3,19 +3,45 @@
 buildpack: python_buildpack
 command: scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py --error-logfile /home/vcap/logs/gunicorn_error.log -w 5 -b 0.0.0.0:$PORT application
 services:
-  - notify-aws
-  - notify-config
   - notify-db
-  - mmg
-  - firetext
-  - hosted-graphite
-  - redis
-  - performance-platform
 env:
   NOTIFY_APP_NAME: public-api
   CW_APP_NAME: api
   # required by cf run-task
   FLASK_APP: application.py
+
+  # Credentials variables
+  ADMIN_BASE_URL: null
+  ADMIN_CLIENT_SECRET: null
+  API_HOST_NAME: null
+  DANGEROUS_SALT: null
+  SECRET_KEY: null
+  ROUTE_SECRET_KEY_1: null
+  ROUTE_SECRET_KEY_2: null
+
+  PERFORMANCE_PLATFORM_ENDPOINTS: null
+
+  NOTIFICATION_QUEUE_PREFIX: null
+  AWS_ACCESS_KEY_ID: null
+  AWS_SECRET_ACCESS_KEY: null
+
+  STATSD_PREFIX: null
+
+  MMG_URL: null
+  MMG_API_KEY: null
+  MMG_INBOUND_SMS_AUTH: null
+  MMG_INBOUND_SMS_USERNAME: null
+
+  FIRETEXT_API_KEY: null
+  LOADTESTING_API_KEY: null
+  FIRETEXT_INBOUND_SMS_AUTH: null
+
+  REDIS_ENABLED: null
+  REDIS_URL: null
+
+  TEMPLATE_PREVIEW_API_HOST: null
+  TEMPLATE_PREVIEW_API_KEY: null
+
 instances: 1
 memory: 1G
 

manifest-api-preview.yml

@@ -3,14 +3,7 @@
 inherit: manifest-api-base.yml
 
 services:
-  - notify-aws
-  - notify-config
   - notify-db
-  - mmg
-  - firetext
-  - hosted-graphite
-  - redis
-  - performance-platform
   - logit-ssl-syslog-drain
 
 routes:

manifest-api-staging.yml

@@ -2,14 +2,7 @@
 
 inherit: manifest-api-base.yml
 services:
-  - notify-aws
-  - notify-config
   - notify-db
-  - mmg
-  - firetext
-  - hosted-graphite
-  - redis
-  - performance-platform
   - logit-ssl-syslog-drain
 
 routes:

manifest-delivery-base.yml

@@ -4,18 +4,43 @@ buildpack: python_buildpack
 health-check-type: none
 no-route: true
 services:
-  - notify-aws
-  - notify-config
   - notify-db
-  - notify-template-preview
-  - mmg
-  - firetext
-  - hosted-graphite
-  - redis
-  - performance-platform
 instances: 1
 memory: 1G
 
+env:
+  # Credentials variables
+  ADMIN_BASE_URL: null
+  ADMIN_CLIENT_SECRET: null
+  API_HOST_NAME: null
+  DANGEROUS_SALT: null
+  SECRET_KEY: null
+  ROUTE_SECRET_KEY_1: null
+  ROUTE_SECRET_KEY_2: null
+
+  PERFORMANCE_PLATFORM_ENDPOINTS: null
+
+  NOTIFICATION_QUEUE_PREFIX: null
+  AWS_ACCESS_KEY_ID: null
+  AWS_SECRET_ACCESS_KEY: null
+
+  STATSD_PREFIX: null
+
+  MMG_URL: null
+  MMG_API_KEY: null
+  MMG_INBOUND_SMS_AUTH: null
+  MMG_INBOUND_SMS_USERNAME: null
+
+  FIRETEXT_API_KEY: null
+  LOADTESTING_API_KEY: null
+  FIRETEXT_INBOUND_SMS_AUTH: null
+
+  REDIS_ENABLED: null
+  REDIS_URL: null
+
+  TEMPLATE_PREVIEW_API_HOST: null
+  TEMPLATE_PREVIEW_API_KEY: null
+
 applications:
   - name: notify-delivery-celery-beat
     command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO

manifest-delivery-preview.yml

@@ -3,15 +3,7 @@
 inherit: manifest-delivery-base.yml
 
 services:
-  - notify-aws
-  - notify-config
   - notify-db
-  - notify-template-preview
-  - mmg
-  - firetext
-  - hosted-graphite
-  - redis
-  - performance-platform
   - logit-ssl-syslog-drain
 
 memory: 1G

manifest-delivery-staging.yml

@@ -3,15 +3,7 @@
 inherit: manifest-delivery-base.yml
 
 services:
-  - notify-aws
-  - notify-config
   - notify-db
-  - notify-template-preview
-  - mmg
-  - firetext
-  - hosted-graphite
-  - redis
-  - performance-platform
   - logit-ssl-syslog-drain
 
 instances: 2

scripts/generate_manifest.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+import json
+import yaml
+
+
+def merge_dicts(a, b):
+    if not (isinstance(a, dict) and isinstance(b, dict)):
+        raise ValueError("Error merging variables: '{}' and '{}'".format(
+            type(a).__name__, type(b).__name__
+        ))
+
+    result = a.copy()
+    for key, val in b.items():
+        if isinstance(result.get(key), dict):
+            result[key] = merge_dicts(a[key], b[key])
+        else:
+            result[key] = val
+
+    return result
+
+
+def load_manifest(manifest_file):
+    with open(manifest_file) as f:
+        manifest = yaml.load(f)
+
+    if 'inherit' in manifest:
+        inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit'))
+        manifest = merge_dicts(load_manifest(inherit_file), manifest)
+
+    return manifest
+
+
+def load_variables(vars_files):
+    variables = {}
+    for vars_file in vars_files:
+        with open(vars_file) as f:
+            variables = merge_dicts(variables, yaml.load(f))
+
+    return {
+        k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v
+        for k, v in variables.items()
+    }
+
+
+def paas_manifest(manifest_file, *vars_files):
+    """Generate a PaaS manifest file from a Jinja2 template"""
+
+    manifest = load_manifest(manifest_file)
+    variables = load_variables(vars_files)
+
+    for key in manifest.get('env', {}):
+        if key in variables:
+            manifest['env'][key] = variables[key]
+
+    return yaml.dump(manifest, default_flow_style=False, allow_unicode=True)
+
+
+if __name__ == "__main__":
+    print('---')
+    print(paas_manifest(*sys.argv[1:]))

scripts/run_app_paas.sh

@@ -22,9 +22,6 @@ function configure_aws_logs {
 
   aws configure set plugins.cwlogs cwlogs
 
-  export AWS_ACCESS_KEY_ID=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_access_key_id')
-  export AWS_SECRET_ACCESS_KEY=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_secret_access_key')
-
   cat > /home/vcap/app/awslogs.conf << EOF
 [general]
 state_file = /home/vcap/logs/awslogs-state

tests/app/test_cloudfoundry_config.py

@@ -6,70 +6,6 @@ import pytest
 from app.cloudfoundry_config import extract_cloudfoundry_config, set_config_env_vars
 
 
-@pytest.fixture
-def notify_config():
-    return {
-        'name': 'notify-config',
-        'credentials': {
-            'admin_base_url': 'admin base url',
-            'api_host_name': 'api host name',
-            'admin_client_secret': 'admin client secret',
-            'secret_key': 'secret key',
-            'dangerous_salt': 'dangerous salt',
-            'allow_ip_inbound_sms': ['111.111.111.111', '100.100.100.100'],
-            'firetext_inbound_sms_auth': ['testkey'],
-            'mmg_inbound_sms_auth': ['testkey'],
-            'mmg_inbound_sms_username': ['username'],
-            'route_secret_key_1': "key_1",
-            'route_secret_key_2': ""
-        }
-    }
-
-
-@pytest.fixture
-def aws_config():
-    return {
-        'name': 'notify-aws',
-        'credentials': {
-            'sqs_queue_prefix': 'sqs queue prefix',
-            'aws_access_key_id': 'aws access key id',
-            'aws_secret_access_key': 'aws secret access key',
-        }
-    }
-
-
-@pytest.fixture
-def hosted_graphite_config():
-    return {
-        'name': 'hosted-graphite',
-        'credentials': {
-            'statsd_prefix': 'statsd prefix'
-        }
-    }
-
-
-@pytest.fixture
-def mmg_config():
-    return {
-        'name': 'mmg',
-        'credentials': {
-            'api_url': 'mmg api url',
-            'api_key': 'mmg api key'
-        }
-    }
-
-
-@pytest.fixture
-def firetext_config():
-    return {
-        'name': 'firetext',
-        'credentials': {
-            'api_key': 'firetext api key',
-            'loadtesting_api_key': 'loadtesting api key'
-        }
-    }
-
-
 @pytest.fixture
 def postgres_config():
     return [
@@ -82,62 +18,10 @@ def postgres_config():
 
 
 @pytest.fixture
-def redis_config():
+def cloudfoundry_config(postgres_config):
-    return {
-        'name': 'redis',
-        'credentials': {
-            'redis_enabled': '1',
-            'redis_url': 'redis url'
-        }
-    }
-
-
-@pytest.fixture
-def performance_platform_config():
-    return {
-        'name': 'performance-platform',
-        'credentials': {
-            'foo': 'my_token',
-            'bar': 'other_token'
-        }
-    }
-
-
-@pytest.fixture
-def template_preview_config():
-    return {
-        'name': 'notify-template-preview',
-        'credentials': {
-            'api_host': 'template-preview api host',
-            'api_key': 'template-preview api key'
-        }
-    }
-
-
-@pytest.fixture
-def cloudfoundry_config(
-        postgres_config,
-        notify_config,
-        aws_config,
-        hosted_graphite_config,
-        mmg_config,
-        firetext_config,
-        redis_config,
-        performance_platform_config,
-        template_preview_config
-):
     return {
         'postgres': postgres_config,
-        'user-provided': [
+        'user-provided': []
-            notify_config,
-            aws_config,
-            hosted_graphite_config,
-            mmg_config,
-            firetext_config,
-            redis_config,
-            performance_platform_config,
-            template_preview_config
-        ]
     }
 
 
@@ -167,100 +51,3 @@ def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config):
 
     assert 'foo' not in os.environ
     assert 'bar' not in os.environ
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_notify_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['ADMIN_BASE_URL'] == 'admin base url'
-    assert os.environ['API_HOST_NAME'] == 'api host name'
-    assert os.environ['ADMIN_CLIENT_SECRET'] == 'admin client secret'
-    assert os.environ['SECRET_KEY'] == 'secret key'
-    assert os.environ['DANGEROUS_SALT'] == 'dangerous salt'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_aws_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['NOTIFICATION_QUEUE_PREFIX'] == 'sqs queue prefix'
-    assert os.environ['AWS_ACCESS_KEY_ID'] == 'aws access key id'
-    assert os.environ['AWS_SECRET_ACCESS_KEY'] == 'aws secret access key'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_hosted_graphite_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['STATSD_PREFIX'] == 'statsd prefix'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_mmg_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['MMG_URL'] == 'mmg api url'
-    assert os.environ['MMG_API_KEY'] == 'mmg api key'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_firetext_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['FIRETEXT_API_KEY'] == 'firetext api key'
-    assert os.environ['LOADTESTING_API_KEY'] == 'loadtesting api key'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_redis_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['REDIS_ENABLED'] == '1'
-    assert os.environ['REDIS_URL'] == 'redis url'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_sms_inbound_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['SMS_INBOUND_WHITELIST'] == json.dumps(['111.111.111.111', '100.100.100.100'])
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_firetext_inbound_sms_auth_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['FIRETEXT_INBOUND_SMS_AUTH'] == json.dumps(['testkey'])
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_mmg_inbound_sms_auth_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['MMG_INBOUND_SMS_AUTH'] == json.dumps(['testkey'])
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_mmg_inbound_sms_username_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['MMG_INBOUND_SMS_USERNAME'] == json.dumps(['username'])
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_performance_platform_config():
-    extract_cloudfoundry_config()
-
-    assert json.loads(os.environ['PERFORMANCE_PLATFORM_ENDPOINTS']) == {
-        'foo': 'my_token',
-        'bar': 'other_token'
-    }
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_template_preview_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['TEMPLATE_PREVIEW_API_HOST'] == 'template-preview api host'
-    assert os.environ['TEMPLATE_PREVIEW_API_KEY'] == 'template-preview api key'