Merge pull request #1758 from alphagov/generate-manifest-env

Add generate_manifest.py script and update deployment tasks

Makefile

@@ -25,6 +25,9 @@ CF_SPACE ?= ${DEPLOY_ENV}
 CF_HOME ?= ${HOME}
 $(eval export CF_HOME)
 
+CF_MANIFEST_FILE ?= manifest-${CF_SPACE}.yml
+NOTIFY_CREDENTIALS ?= ~/.notify-credentials
+
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -167,12 +170,21 @@ cf-login: ## Log in to Cloud Foundry
 	@echo "Logging in to Cloud Foundry on ${CF_API}"
 	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
 
+.PHONY: generate-manifest
+generate-manifest:
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
+	$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
+
+	@./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \
+	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg)
+
 .PHONY: cf-deploy
 cf-deploy: ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	@cf app --guid notify-admin || exit 1
 	cf rename notify-admin notify-admin-rollback
-	cf push -f manifest-${CF_SPACE}.yml
+	cf push -f <(make -s generate-manifest)
 	cf scale -i $$(cf curl /v2/apps/$$(cf app --guid notify-admin-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") notify-admin
 	cf stop notify-admin-rollback
 	cf delete -f notify-admin-rollback
@@ -180,7 +192,7 @@ cf-deploy: ## Deploys the app to Cloud Foundry
 .PHONY: cf-deploy-prototype
 cf-deploy-prototype: cf-target ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
-	cf push -f manifest-prototype-${CF_SPACE}.yml
+	cf push -f <(make -s CF_MANIFEST_FILE=manifest-prototype-${CF_SPACE}.yml generate-manifest)
 
 .PHONY: cf-rollback
 cf-rollback: ## Rollbacks the app to the previous release
@@ -191,7 +203,7 @@ cf-rollback: ## Rollbacks the app to the previous release
 
 .PHONY: cf-push
 cf-push:
-	cf push -f manifest-${CF_SPACE}.yml
+	cf push -f <(make -s generate-manifest)
 
 .PHONY: cf-target
 cf-target: check-env-vars

app/cloudfoundry_config.py

@@ -8,53 +8,6 @@ import json
 
 
 def extract_cloudfoundry_config():
-    vcap_services = json.loads(os.environ['VCAP_SERVICES'])
-
-    set_config_env_vars(vcap_services)
-
-
-def set_config_env_vars(vcap_services):
     vcap_application = json.loads(os.environ.get('VCAP_APPLICATION'))
     os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
     os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'
-
-    for s in vcap_services['user-provided']:
-        if s['name'] == 'notify-config':
-            extract_notify_config(s)
-        elif s['name'] == 'notify-aws':
-            extract_notify_aws_config(s)
-        elif s['name'] == 'hosted-graphite':
-            extract_hosted_graphite_config(s)
-        elif s['name'] == 'deskpro':
-            extract_deskpro_config(s)
-        elif s['name'] == 'notify-template-preview':
-            extract_template_preview_config(s)
-
-
-def extract_notify_config(notify_config):
-    os.environ['ADMIN_CLIENT_SECRET'] = notify_config['credentials']['admin_client_secret']
-    os.environ['API_HOST_NAME'] = notify_config['credentials']['api_host_name']
-    os.environ['ADMIN_BASE_URL'] = notify_config['credentials']['admin_base_url']
-    os.environ['SECRET_KEY'] = notify_config['credentials']['secret_key']
-    os.environ['DANGEROUS_SALT'] = notify_config['credentials']['dangerous_salt']
-    os.environ['ROUTE_SECRET_KEY_1'] = notify_config['credentials']['route_secret_key_1']
-    os.environ['ROUTE_SECRET_KEY_2'] = notify_config['credentials']['route_secret_key_2']
-
-
-def extract_notify_aws_config(aws_config):
-    os.environ['AWS_ACCESS_KEY_ID'] = aws_config['credentials']['aws_access_key_id']
-    os.environ['AWS_SECRET_ACCESS_KEY'] = aws_config['credentials']['aws_secret_access_key']
-
-
-def extract_hosted_graphite_config(hosted_graphite_config):
-    os.environ['STATSD_PREFIX'] = hosted_graphite_config['credentials']['statsd_prefix']
-
-
-def extract_deskpro_config(deskpro_config):
-    os.environ['DESKPRO_API_HOST'] = deskpro_config['credentials']['api_host']
-    os.environ['DESKPRO_API_KEY'] = deskpro_config['credentials']['api_key']
-
-
-def extract_template_preview_config(template_preview_config):
-    os.environ['TEMPLATE_PREVIEW_API_HOST'] = template_preview_config['credentials']['api_host']
-    os.environ['TEMPLATE_PREVIEW_API_KEY'] = template_preview_config['credentials']['api_key']

app/config.py

@@ -1,8 +1,8 @@
 import os
 
 
-if os.environ.get('VCAP_SERVICES'):
+if os.environ.get('VCAP_APPLICATION'):
-    # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
+    # on cloudfoundry, config is a json blob in VCAP_APPLICATION - unpack it, and populate
     # standard environment variables from it
     from app.cloudfoundry_config import extract_cloudfoundry_config
     extract_cloudfoundry_config()

manifest-base.yml

@@ -2,16 +2,30 @@
 
 buildpack: python_buildpack
 command: scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py --error-logfile /home/vcap/logs/gunicorn_error.log -w 5 -b 0.0.0.0:$PORT application
-services:
-  - notify-aws
-  - notify-config
-  - notify-template-preview
-  - hosted-graphite
-  - deskpro
 instances: 1
 memory: 1G
 env:
   NOTIFY_APP_NAME: admin
 
+  # Credentials variables
+  ADMIN_CLIENT_SECRET: null
+  ADMIN_BASE_URL: null
+  API_HOST_NAME: null
+  DANGEROUS_SALT: null
+  SECRET_KEY: null
+  ROUTE_SECRET_KEY_1: null
+  ROUTE_SECRET_KEY_2: null
+
+  AWS_ACCESS_KEY_ID: null
+  AWS_SECRET_ACCESS_KEY: null
+
+  STATSD_PREFIX: null
+
+  DESKPRO_API_HOST: null
+  DESKPRO_API_KEY: null
+
+  TEMPLATE_PREVIEW_API_HOST: null
+  TEMPLATE_PREVIEW_API_KEY: null
+
 applications:
   - name: notify-admin

manifest-preview.yml

@@ -3,11 +3,6 @@
 inherit: manifest-base.yml
 
 services:
-  - notify-aws
-  - notify-config
-  - notify-template-preview
-  - hosted-graphite
-  - deskpro
   - logit-ssl-syslog-drain
 
 routes:

manifest-prototype-base.yml

@@ -2,16 +2,30 @@
 
 buildpack: python_buildpack
 command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT application
-services:
-  - notify-aws
-  - notify-config
-  - notify-template-preview
-  - hosted-graphite
-  - deskpro
 instances: 1
 memory: 1G
 env:
   NOTIFY_APP_NAME: admin
 
+  # Credentials variables
+  ADMIN_CLIENT_SECRET: null
+  ADMIN_BASE_URL: null
+  API_HOST_NAME: null
+  DANGEROUS_SALT: null
+  SECRET_KEY: null
+  ROUTE_SECRET_KEY_1: null
+  ROUTE_SECRET_KEY_2: null
+
+  AWS_ACCESS_KEY_ID: null
+  AWS_SECRET_ACCESS_KEY: null
+
+  STATSD_PREFIX: null
+
+  DESKPRO_API_HOST: null
+  DESKPRO_API_KEY: null
+
+  TEMPLATE_PREVIEW_API_HOST: null
+  TEMPLATE_PREVIEW_API_KEY: null
+
 applications:
   - name: notify-admin-prototype

manifest-staging.yml

@@ -7,11 +7,6 @@ routes:
   - route: www.staging-notify.works
 
 services:
-  - notify-aws
-  - notify-config
-  - notify-template-preview
-  - hosted-graphite
-  - deskpro
   - logit-ssl-syslog-drain
 
 instances: 2

scripts/generate_manifest.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+import json
+import yaml
+
+
+def merge_dicts(a, b):
+    if not (isinstance(a, dict) and isinstance(b, dict)):
+        raise ValueError("Error merging variables: '{}' and '{}'".format(
+            type(a).__name__, type(b).__name__
+        ))
+
+    result = a.copy()
+    for key, val in b.items():
+        if isinstance(result.get(key), dict):
+            result[key] = merge_dicts(a[key], b[key])
+        else:
+            result[key] = val
+
+    return result
+
+
+def load_manifest(manifest_file):
+    with open(manifest_file) as f:
+        manifest = yaml.load(f)
+
+    if 'inherit' in manifest:
+        inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit'))
+        manifest = merge_dicts(load_manifest(inherit_file), manifest)
+
+    return manifest
+
+
+def load_variables(vars_files):
+    variables = {}
+    for vars_file in vars_files:
+        with open(vars_file) as f:
+            variables = merge_dicts(variables, yaml.load(f))
+
+    return {
+        k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v
+        for k, v in variables.items()
+    }
+
+
+def paas_manifest(manifest_file, *vars_files):
+    manifest = load_manifest(manifest_file)
+    variables = load_variables(vars_files)
+
+    for key in manifest.get('env', {}):
+        if key in variables:
+            manifest['env'][key] = variables[key]
+
+    return yaml.dump(manifest, default_flow_style=False, allow_unicode=True)
+
+
+if __name__ == "__main__":
+    print('---')  # noqa
+    print(paas_manifest(*sys.argv[1:]))  # noqa

scripts/run_app_paas.sh

@@ -18,9 +18,6 @@ function check_params {
 function configure_aws_logs {
   aws configure set plugins.cwlogs cwlogs
 
-  export AWS_ACCESS_KEY_ID=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_access_key_id')
-  export AWS_SECRET_ACCESS_KEY=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_secret_access_key')
-
   cat > /home/vcap/app/awslogs.conf << EOF
 [general]
 state_file = /home/vcap/logs/awslogs-state

tests/app/test_cloudfoundry_config.py

@@ -1,92 +1,12 @@
 import os
-import json
 
 import pytest
 
-from app.cloudfoundry_config import extract_cloudfoundry_config, set_config_env_vars
+from app.cloudfoundry_config import extract_cloudfoundry_config
 
 
 @pytest.fixture
-def notify_config():
+def cloudfoundry_environ(monkeypatch):
-    return {
-        'name': 'notify-config',
-        'credentials': {
-            'api_host_name': 'api host name',
-            'admin_base_url': 'admin base url',
-            'admin_client_secret': 'admin client secret',
-            'secret_key': 'secret key',
-            'dangerous_salt': 'dangerous salt',
-            'route_secret_key_1': 'key 1',
-            'route_secret_key_2': 'key 2',
-        }
-    }
-
-
-@pytest.fixture
-def aws_config():
-    return {
-        'name': 'notify-aws',
-        'credentials': {
-            'aws_access_key_id': 'aws access key id',
-            'aws_secret_access_key': 'aws secret access key',
-        }
-    }
-
-
-@pytest.fixture
-def hosted_graphite_config():
-    return {
-        'name': 'hosted-graphite',
-        'credentials': {
-            'statsd_prefix': 'statsd prefix'
-        }
-    }
-
-
-@pytest.fixture
-def deskpro_config():
-    return {
-        'name': 'deskpro',
-        'credentials': {
-            'api_host': 'deskpro api host',
-            'api_key': 'deskpro api key'
-        }
-    }
-
-
-@pytest.fixture
-def template_preview_config():
-    return {
-        'name': 'notify-template-preview',
-        'credentials': {
-            'api_host': 'template-preview api host',
-            'api_key': 'template-preview api key'
-        }
-    }
-
-
-@pytest.fixture
-def cloudfoundry_config(
-        notify_config,
-        aws_config,
-        hosted_graphite_config,
-        deskpro_config,
-        template_preview_config,
-):
-    return {
-        'user-provided': [
-            notify_config,
-            aws_config,
-            hosted_graphite_config,
-            deskpro_config,
-            template_preview_config,
-        ]
-    }
-
-
-@pytest.fixture
-def cloudfoundry_environ(monkeypatch, cloudfoundry_config):
-    monkeypatch.setenv('VCAP_SERVICES', json.dumps(cloudfoundry_config))
     monkeypatch.setenv('VCAP_APPLICATION', '{"space_name":"🚀🌌"}')
 
 
@@ -96,60 +16,3 @@ def test_extract_cloudfoundry_config_populates_other_vars():
 
     assert os.environ['NOTIFY_ENVIRONMENT'] == '🚀🌌'
     assert os.environ['NOTIFY_LOG_PATH'] == '/home/vcap/logs/app.log'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config):
-    cloudfoundry_config['foo'] = {'credentials': {'foo': 'foo'}}
-    cloudfoundry_config['user-provided'].append({
-        'name': 'bar', 'credentials': {'bar': 'bar'}
-    })
-
-    set_config_env_vars(cloudfoundry_config)
-
-    assert 'foo' not in os.environ
-    assert 'bar' not in os.environ
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_notify_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['API_HOST_NAME'] == 'api host name'
-    assert os.environ['ADMIN_BASE_URL'] == 'admin base url'
-    assert os.environ['ADMIN_CLIENT_SECRET'] == 'admin client secret'
-    assert os.environ['SECRET_KEY'] == 'secret key'
-    assert os.environ['DANGEROUS_SALT'] == 'dangerous salt'
-    assert os.environ['ROUTE_SECRET_KEY_1'] == 'key 1'
-    assert os.environ['ROUTE_SECRET_KEY_2'] == 'key 2'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_aws_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['AWS_ACCESS_KEY_ID'] == 'aws access key id'
-    assert os.environ['AWS_SECRET_ACCESS_KEY'] == 'aws secret access key'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_hosted_graphite_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['STATSD_PREFIX'] == 'statsd prefix'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_deskpro_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['DESKPRO_API_HOST'] == 'deskpro api host'
-    assert os.environ['DESKPRO_API_KEY'] == 'deskpro api key'
-
-
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_template_preview_config():
-    extract_cloudfoundry_config()
-
-    assert os.environ['TEMPLATE_PREVIEW_API_HOST'] == 'template-preview api host'
-    assert os.environ['TEMPLATE_PREVIEW_API_KEY'] == 'template-preview api key'

tests/app/test_config.py

@@ -26,7 +26,7 @@ def reload_config():
 
 def test_load_cloudfoundry_config_if_available(monkeypatch, reload_config):
     os.environ['API_HOST_NAME'] = 'env'
-    monkeypatch.setenv('VCAP_SERVICES', 'some json blob')
+    monkeypatch.setenv('VCAP_APPLICATION', 'some json blob')
 
     with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config:
         # reload config so that its module level code (ie: all of it) is re-instantiated
@@ -41,7 +41,7 @@ def test_load_cloudfoundry_config_if_available(monkeypatch, reload_config):
 def test_load_config_if_cloudfoundry_not_available(monkeypatch, reload_config):
     os.environ['API_HOST_NAME'] = 'env'
 
-    monkeypatch.delenv('VCAP_SERVICES', raising=False)
+    monkeypatch.delenv('VCAP_APPLICATION', raising=False)
 
     with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config:
         # reload config so that its module level code (ie: all of it) is re-instantiated