Add generate_manifest.py script and update deployment tasks

Brings in the new environment variables deployment process introduced
in alphagov/notifications-api#1543.

The script is a copy of the API one and make steps are modified to
fit with the existing admin deployment targets.

Makefile

@@ -25,6 +25,9 @@ CF_SPACE ?= ${DEPLOY_ENV}
 CF_HOME ?= ${HOME}
 $(eval export CF_HOME)
 
+CF_MANIFEST_FILE ?= manifest-${CF_SPACE}.yml
+NOTIFY_CREDENTIALS ?= ~/.notify-credentials
+
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -167,12 +170,21 @@ cf-login: ## Log in to Cloud Foundry
 	@echo "Logging in to Cloud Foundry on ${CF_API}"
 	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
 
+.PHONY: generate-manifest
+generate-manifest:
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
+	$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
+
+	@./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \
+	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg)
+
 .PHONY: cf-deploy
 cf-deploy: ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	@cf app --guid notify-admin || exit 1
 	cf rename notify-admin notify-admin-rollback
-	cf push -f manifest-${CF_SPACE}.yml
+	cf push -f <(make -s generate-manifest)
 	cf scale -i $$(cf curl /v2/apps/$$(cf app --guid notify-admin-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") notify-admin
 	cf stop notify-admin-rollback
 	cf delete -f notify-admin-rollback
@@ -180,7 +192,7 @@ cf-deploy: ## Deploys the app to Cloud Foundry
 .PHONY: cf-deploy-prototype
 cf-deploy-prototype: cf-target ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
-	cf push -f manifest-prototype-${CF_SPACE}.yml
+	cf push -f <(make -s CF_MANIFEST_FILE=manifest-prototype-${CF_SPACE}.yml generate-manifest)
 
 .PHONY: cf-rollback
 cf-rollback: ## Rollbacks the app to the previous release
@@ -191,7 +203,7 @@ cf-rollback: ## Rollbacks the app to the previous release
 
 .PHONY: cf-push
 cf-push:
-	cf push -f manifest-${CF_SPACE}.yml
+	cf push -f <(make -s generate-manifest)
 
 .PHONY: cf-target
 cf-target: check-env-vars

scripts/generate_manifest.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+import json
+import yaml
+
+
+def merge_dicts(a, b):
+    if not (isinstance(a, dict) and isinstance(b, dict)):
+        raise ValueError("Error merging variables: '{}' and '{}'".format(
+            type(a).__name__, type(b).__name__
+        ))
+
+    result = a.copy()
+    for key, val in b.items():
+        if isinstance(result.get(key), dict):
+            result[key] = merge_dicts(a[key], b[key])
+        else:
+            result[key] = val
+
+    return result
+
+
+def load_manifest(manifest_file):
+    with open(manifest_file) as f:
+        manifest = yaml.load(f)
+
+    if 'inherit' in manifest:
+        inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit'))
+        manifest = merge_dicts(load_manifest(inherit_file), manifest)
+
+    return manifest
+
+
+def load_variables(vars_files):
+    variables = {}
+    for vars_file in vars_files:
+        with open(vars_file) as f:
+            variables = merge_dicts(variables, yaml.load(f))
+
+    return {
+        k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v
+        for k, v in variables.items()
+    }
+
+
+def paas_manifest(manifest_file, *vars_files):
+    manifest = load_manifest(manifest_file)
+    variables = load_variables(vars_files)
+
+    for key in manifest.get('env', {}):
+        if key in variables:
+            manifest['env'][key] = variables[key]
+
+    return yaml.dump(manifest, default_flow_style=False, allow_unicode=True)
+
+
+if __name__ == "__main__":
+    print('---')  # noqa
+    print(paas_manifest(*sys.argv[1:]))  # noqa