Merge pull request #2909 from alphagov/single-app-manifest

move manifest to single jinja template

Makefile

@@ -25,7 +25,6 @@ CF_SPACE ?= ${DEPLOY_ENV}
 CF_HOME ?= ${HOME}
 $(eval export CF_HOME)
 
-CF_MANIFEST_FILE ?= manifest-${CF_SPACE}.yml
 NOTIFY_CREDENTIALS ?= ~/.notify-credentials
 
 .PHONY: help
@@ -195,12 +194,16 @@ cf-login: ## Log in to Cloud Foundry
 
 .PHONY: generate-manifest
 generate-manifest:
+	$(if ${CF_APP},,$(error Must specify CF_APP))
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
 	$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
 
-	@./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \
+	@jinja2 --strict manifest.yml.j2 \
-	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg)
+	    -D environment=${CF_SPACE} \
+	    -D CF_APP=${CF_APP} \
+	    --format=yaml \
+	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
 
 .PHONY: cf-deploy
 cf-deploy: ## Deploys the app to Cloud Foundry
@@ -213,18 +216,12 @@ cf-deploy: ## Deploys the app to Cloud Foundry
 	cf delete -f notify-admin-rollback
 
 .PHONY: cf-deploy-prototype
-cf-deploy-prototype: cf-target ## Deploys the app to Cloud Foundry
+cf-deploy-prototype: cf-target ## Deploys the first prototype to Cloud Foundry
-	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	cf push -f <(CF_APP=notify-admin-prototype make -s generate-manifest)
-	(make -s CF_MANIFEST_FILE=manifest-prototype-${CF_SPACE}.yml generate-manifest) > /tmp/admin_manifest.yml
-	cf push -f /tmp/admin_manifest.yml
-	rm /tmp/admin_manifest.yml
 
 .PHONY: cf-deploy-prototype-2
-cf-deploy-prototype-2: cf-target ## Deploys the app to Cloud Foundry
+cf-deploy-prototype-2: cf-target ## Deploys the second prototype to Cloud Foundry
-	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	cf push -f <(CF_APP=notify-admin-prototype-2 make -s generate-manifest)
-	(make -s CF_MANIFEST_FILE=manifest-prototype-2-${CF_SPACE}.yml generate-manifest) > /tmp/admin_manifest.yml
-	cf push -f /tmp/admin_manifest.yml
-	rm /tmp/admin_manifest.yml
 
 .PHONY: cf-rollback
 cf-rollback: ## Rollbacks the app to the previous release

Procfile

@@ -0,0 +1 @@
+web: unset GUNICORN_CMD_ARGS; scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application

manifest-base.yml

@@ -1,46 +0,0 @@
----
-
-buildpack: python_buildpack
-command: >
-  unset GUNICORN_CMD_ARGS;
-  scripts/run_app_paas.sh
-  gunicorn
-  -c /home/vcap/app/gunicorn_config.py
-  application
-
-instances: 1
-memory: 1G
-
-services:
-  - logit-ssl-syslog-drain
-
-env:
-  NOTIFY_APP_NAME: admin
-
-  # Credentials variables
-  ADMIN_CLIENT_SECRET: null
-  ADMIN_BASE_URL: null
-  API_HOST_NAME: null
-  DANGEROUS_SALT: null
-  SECRET_KEY: null
-  ROUTE_SECRET_KEY_1: null
-  ROUTE_SECRET_KEY_2: null
-
-  AWS_ACCESS_KEY_ID: null
-  AWS_SECRET_ACCESS_KEY: null
-
-  ANTIVIRUS_API_HOST: null
-  ANTIVIRUS_API_KEY: null
-
-  STATSD_PREFIX: null
-
-  ZENDESK_API_KEY: null
-
-  TEMPLATE_PREVIEW_API_HOST: null
-  TEMPLATE_PREVIEW_API_KEY: null
-
-  REDIS_ENABLED: null
-  REDIS_URL: null
-
-applications:
-  - name: notify-admin

manifest-preview.yml

@@ -1,7 +0,0 @@
----
-
-inherit: manifest-base.yml
-
-routes:
-  - route: notify-admin-preview.cloudapps.digital
-  - route: www.notify.works

manifest-production.yml

@@ -1,9 +0,0 @@
----
-
-inherit: manifest-base.yml
-
-routes:
-  - route: notify-admin-production.cloudapps.digital
-  - route: www.notifications.service.gov.uk
-instances: 2
-memory: 1G

manifest-prototype-2-base.yml

@@ -1,30 +0,0 @@
----
-
-buildpack: python_buildpack
-command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT application
-instances: 1
-memory: 1G
-env:
-  NOTIFY_APP_NAME: admin
-
-  # Credentials variables
-  ADMIN_CLIENT_SECRET: null
-  ADMIN_BASE_URL: null
-  API_HOST_NAME: null
-  DANGEROUS_SALT: null
-  SECRET_KEY: null
-  ROUTE_SECRET_KEY_1: null
-  ROUTE_SECRET_KEY_2: null
-
-  AWS_ACCESS_KEY_ID: null
-  AWS_SECRET_ACCESS_KEY: null
-
-  STATSD_PREFIX: null
-
-  ZENDESK_API_KEY: null
-
-  TEMPLATE_PREVIEW_API_HOST: null
-  TEMPLATE_PREVIEW_API_KEY: null
-
-applications:
-  - name: notify-admin-prototype-2

manifest-prototype-2-preview.yml

@@ -1,6 +0,0 @@
----
-
-inherit: manifest-prototype-2-base.yml
-
-routes:
-  - route: notify-admin-prototype-2-preview.cloudapps.digital

manifest-prototype-2-production.yml

@@ -1,6 +0,0 @@
----
-
-inherit: manifest-prototype-2-base.yml
-
-routes:
-  - route: notify-admin-prototype-2-production.cloudapps.digital

manifest-prototype-base.yml

@@ -1,30 +0,0 @@
----
-
-buildpack: python_buildpack
-command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT application
-instances: 1
-memory: 1G
-env:
-  NOTIFY_APP_NAME: admin
-
-  # Credentials variables
-  ADMIN_CLIENT_SECRET: null
-  ADMIN_BASE_URL: null
-  API_HOST_NAME: null
-  DANGEROUS_SALT: null
-  SECRET_KEY: null
-  ROUTE_SECRET_KEY_1: null
-  ROUTE_SECRET_KEY_2: null
-
-  AWS_ACCESS_KEY_ID: null
-  AWS_SECRET_ACCESS_KEY: null
-
-  STATSD_PREFIX: null
-
-  ZENDESK_API_KEY: null
-
-  TEMPLATE_PREVIEW_API_HOST: null
-  TEMPLATE_PREVIEW_API_KEY: null
-
-applications:
-  - name: notify-admin-prototype

manifest-prototype-preview.yml

@@ -1,6 +0,0 @@
----
-
-inherit: manifest-prototype-base.yml
-
-routes:
-  - route: notify-admin-prototype-preview.cloudapps.digital

manifest-prototype-production.yml

@@ -1,6 +0,0 @@
----
-
-inherit: manifest-prototype-base.yml
-
-routes:
-  - route: notify-admin-prototype-production.cloudapps.digital

manifest-prototype-staging.yml

@@ -1,6 +0,0 @@
----
-
-inherit: manifest-prototype-base.yml
-
-routes:
-  - route: notify-admin-prototype-staging.cloudapps.digital

manifest-sandbox.yml

@@ -1,6 +0,0 @@
----
-
-inherit: manifest-base.yml
-
-routes:
-  - route: notify-admin-sandbox.cloudapps.digital

manifest-staging.yml

@@ -1,10 +0,0 @@
----
-
-inherit: manifest-base.yml
-
-routes:
-  - route: notify-admin-staging.cloudapps.digital
-  - route: www.staging-notify.works
-
-instances: 2
-memory: 1G

manifest.yml.j2

@@ -0,0 +1,57 @@
+{%- set apps = {
+  'notify-admin': {
+    'routes': {
+      'preview': ['www.notify.works'],
+      'staging': ['www.staging-notify.works'],
+      'production': ['www.notifications.service.gov.uk'],
+    }
+  },
+  'notify-admin-prototype': {},
+  'notify-admin-prototype-2': {}
+} -%}
+
+{%- set app = apps[CF_APP] -%}
+
+---
+applications:
+  - name: {{ CF_APP }}
+    buildpack: python_buildpack
+
+    memory: 1G
+
+    routes:
+      - route: {{ CF_APP }}-{{ environment }}.cloudapps.digital
+      {%- for route in app.get('routes', {}).get(environment, []) %}
+      - route: {{ route }}
+      {%- endfor %}
+
+    services:
+      - logit-ssl-syslog-drain
+
+    env:
+      NOTIFY_APP_NAME: admin
+
+      # Credentials variables
+      ADMIN_CLIENT_SECRET: {{ ADMIN_CLIENT_SECRET }}
+      ADMIN_BASE_URL: {{ ADMIN_BASE_URL }}
+      API_HOST_NAME: {{ API_HOST_NAME }}
+      DANGEROUS_SALT: {{ DANGEROUS_SALT }}
+      SECRET_KEY: {{ SECRET_KEY }}
+      ROUTE_SECRET_KEY_1: {{ ROUTE_SECRET_KEY_1 }}
+      ROUTE_SECRET_KEY_2: {{ ROUTE_SECRET_KEY_2 }}
+
+      AWS_ACCESS_KEY_ID: {{ AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: {{ AWS_SECRET_ACCESS_KEY }}
+
+      ANTIVIRUS_API_HOST: {{ ANTIVIRUS_API_HOST }}
+      ANTIVIRUS_API_KEY: {{ ANTIVIRUS_API_KEY }}
+
+      STATSD_PREFIX: {{ STATSD_PREFIX }}
+
+      ZENDESK_API_KEY: {{ ZENDESK_API_KEY }}
+
+      TEMPLATE_PREVIEW_API_HOST: {{ TEMPLATE_PREVIEW_API_HOST }}
+      TEMPLATE_PREVIEW_API_KEY: {{ TEMPLATE_PREVIEW_API_KEY }}
+
+      REDIS_ENABLED: {{ REDIS_ENABLED }}
+      REDIS_URL: {{ REDIS_URL }}

requirements_for_test.txt

@@ -11,3 +11,5 @@ freezegun==0.3.11
 flake8==3.7.7
 flake8-print==3.1.0
 requests-mock==1.5.2
+# used for creating manifest file locally
+jinja2-cli[yaml]==0.6.0

scripts/generate_manifest.py

@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import sys
-
-import json
-import yaml
-
-
-def merge_dicts(a, b):
-    if not (isinstance(a, dict) and isinstance(b, dict)):
-        raise ValueError("Error merging variables: '{}' and '{}'".format(
-            type(a).__name__, type(b).__name__
-        ))
-
-    result = a.copy()
-    for key, val in b.items():
-        if isinstance(result.get(key), dict):
-            result[key] = merge_dicts(a[key], b[key])
-        else:
-            result[key] = val
-
-    return result
-
-
-def load_manifest(manifest_file):
-    with open(manifest_file) as f:
-        manifest = yaml.load(f)
-
-    if 'inherit' in manifest:
-        inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit'))
-        manifest = merge_dicts(load_manifest(inherit_file), manifest)
-
-    return manifest
-
-
-def load_variables(vars_files):
-    variables = {}
-    for vars_file in vars_files:
-        with open(vars_file) as f:
-            variables = merge_dicts(variables, yaml.load(f))
-
-    return {
-        k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v
-        for k, v in variables.items()
-    }
-
-
-def paas_manifest(manifest_file, *vars_files):
-    manifest = load_manifest(manifest_file)
-    variables = load_variables(vars_files)
-
-    for key in manifest.get('env', {}):
-        if key in variables:
-            manifest['env'][key] = variables[key]
-
-    return yaml.dump(manifest, default_flow_style=False, allow_unicode=True)
-
-
-if __name__ == "__main__":
-    print('---')  # noqa
-    print(paas_manifest(*sys.argv[1:]))  # noqa