From 9547fedcc3723a9f92a5e9b63ae6dea22441081f Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Fri, 12 Apr 2019 11:05:45 +0100 Subject: [PATCH] move manifest to single jinja template uses the same pattern as other apps. Infers route based on app name, you can declare more in the dict at the top if you need to. --- Makefile | 23 +++++------ Procfile | 1 + manifest-base.yml | 46 --------------------- manifest-preview.yml | 7 ---- manifest-production.yml | 9 ----- manifest-prototype-2-base.yml | 30 -------------- manifest-prototype-2-preview.yml | 6 --- manifest-prototype-2-production.yml | 6 --- manifest-prototype-base.yml | 30 -------------- manifest-prototype-preview.yml | 6 --- manifest-prototype-production.yml | 6 --- manifest-prototype-staging.yml | 6 --- manifest-sandbox.yml | 6 --- manifest-staging.yml | 10 ----- manifest.yml.j2 | 57 ++++++++++++++++++++++++++ requirements.txt | 8 ++-- requirements_for_test.txt | 2 + scripts/generate_manifest.py | 63 ----------------------------- 18 files changed, 74 insertions(+), 248 deletions(-) create mode 100644 Procfile delete mode 100644 manifest-base.yml delete mode 100644 manifest-preview.yml delete mode 100644 manifest-production.yml delete mode 100644 manifest-prototype-2-base.yml delete mode 100644 manifest-prototype-2-preview.yml delete mode 100644 manifest-prototype-2-production.yml delete mode 100644 manifest-prototype-base.yml delete mode 100644 manifest-prototype-preview.yml delete mode 100644 manifest-prototype-production.yml delete mode 100644 manifest-prototype-staging.yml delete mode 100644 manifest-sandbox.yml delete mode 100644 manifest-staging.yml create mode 100644 manifest.yml.j2 delete mode 100755 scripts/generate_manifest.py diff --git a/Makefile b/Makefile index 6b6c3554e..b9fa7ef31 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,6 @@ CF_SPACE ?= ${DEPLOY_ENV} CF_HOME ?= ${HOME} $(eval export CF_HOME) -CF_MANIFEST_FILE ?= manifest-${CF_SPACE}.yml NOTIFY_CREDENTIALS ?= ~/.notify-credentials .PHONY: help @@ -195,12 +194,16 @@ cf-login: ## Log in to Cloud Foundry .PHONY: generate-manifest generate-manifest: + $(if ${CF_APP},,$(error Must specify CF_APP)) $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) - @./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ - <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) + @jinja2 --strict manifest.yml.j2 \ + -D environment=${CF_SPACE} \ + -D CF_APP=${CF_APP} \ + --format=yaml \ + <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1 .PHONY: cf-deploy cf-deploy: ## Deploys the app to Cloud Foundry @@ -213,18 +216,12 @@ cf-deploy: ## Deploys the app to Cloud Foundry cf delete -f notify-admin-rollback .PHONY: cf-deploy-prototype -cf-deploy-prototype: cf-target ## Deploys the app to Cloud Foundry - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - (make -s CF_MANIFEST_FILE=manifest-prototype-${CF_SPACE}.yml generate-manifest) > /tmp/admin_manifest.yml - cf push -f /tmp/admin_manifest.yml - rm /tmp/admin_manifest.yml +cf-deploy-prototype: cf-target ## Deploys the first prototype to Cloud Foundry + cf push -f <(CF_APP=notify-admin-prototype make -s generate-manifest) .PHONY: cf-deploy-prototype-2 -cf-deploy-prototype-2: cf-target ## Deploys the app to Cloud Foundry - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - (make -s CF_MANIFEST_FILE=manifest-prototype-2-${CF_SPACE}.yml generate-manifest) > /tmp/admin_manifest.yml - cf push -f /tmp/admin_manifest.yml - rm /tmp/admin_manifest.yml +cf-deploy-prototype-2: cf-target ## Deploys the second prototype to Cloud Foundry + cf push -f <(CF_APP=notify-admin-prototype-2 make -s generate-manifest) .PHONY: cf-rollback cf-rollback: ## Rollbacks the app to the previous release diff --git a/Procfile b/Procfile new file mode 100644 index 000000000..79b31cf9e --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: unset GUNICORN_CMD_ARGS; scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application diff --git a/manifest-base.yml b/manifest-base.yml deleted file mode 100644 index c28457046..000000000 --- a/manifest-base.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- - -buildpack: python_buildpack -command: > - unset GUNICORN_CMD_ARGS; - scripts/run_app_paas.sh - gunicorn - -c /home/vcap/app/gunicorn_config.py - application - -instances: 1 -memory: 1G - -services: - - logit-ssl-syslog-drain - -env: - NOTIFY_APP_NAME: admin - - # Credentials variables - ADMIN_CLIENT_SECRET: null - ADMIN_BASE_URL: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - ANTIVIRUS_API_HOST: null - ANTIVIRUS_API_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - - REDIS_ENABLED: null - REDIS_URL: null - -applications: - - name: notify-admin diff --git a/manifest-preview.yml b/manifest-preview.yml deleted file mode 100644 index a8f10e896..000000000 --- a/manifest-preview.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -inherit: manifest-base.yml - -routes: - - route: notify-admin-preview.cloudapps.digital - - route: www.notify.works diff --git a/manifest-production.yml b/manifest-production.yml deleted file mode 100644 index 1eca336c7..000000000 --- a/manifest-production.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -inherit: manifest-base.yml - -routes: - - route: notify-admin-production.cloudapps.digital - - route: www.notifications.service.gov.uk -instances: 2 -memory: 1G diff --git a/manifest-prototype-2-base.yml b/manifest-prototype-2-base.yml deleted file mode 100644 index 284fb6427..000000000 --- a/manifest-prototype-2-base.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -buildpack: python_buildpack -command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT application -instances: 1 -memory: 1G -env: - NOTIFY_APP_NAME: admin - - # Credentials variables - ADMIN_CLIENT_SECRET: null - ADMIN_BASE_URL: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - -applications: - - name: notify-admin-prototype-2 diff --git a/manifest-prototype-2-preview.yml b/manifest-prototype-2-preview.yml deleted file mode 100644 index 1a998d2d9..000000000 --- a/manifest-prototype-2-preview.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-2-base.yml - -routes: - - route: notify-admin-prototype-2-preview.cloudapps.digital diff --git a/manifest-prototype-2-production.yml b/manifest-prototype-2-production.yml deleted file mode 100644 index ffffa7686..000000000 --- a/manifest-prototype-2-production.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-2-base.yml - -routes: - - route: notify-admin-prototype-2-production.cloudapps.digital diff --git a/manifest-prototype-base.yml b/manifest-prototype-base.yml deleted file mode 100644 index d2165800e..000000000 --- a/manifest-prototype-base.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -buildpack: python_buildpack -command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT application -instances: 1 -memory: 1G -env: - NOTIFY_APP_NAME: admin - - # Credentials variables - ADMIN_CLIENT_SECRET: null - ADMIN_BASE_URL: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - -applications: - - name: notify-admin-prototype diff --git a/manifest-prototype-preview.yml b/manifest-prototype-preview.yml deleted file mode 100644 index 140d17a98..000000000 --- a/manifest-prototype-preview.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-base.yml - -routes: - - route: notify-admin-prototype-preview.cloudapps.digital diff --git a/manifest-prototype-production.yml b/manifest-prototype-production.yml deleted file mode 100644 index 4fc239945..000000000 --- a/manifest-prototype-production.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-base.yml - -routes: - - route: notify-admin-prototype-production.cloudapps.digital diff --git a/manifest-prototype-staging.yml b/manifest-prototype-staging.yml deleted file mode 100644 index 0c840b43f..000000000 --- a/manifest-prototype-staging.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-base.yml - -routes: - - route: notify-admin-prototype-staging.cloudapps.digital diff --git a/manifest-sandbox.yml b/manifest-sandbox.yml deleted file mode 100644 index 8f22b1e17..000000000 --- a/manifest-sandbox.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-base.yml - -routes: - - route: notify-admin-sandbox.cloudapps.digital diff --git a/manifest-staging.yml b/manifest-staging.yml deleted file mode 100644 index c4a719da0..000000000 --- a/manifest-staging.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -inherit: manifest-base.yml - -routes: - - route: notify-admin-staging.cloudapps.digital - - route: www.staging-notify.works - -instances: 2 -memory: 1G diff --git a/manifest.yml.j2 b/manifest.yml.j2 new file mode 100644 index 000000000..a14826014 --- /dev/null +++ b/manifest.yml.j2 @@ -0,0 +1,57 @@ +{%- set apps = { + 'notify-admin': { + 'routes': { + 'preview': ['www.notify.works'], + 'staging': ['www.staging-notify.works'], + 'production': ['www.notifications.service.gov.uk'], + } + }, + 'notify-admin-prototype': {}, + 'notify-admin-prototype-2': {} +} -%} + +{%- set app = apps[CF_APP] -%} + +--- +applications: + - name: {{ CF_APP }} + buildpack: python_buildpack + + memory: 1G + + routes: + - route: {{ CF_APP }}-{{ environment }}.cloudapps.digital + {%- for route in app.get('routes', {}).get(environment, []) %} + - route: {{ route }} + {%- endfor %} + + services: + - logit-ssl-syslog-drain + + env: + NOTIFY_APP_NAME: admin + + # Credentials variables + ADMIN_CLIENT_SECRET: {{ ADMIN_CLIENT_SECRET }} + ADMIN_BASE_URL: {{ ADMIN_BASE_URL }} + API_HOST_NAME: {{ API_HOST_NAME }} + DANGEROUS_SALT: {{ DANGEROUS_SALT }} + SECRET_KEY: {{ SECRET_KEY }} + ROUTE_SECRET_KEY_1: {{ ROUTE_SECRET_KEY_1 }} + ROUTE_SECRET_KEY_2: {{ ROUTE_SECRET_KEY_2 }} + + AWS_ACCESS_KEY_ID: {{ AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: {{ AWS_SECRET_ACCESS_KEY }} + + ANTIVIRUS_API_HOST: {{ ANTIVIRUS_API_HOST }} + ANTIVIRUS_API_KEY: {{ ANTIVIRUS_API_KEY }} + + STATSD_PREFIX: {{ STATSD_PREFIX }} + + ZENDESK_API_KEY: {{ ZENDESK_API_KEY }} + + TEMPLATE_PREVIEW_API_HOST: {{ TEMPLATE_PREVIEW_API_HOST }} + TEMPLATE_PREVIEW_API_KEY: {{ TEMPLATE_PREVIEW_API_KEY }} + + REDIS_ENABLED: {{ REDIS_ENABLED }} + REDIS_URL: {{ REDIS_URL }} diff --git a/requirements.txt b/requirements.txt index e2bb6204d..8bd7bb210 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,10 +28,10 @@ itsdangerous==0.24 # pyup: <1.0.0 git+https://github.com/alphagov/notifications-utils.git@31.2.4#egg=notifications-utils==31.2.4 ## The following requirements were added by pip freeze: -awscli==1.16.136 +awscli==1.16.140 bleach==3.1.0 boto3==1.6.16 -botocore==1.12.126 +botocore==1.12.130 certifi==2019.3.9 chardet==3.0.4 Click==7.0 @@ -45,7 +45,7 @@ future==0.17.1 greenlet==0.4.15 idna==2.8 jdcal==1.4 -Jinja2==2.10 +Jinja2==2.10.1 jmespath==0.9.4 lml==0.0.9 lxml==4.3.3 @@ -72,7 +72,7 @@ statsd==3.3.0 texttable==1.6.1 urllib3==1.24.1 webencodings==0.5.1 -Werkzeug==0.15.1 +Werkzeug==0.15.2 WTForms==2.2.1 xlrd==1.2.0 xlwt==1.3.0 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 266092efd..3b0140c52 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -11,3 +11,5 @@ freezegun==0.3.11 flake8==3.7.7 flake8-print==3.1.0 requests-mock==1.5.2 +# used for creating manifest file locally +jinja2-cli[yaml]==0.6.0 diff --git a/scripts/generate_manifest.py b/scripts/generate_manifest.py deleted file mode 100755 index ce7982763..000000000 --- a/scripts/generate_manifest.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys - -import json -import yaml - - -def merge_dicts(a, b): - if not (isinstance(a, dict) and isinstance(b, dict)): - raise ValueError("Error merging variables: '{}' and '{}'".format( - type(a).__name__, type(b).__name__ - )) - - result = a.copy() - for key, val in b.items(): - if isinstance(result.get(key), dict): - result[key] = merge_dicts(a[key], b[key]) - else: - result[key] = val - - return result - - -def load_manifest(manifest_file): - with open(manifest_file) as f: - manifest = yaml.load(f) - - if 'inherit' in manifest: - inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit')) - manifest = merge_dicts(load_manifest(inherit_file), manifest) - - return manifest - - -def load_variables(vars_files): - variables = {} - for vars_file in vars_files: - with open(vars_file) as f: - variables = merge_dicts(variables, yaml.load(f)) - - return { - k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v - for k, v in variables.items() - } - - -def paas_manifest(manifest_file, *vars_files): - manifest = load_manifest(manifest_file) - variables = load_variables(vars_files) - - for key in manifest.get('env', {}): - if key in variables: - manifest['env'][key] = variables[key] - - return yaml.dump(manifest, default_flow_style=False, allow_unicode=True) - - -if __name__ == "__main__": - print('---') # noqa - print(paas_manifest(*sys.argv[1:])) # noqa