diff --git a/Makefile b/Makefile index 6b6c3554e..b9fa7ef31 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,6 @@ CF_SPACE ?= ${DEPLOY_ENV} CF_HOME ?= ${HOME} $(eval export CF_HOME) -CF_MANIFEST_FILE ?= manifest-${CF_SPACE}.yml NOTIFY_CREDENTIALS ?= ~/.notify-credentials .PHONY: help @@ -195,12 +194,16 @@ cf-login: ## Log in to Cloud Foundry .PHONY: generate-manifest generate-manifest: + $(if ${CF_APP},,$(error Must specify CF_APP)) $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) - @./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ - <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) + @jinja2 --strict manifest.yml.j2 \ + -D environment=${CF_SPACE} \ + -D CF_APP=${CF_APP} \ + --format=yaml \ + <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1 .PHONY: cf-deploy cf-deploy: ## Deploys the app to Cloud Foundry @@ -213,18 +216,12 @@ cf-deploy: ## Deploys the app to Cloud Foundry cf delete -f notify-admin-rollback .PHONY: cf-deploy-prototype -cf-deploy-prototype: cf-target ## Deploys the app to Cloud Foundry - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - (make -s CF_MANIFEST_FILE=manifest-prototype-${CF_SPACE}.yml generate-manifest) > /tmp/admin_manifest.yml - cf push -f /tmp/admin_manifest.yml - rm /tmp/admin_manifest.yml +cf-deploy-prototype: cf-target ## Deploys the first prototype to Cloud Foundry + cf push -f <(CF_APP=notify-admin-prototype make -s generate-manifest) .PHONY: cf-deploy-prototype-2 -cf-deploy-prototype-2: cf-target ## Deploys the app to Cloud Foundry - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - (make -s CF_MANIFEST_FILE=manifest-prototype-2-${CF_SPACE}.yml generate-manifest) > /tmp/admin_manifest.yml - cf push -f /tmp/admin_manifest.yml - rm /tmp/admin_manifest.yml +cf-deploy-prototype-2: cf-target ## Deploys the second prototype to Cloud Foundry + cf push -f <(CF_APP=notify-admin-prototype-2 make -s generate-manifest) .PHONY: cf-rollback cf-rollback: ## Rollbacks the app to the previous release diff --git a/Procfile b/Procfile new file mode 100644 index 000000000..79b31cf9e --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: unset GUNICORN_CMD_ARGS; scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application diff --git a/manifest-base.yml b/manifest-base.yml deleted file mode 100644 index c28457046..000000000 --- a/manifest-base.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- - -buildpack: python_buildpack -command: > - unset GUNICORN_CMD_ARGS; - scripts/run_app_paas.sh - gunicorn - -c /home/vcap/app/gunicorn_config.py - application - -instances: 1 -memory: 1G - -services: - - logit-ssl-syslog-drain - -env: - NOTIFY_APP_NAME: admin - - # Credentials variables - ADMIN_CLIENT_SECRET: null - ADMIN_BASE_URL: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - ANTIVIRUS_API_HOST: null - ANTIVIRUS_API_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - - REDIS_ENABLED: null - REDIS_URL: null - -applications: - - name: notify-admin diff --git a/manifest-preview.yml b/manifest-preview.yml deleted file mode 100644 index a8f10e896..000000000 --- a/manifest-preview.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -inherit: manifest-base.yml - -routes: - - route: notify-admin-preview.cloudapps.digital - - route: www.notify.works diff --git a/manifest-production.yml b/manifest-production.yml deleted file mode 100644 index 1eca336c7..000000000 --- a/manifest-production.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -inherit: manifest-base.yml - -routes: - - route: notify-admin-production.cloudapps.digital - - route: www.notifications.service.gov.uk -instances: 2 -memory: 1G diff --git a/manifest-prototype-2-base.yml b/manifest-prototype-2-base.yml deleted file mode 100644 index 284fb6427..000000000 --- a/manifest-prototype-2-base.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -buildpack: python_buildpack -command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT application -instances: 1 -memory: 1G -env: - NOTIFY_APP_NAME: admin - - # Credentials variables - ADMIN_CLIENT_SECRET: null - ADMIN_BASE_URL: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - -applications: - - name: notify-admin-prototype-2 diff --git a/manifest-prototype-2-preview.yml b/manifest-prototype-2-preview.yml deleted file mode 100644 index 1a998d2d9..000000000 --- a/manifest-prototype-2-preview.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-2-base.yml - -routes: - - route: notify-admin-prototype-2-preview.cloudapps.digital diff --git a/manifest-prototype-2-production.yml b/manifest-prototype-2-production.yml deleted file mode 100644 index ffffa7686..000000000 --- a/manifest-prototype-2-production.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-2-base.yml - -routes: - - route: notify-admin-prototype-2-production.cloudapps.digital diff --git a/manifest-prototype-base.yml b/manifest-prototype-base.yml deleted file mode 100644 index d2165800e..000000000 --- a/manifest-prototype-base.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -buildpack: python_buildpack -command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT application -instances: 1 -memory: 1G -env: - NOTIFY_APP_NAME: admin - - # Credentials variables - ADMIN_CLIENT_SECRET: null - ADMIN_BASE_URL: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - -applications: - - name: notify-admin-prototype diff --git a/manifest-prototype-preview.yml b/manifest-prototype-preview.yml deleted file mode 100644 index 140d17a98..000000000 --- a/manifest-prototype-preview.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-base.yml - -routes: - - route: notify-admin-prototype-preview.cloudapps.digital diff --git a/manifest-prototype-production.yml b/manifest-prototype-production.yml deleted file mode 100644 index 4fc239945..000000000 --- a/manifest-prototype-production.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-base.yml - -routes: - - route: notify-admin-prototype-production.cloudapps.digital diff --git a/manifest-prototype-staging.yml b/manifest-prototype-staging.yml deleted file mode 100644 index 0c840b43f..000000000 --- a/manifest-prototype-staging.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-prototype-base.yml - -routes: - - route: notify-admin-prototype-staging.cloudapps.digital diff --git a/manifest-sandbox.yml b/manifest-sandbox.yml deleted file mode 100644 index 8f22b1e17..000000000 --- a/manifest-sandbox.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-base.yml - -routes: - - route: notify-admin-sandbox.cloudapps.digital diff --git a/manifest-staging.yml b/manifest-staging.yml deleted file mode 100644 index c4a719da0..000000000 --- a/manifest-staging.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -inherit: manifest-base.yml - -routes: - - route: notify-admin-staging.cloudapps.digital - - route: www.staging-notify.works - -instances: 2 -memory: 1G diff --git a/manifest.yml.j2 b/manifest.yml.j2 new file mode 100644 index 000000000..a14826014 --- /dev/null +++ b/manifest.yml.j2 @@ -0,0 +1,57 @@ +{%- set apps = { + 'notify-admin': { + 'routes': { + 'preview': ['www.notify.works'], + 'staging': ['www.staging-notify.works'], + 'production': ['www.notifications.service.gov.uk'], + } + }, + 'notify-admin-prototype': {}, + 'notify-admin-prototype-2': {} +} -%} + +{%- set app = apps[CF_APP] -%} + +--- +applications: + - name: {{ CF_APP }} + buildpack: python_buildpack + + memory: 1G + + routes: + - route: {{ CF_APP }}-{{ environment }}.cloudapps.digital + {%- for route in app.get('routes', {}).get(environment, []) %} + - route: {{ route }} + {%- endfor %} + + services: + - logit-ssl-syslog-drain + + env: + NOTIFY_APP_NAME: admin + + # Credentials variables + ADMIN_CLIENT_SECRET: {{ ADMIN_CLIENT_SECRET }} + ADMIN_BASE_URL: {{ ADMIN_BASE_URL }} + API_HOST_NAME: {{ API_HOST_NAME }} + DANGEROUS_SALT: {{ DANGEROUS_SALT }} + SECRET_KEY: {{ SECRET_KEY }} + ROUTE_SECRET_KEY_1: {{ ROUTE_SECRET_KEY_1 }} + ROUTE_SECRET_KEY_2: {{ ROUTE_SECRET_KEY_2 }} + + AWS_ACCESS_KEY_ID: {{ AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: {{ AWS_SECRET_ACCESS_KEY }} + + ANTIVIRUS_API_HOST: {{ ANTIVIRUS_API_HOST }} + ANTIVIRUS_API_KEY: {{ ANTIVIRUS_API_KEY }} + + STATSD_PREFIX: {{ STATSD_PREFIX }} + + ZENDESK_API_KEY: {{ ZENDESK_API_KEY }} + + TEMPLATE_PREVIEW_API_HOST: {{ TEMPLATE_PREVIEW_API_HOST }} + TEMPLATE_PREVIEW_API_KEY: {{ TEMPLATE_PREVIEW_API_KEY }} + + REDIS_ENABLED: {{ REDIS_ENABLED }} + REDIS_URL: {{ REDIS_URL }} diff --git a/requirements.txt b/requirements.txt index e2bb6204d..8bd7bb210 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,10 +28,10 @@ itsdangerous==0.24 # pyup: <1.0.0 git+https://github.com/alphagov/notifications-utils.git@31.2.4#egg=notifications-utils==31.2.4 ## The following requirements were added by pip freeze: -awscli==1.16.136 +awscli==1.16.140 bleach==3.1.0 boto3==1.6.16 -botocore==1.12.126 +botocore==1.12.130 certifi==2019.3.9 chardet==3.0.4 Click==7.0 @@ -45,7 +45,7 @@ future==0.17.1 greenlet==0.4.15 idna==2.8 jdcal==1.4 -Jinja2==2.10 +Jinja2==2.10.1 jmespath==0.9.4 lml==0.0.9 lxml==4.3.3 @@ -72,7 +72,7 @@ statsd==3.3.0 texttable==1.6.1 urllib3==1.24.1 webencodings==0.5.1 -Werkzeug==0.15.1 +Werkzeug==0.15.2 WTForms==2.2.1 xlrd==1.2.0 xlwt==1.3.0 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 266092efd..3b0140c52 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -11,3 +11,5 @@ freezegun==0.3.11 flake8==3.7.7 flake8-print==3.1.0 requests-mock==1.5.2 +# used for creating manifest file locally +jinja2-cli[yaml]==0.6.0 diff --git a/scripts/generate_manifest.py b/scripts/generate_manifest.py deleted file mode 100755 index ce7982763..000000000 --- a/scripts/generate_manifest.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys - -import json -import yaml - - -def merge_dicts(a, b): - if not (isinstance(a, dict) and isinstance(b, dict)): - raise ValueError("Error merging variables: '{}' and '{}'".format( - type(a).__name__, type(b).__name__ - )) - - result = a.copy() - for key, val in b.items(): - if isinstance(result.get(key), dict): - result[key] = merge_dicts(a[key], b[key]) - else: - result[key] = val - - return result - - -def load_manifest(manifest_file): - with open(manifest_file) as f: - manifest = yaml.load(f) - - if 'inherit' in manifest: - inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit')) - manifest = merge_dicts(load_manifest(inherit_file), manifest) - - return manifest - - -def load_variables(vars_files): - variables = {} - for vars_file in vars_files: - with open(vars_file) as f: - variables = merge_dicts(variables, yaml.load(f)) - - return { - k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v - for k, v in variables.items() - } - - -def paas_manifest(manifest_file, *vars_files): - manifest = load_manifest(manifest_file) - variables = load_variables(vars_files) - - for key in manifest.get('env', {}): - if key in variables: - manifest['env'][key] = variables[key] - - return yaml.dump(manifest, default_flow_style=False, allow_unicode=True) - - -if __name__ == "__main__": - print('---') # noqa - print(paas_manifest(*sys.argv[1:])) # noqa