create manifest from jinja template

newer versions of cf api don't allow you to have multiple apps per
manifest file. So, instead of our current inheritance based model, move
to the newer doc-dl/antivirus/template-preview approved jinja based
model.

the new single manifest.yml.j2 file sets a bunch of variables based on
the CF_APP variable - things like NOTIFY_APP_NAME, default instances,
etc. Then the manifest is built up to define all of the app options
based on these defaults. Things default to sensible values, which can
vary based on environment.

When adding new environment variables, you'll need to add them to the
manifest file. If they're json encoded lists, you'll need to pass them
back to the `tojson` filter, or jinja2 will print them as python lists,
with single quotes around strings.

Makefile

@@ -209,8 +209,8 @@ cf-login: ## Log in to Cloud Foundry
 	@echo "Logging in to Cloud Foundry on ${CF_API}"
 	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
 
-.PHONY: generate-manifest
-generate-manifest:
+.PHONY: generate-manifest-old
+generate-manifest-old:
 	$(if ${CF_APP},,$(error Must specify CF_APP))
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
@@ -219,6 +219,19 @@ generate-manifest:
 	@./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \
 	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg)
 
+.PHONY: generate-manifest
+generate-manifest:
+	$(if ${CF_APP},,$(error Must specify CF_APP))
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
+	$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
+
+	@jinja2 --strict manifest.yml.j2 \
+	    -D environment=${CF_SPACE} \
+	    -D CF_APP=${CF_APP} \
+	    --format=yaml \
+	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
+
 .PHONY: cf-deploy
 cf-deploy: ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
@@ -242,7 +255,7 @@ cf-deploy-api-db-migration:
 	cf unbind-service notify-api-db-migration notify-db
 	cf unbind-service notify-api-db-migration notify-config
 	cf unbind-service notify-api-db-migration notify-aws
-	cf push notify-api-db-migration -f <(make -s CF_APP=api generate-manifest)
+	cf push notify-api-db-migration -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
 	cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
 
 .PHONY: cf-check-api-db-migration-task

manifest.yml.j2

@@ -0,0 +1,95 @@
+{%- set app_vars = {
+  'notify-api': {'NOTIFY_APP_NAME': 'public-api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': {
+      'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'],
+      'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'],
+      'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'],
+    }
+  },
+  'notify-api-db-migration': {'NOTIFY_APP_NAME': 'public-api', 'instances': 0},
+
+  'notify-delivery-celery-beat': {'NOTIFY_APP_NAME': 'delivery-celery-beat', 'instances': 1, 'memory': '128M'},
+  'notify-delivery-worker-database': {'NOTIFY_APP_NAME': 'delivery-worker-database'},
+  'notify-delivery-worker-research': {'NOTIFY_APP_NAME': 'delivery-worker-research'},
+  'notify-delivery-worker-sender': {'NOTIFY_APP_NAME': 'delivery-worker-sender', 'instances': 1, 'disk_quota': '2G', 'memory': '3G'},
+  'notify-delivery-worker-periodic': {'NOTIFY_APP_NAME': 'delivery-worker-periodic', 'instances': 1},
+  'notify-delivery-worker-priority': {'NOTIFY_APP_NAME': 'delivery-worker-priority'},
+  'notify-delivery-worker': {'NOTIFY_APP_NAME': 'delivery-worker'},
+  'notify-delivery-worker-internal': {'NOTIFY_APP_NAME': 'delivery-worker-internal'},
+  'notify-delivery-worker-receipts': {'NOTIFY_APP_NAME': 'delivery-worker-receipts'},
+  'notify-delivery-worker-service-callbacks': {'NOTIFY_APP_NAME': 'delivery-worker-service-callbacks', 'disk_quota': '2G'},
+} -%}
+
+{%- set env_defaults = {
+  'preview': {'instances': 1},
+  'staging': {'instances': 2},
+  'production': {'instances': 2}
+} -%}
+
+{%- set app = app_vars[CF_APP] -%}
+
+---
+
+applications:
+  - name: {{ CF_APP }}
+    buildpack: python_buildpack
+
+    instances: {{ app.get('instances', env_defaults[environment]['instances']) }}
+    memory: {{ app.get('memory', '1G') }}
+    disk_quota: {{ app.get('disk_quota', '1G')}}
+
+    {% if 'routes' in app -%}
+    routes:
+      {%- for route in app['routes'][environment] %}
+      - route: {{ route }}
+      {%- endfor -%}
+    {%- else -%}
+    health-check-type: none
+    no-route: true
+    {% endif %}
+
+    services:
+      - notify-db
+      - logit-ssl-syslog-drain
+
+    env:
+      NOTIFY_APP_NAME: {{ app['NOTIFY_APP_NAME'] }}
+      SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }}
+      FLASK_APP: application.py
+
+      # Credentials variables
+      ADMIN_BASE_URL: '{{ ADMIN_BASE_URL }}'
+      ADMIN_CLIENT_SECRET: '{{ ADMIN_CLIENT_SECRET }}'
+      API_HOST_NAME: '{{ API_HOST_NAME }}'
+      DANGEROUS_SALT: '{{ DANGEROUS_SALT }}'
+      SECRET_KEY: '{{ SECRET_KEY }}'
+      ROUTE_SECRET_KEY_1: '{{ ROUTE_SECRET_KEY_1 }}'
+      ROUTE_SECRET_KEY_2: '{{ ROUTE_SECRET_KEY_2 }}'
+      CRONITOR_KEYS: '{{ CRONITOR_KEYS | tojson }}'
+
+      PERFORMANCE_PLATFORM_ENDPOINTS: '{{ PERFORMANCE_PLATFORM_ENDPOINTS | tojson }}'
+
+      DOCUMENT_DOWNLOAD_API_HOST: '{{ DOCUMENT_DOWNLOAD_API_HOST }}'
+      DOCUMENT_DOWNLOAD_API_KEY: '{{ DOCUMENT_DOWNLOAD_API_KEY }}'
+
+      NOTIFICATION_QUEUE_PREFIX: '{{ NOTIFICATION_QUEUE_PREFIX }}'
+      AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}'
+      AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}'
+
+      STATSD_PREFIX: '{{ STATSD_PREFIX }}'
+
+      ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}'
+
+      MMG_URL: '{{ MMG_URL }}'
+      MMG_API_KEY: '{{ MMG_API_KEY }}'
+      MMG_INBOUND_SMS_AUTH: '{{ MMG_INBOUND_SMS_AUTH | tojson }}'
+      MMG_INBOUND_SMS_USERNAME: '{{ MMG_INBOUND_SMS_USERNAME | tojson }}'
+
+      FIRETEXT_API_KEY: '{{ FIRETEXT_API_KEY }}'
+      LOADTESTING_API_KEY: '{{ LOADTESTING_API_KEY }}'
+      FIRETEXT_INBOUND_SMS_AUTH: '{{ FIRETEXT_INBOUND_SMS_AUTH | tojson }}'
+
+      REDIS_ENABLED: '{{ REDIS_ENABLED }}'
+      REDIS_URL: '{{ REDIS_URL }}'
+
+      TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}'
+      TEMPLATE_PREVIEW_API_KEY: '{{ TEMPLATE_PREVIEW_API_KEY }}'