From 66ca98fbfb563097f42d13803e6fd75c2d999d34 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 10 Apr 2019 15:15:48 +0100 Subject: [PATCH] create manifest from jinja template newer versions of cf api don't allow you to have multiple apps per manifest file. So, instead of our current inheritance based model, move to the newer doc-dl/antivirus/template-preview approved jinja based model. the new single manifest.yml.j2 file sets a bunch of variables based on the CF_APP variable - things like NOTIFY_APP_NAME, default instances, etc. Then the manifest is built up to define all of the app options based on these defaults. Things default to sensible values, which can vary based on environment. When adding new environment variables, you'll need to add them to the manifest file. If they're json encoded lists, you'll need to pass them back to the `tojson` filter, or jinja2 will print them as python lists, with single quotes around strings. --- Makefile | 19 ++++++++-- manifest.yml.j2 | 95 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 manifest.yml.j2 diff --git a/Makefile b/Makefile index 3612540bc..d312b845a 100644 --- a/Makefile +++ b/Makefile @@ -209,8 +209,8 @@ cf-login: ## Log in to Cloud Foundry @echo "Logging in to Cloud Foundry on ${CF_API}" @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" -.PHONY: generate-manifest -generate-manifest: +.PHONY: generate-manifest-old +generate-manifest-old: $(if ${CF_APP},,$(error Must specify CF_APP)) $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) @@ -219,6 +219,19 @@ generate-manifest: @./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) +.PHONY: generate-manifest +generate-manifest: + $(if ${CF_APP},,$(error Must specify CF_APP)) + $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) + $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) + $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) + + @jinja2 --strict manifest.yml.j2 \ + -D environment=${CF_SPACE} \ + -D CF_APP=${CF_APP} \ + --format=yaml \ + <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1 + .PHONY: cf-deploy cf-deploy: ## Deploys the app to Cloud Foundry $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) @@ -242,7 +255,7 @@ cf-deploy-api-db-migration: cf unbind-service notify-api-db-migration notify-db cf unbind-service notify-api-db-migration notify-config cf unbind-service notify-api-db-migration notify-aws - cf push notify-api-db-migration -f <(make -s CF_APP=api generate-manifest) + cf push notify-api-db-migration -f <(make -s CF_APP=notify-api-db-migration generate-manifest) cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task diff --git a/manifest.yml.j2 b/manifest.yml.j2 new file mode 100644 index 000000000..67bb8bf02 --- /dev/null +++ b/manifest.yml.j2 @@ -0,0 +1,95 @@ +{%- set app_vars = { + 'notify-api': {'NOTIFY_APP_NAME': 'public-api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': { + 'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'], + 'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'], + 'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'], + } + }, + 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'public-api', 'instances': 0}, + + 'notify-delivery-celery-beat': {'NOTIFY_APP_NAME': 'delivery-celery-beat', 'instances': 1, 'memory': '128M'}, + 'notify-delivery-worker-database': {'NOTIFY_APP_NAME': 'delivery-worker-database'}, + 'notify-delivery-worker-research': {'NOTIFY_APP_NAME': 'delivery-worker-research'}, + 'notify-delivery-worker-sender': {'NOTIFY_APP_NAME': 'delivery-worker-sender', 'instances': 1, 'disk_quota': '2G', 'memory': '3G'}, + 'notify-delivery-worker-periodic': {'NOTIFY_APP_NAME': 'delivery-worker-periodic', 'instances': 1}, + 'notify-delivery-worker-priority': {'NOTIFY_APP_NAME': 'delivery-worker-priority'}, + 'notify-delivery-worker': {'NOTIFY_APP_NAME': 'delivery-worker'}, + 'notify-delivery-worker-internal': {'NOTIFY_APP_NAME': 'delivery-worker-internal'}, + 'notify-delivery-worker-receipts': {'NOTIFY_APP_NAME': 'delivery-worker-receipts'}, + 'notify-delivery-worker-service-callbacks': {'NOTIFY_APP_NAME': 'delivery-worker-service-callbacks', 'disk_quota': '2G'}, +} -%} + +{%- set env_defaults = { + 'preview': {'instances': 1}, + 'staging': {'instances': 2}, + 'production': {'instances': 2} +} -%} + +{%- set app = app_vars[CF_APP] -%} + +--- + +applications: + - name: {{ CF_APP }} + buildpack: python_buildpack + + instances: {{ app.get('instances', env_defaults[environment]['instances']) }} + memory: {{ app.get('memory', '1G') }} + disk_quota: {{ app.get('disk_quota', '1G')}} + + {% if 'routes' in app -%} + routes: + {%- for route in app['routes'][environment] %} + - route: {{ route }} + {%- endfor -%} + {%- else -%} + health-check-type: none + no-route: true + {% endif %} + + services: + - notify-db + - logit-ssl-syslog-drain + + env: + NOTIFY_APP_NAME: {{ app['NOTIFY_APP_NAME'] }} + SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }} + FLASK_APP: application.py + + # Credentials variables + ADMIN_BASE_URL: '{{ ADMIN_BASE_URL }}' + ADMIN_CLIENT_SECRET: '{{ ADMIN_CLIENT_SECRET }}' + API_HOST_NAME: '{{ API_HOST_NAME }}' + DANGEROUS_SALT: '{{ DANGEROUS_SALT }}' + SECRET_KEY: '{{ SECRET_KEY }}' + ROUTE_SECRET_KEY_1: '{{ ROUTE_SECRET_KEY_1 }}' + ROUTE_SECRET_KEY_2: '{{ ROUTE_SECRET_KEY_2 }}' + CRONITOR_KEYS: '{{ CRONITOR_KEYS | tojson }}' + + PERFORMANCE_PLATFORM_ENDPOINTS: '{{ PERFORMANCE_PLATFORM_ENDPOINTS | tojson }}' + + DOCUMENT_DOWNLOAD_API_HOST: '{{ DOCUMENT_DOWNLOAD_API_HOST }}' + DOCUMENT_DOWNLOAD_API_KEY: '{{ DOCUMENT_DOWNLOAD_API_KEY }}' + + NOTIFICATION_QUEUE_PREFIX: '{{ NOTIFICATION_QUEUE_PREFIX }}' + AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}' + AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}' + + STATSD_PREFIX: '{{ STATSD_PREFIX }}' + + ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}' + + MMG_URL: '{{ MMG_URL }}' + MMG_API_KEY: '{{ MMG_API_KEY }}' + MMG_INBOUND_SMS_AUTH: '{{ MMG_INBOUND_SMS_AUTH | tojson }}' + MMG_INBOUND_SMS_USERNAME: '{{ MMG_INBOUND_SMS_USERNAME | tojson }}' + + FIRETEXT_API_KEY: '{{ FIRETEXT_API_KEY }}' + LOADTESTING_API_KEY: '{{ LOADTESTING_API_KEY }}' + FIRETEXT_INBOUND_SMS_AUTH: '{{ FIRETEXT_INBOUND_SMS_AUTH | tojson }}' + + REDIS_ENABLED: '{{ REDIS_ENABLED }}' + REDIS_URL: '{{ REDIS_URL }}' + + TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}' + TEMPLATE_PREVIEW_API_KEY: '{{ TEMPLATE_PREVIEW_API_KEY }}'