From fe77d4f6544eee2dc707955e6aa35dd475d1c45b Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 9 Apr 2019 17:00:54 +0100 Subject: [PATCH 1/7] move commands from manifest to procfile cf v3 commands don't appear to support commands in manifest files. They say that they do, but in practice they fail on cf v3-zdt-push with an error message "No process types returned from stager". This can be solved by moving the command from the manifest to a Procfile. However, the Procfile is part of the source code, and as such is the same for each app. To get around this, make the Procfile command invoke a new wrapper script, which checks the NOTIFY_APP_NAME env var and then calls the correct command --- Procfile | 1 + scripts/paas_app_wrapper.sh | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 Procfile create mode 100755 scripts/paas_app_wrapper.sh diff --git a/Procfile b/Procfile new file mode 100644 index 000000000..8662d896e --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: ./scripts/paas_app_wrapper.sh diff --git a/scripts/paas_app_wrapper.sh b/scripts/paas_app_wrapper.sh new file mode 100755 index 000000000..1bc112b0f --- /dev/null +++ b/scripts/paas_app_wrapper.sh @@ -0,0 +1,48 @@ +#!/bin/bash +case $NOTIFY_APP_NAME in + public-api) + unset GUNICORN_CMD_ARGS + scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application + ;; + delivery-worker) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null + ;; + delivery-worker-database) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q database-tasks 2> /dev/null + ;; + delivery-worker-research) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \ + -Q research-mode-tasks 2> /dev/null + ;; + delivery-worker-sender) + scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO \ + -Q send-sms-tasks,send-email-tasks + ;; + delivery-worker-periodic) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \ + -Q periodic-tasks,statistics-tasks 2> /dev/null + ;; + delivery-worker-priority) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \ + -Q priority-tasks 2> /dev/null + ;; + # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority + delivery-worker-internal) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q notify-internal-tasks 2> /dev/null + ;; + delivery-worker-receipts) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q ses-callbacks 2> /dev/null + ;; + delivery-worker-service-callbacks) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q service-callbacks 2> /dev/null + ;; + *) + echo "Unknown notify_app_name $NOTIFY_APP_NAME" + exit 1 + ;; +esac From 66ca98fbfb563097f42d13803e6fd75c2d999d34 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 10 Apr 2019 15:15:48 +0100 Subject: [PATCH 2/7] create manifest from jinja template newer versions of cf api don't allow you to have multiple apps per manifest file. So, instead of our current inheritance based model, move to the newer doc-dl/antivirus/template-preview approved jinja based model. the new single manifest.yml.j2 file sets a bunch of variables based on the CF_APP variable - things like NOTIFY_APP_NAME, default instances, etc. Then the manifest is built up to define all of the app options based on these defaults. Things default to sensible values, which can vary based on environment. When adding new environment variables, you'll need to add them to the manifest file. If they're json encoded lists, you'll need to pass them back to the `tojson` filter, or jinja2 will print them as python lists, with single quotes around strings. --- Makefile | 19 ++++++++-- manifest.yml.j2 | 95 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 manifest.yml.j2 diff --git a/Makefile b/Makefile index 3612540bc..d312b845a 100644 --- a/Makefile +++ b/Makefile @@ -209,8 +209,8 @@ cf-login: ## Log in to Cloud Foundry @echo "Logging in to Cloud Foundry on ${CF_API}" @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" -.PHONY: generate-manifest -generate-manifest: +.PHONY: generate-manifest-old +generate-manifest-old: $(if ${CF_APP},,$(error Must specify CF_APP)) $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) @@ -219,6 +219,19 @@ generate-manifest: @./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) +.PHONY: generate-manifest +generate-manifest: + $(if ${CF_APP},,$(error Must specify CF_APP)) + $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) + $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) + $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) + + @jinja2 --strict manifest.yml.j2 \ + -D environment=${CF_SPACE} \ + -D CF_APP=${CF_APP} \ + --format=yaml \ + <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1 + .PHONY: cf-deploy cf-deploy: ## Deploys the app to Cloud Foundry $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) @@ -242,7 +255,7 @@ cf-deploy-api-db-migration: cf unbind-service notify-api-db-migration notify-db cf unbind-service notify-api-db-migration notify-config cf unbind-service notify-api-db-migration notify-aws - cf push notify-api-db-migration -f <(make -s CF_APP=api generate-manifest) + cf push notify-api-db-migration -f <(make -s CF_APP=notify-api-db-migration generate-manifest) cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task diff --git a/manifest.yml.j2 b/manifest.yml.j2 new file mode 100644 index 000000000..67bb8bf02 --- /dev/null +++ b/manifest.yml.j2 @@ -0,0 +1,95 @@ +{%- set app_vars = { + 'notify-api': {'NOTIFY_APP_NAME': 'public-api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': { + 'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'], + 'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'], + 'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'], + } + }, + 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'public-api', 'instances': 0}, + + 'notify-delivery-celery-beat': {'NOTIFY_APP_NAME': 'delivery-celery-beat', 'instances': 1, 'memory': '128M'}, + 'notify-delivery-worker-database': {'NOTIFY_APP_NAME': 'delivery-worker-database'}, + 'notify-delivery-worker-research': {'NOTIFY_APP_NAME': 'delivery-worker-research'}, + 'notify-delivery-worker-sender': {'NOTIFY_APP_NAME': 'delivery-worker-sender', 'instances': 1, 'disk_quota': '2G', 'memory': '3G'}, + 'notify-delivery-worker-periodic': {'NOTIFY_APP_NAME': 'delivery-worker-periodic', 'instances': 1}, + 'notify-delivery-worker-priority': {'NOTIFY_APP_NAME': 'delivery-worker-priority'}, + 'notify-delivery-worker': {'NOTIFY_APP_NAME': 'delivery-worker'}, + 'notify-delivery-worker-internal': {'NOTIFY_APP_NAME': 'delivery-worker-internal'}, + 'notify-delivery-worker-receipts': {'NOTIFY_APP_NAME': 'delivery-worker-receipts'}, + 'notify-delivery-worker-service-callbacks': {'NOTIFY_APP_NAME': 'delivery-worker-service-callbacks', 'disk_quota': '2G'}, +} -%} + +{%- set env_defaults = { + 'preview': {'instances': 1}, + 'staging': {'instances': 2}, + 'production': {'instances': 2} +} -%} + +{%- set app = app_vars[CF_APP] -%} + +--- + +applications: + - name: {{ CF_APP }} + buildpack: python_buildpack + + instances: {{ app.get('instances', env_defaults[environment]['instances']) }} + memory: {{ app.get('memory', '1G') }} + disk_quota: {{ app.get('disk_quota', '1G')}} + + {% if 'routes' in app -%} + routes: + {%- for route in app['routes'][environment] %} + - route: {{ route }} + {%- endfor -%} + {%- else -%} + health-check-type: none + no-route: true + {% endif %} + + services: + - notify-db + - logit-ssl-syslog-drain + + env: + NOTIFY_APP_NAME: {{ app['NOTIFY_APP_NAME'] }} + SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }} + FLASK_APP: application.py + + # Credentials variables + ADMIN_BASE_URL: '{{ ADMIN_BASE_URL }}' + ADMIN_CLIENT_SECRET: '{{ ADMIN_CLIENT_SECRET }}' + API_HOST_NAME: '{{ API_HOST_NAME }}' + DANGEROUS_SALT: '{{ DANGEROUS_SALT }}' + SECRET_KEY: '{{ SECRET_KEY }}' + ROUTE_SECRET_KEY_1: '{{ ROUTE_SECRET_KEY_1 }}' + ROUTE_SECRET_KEY_2: '{{ ROUTE_SECRET_KEY_2 }}' + CRONITOR_KEYS: '{{ CRONITOR_KEYS | tojson }}' + + PERFORMANCE_PLATFORM_ENDPOINTS: '{{ PERFORMANCE_PLATFORM_ENDPOINTS | tojson }}' + + DOCUMENT_DOWNLOAD_API_HOST: '{{ DOCUMENT_DOWNLOAD_API_HOST }}' + DOCUMENT_DOWNLOAD_API_KEY: '{{ DOCUMENT_DOWNLOAD_API_KEY }}' + + NOTIFICATION_QUEUE_PREFIX: '{{ NOTIFICATION_QUEUE_PREFIX }}' + AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}' + AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}' + + STATSD_PREFIX: '{{ STATSD_PREFIX }}' + + ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}' + + MMG_URL: '{{ MMG_URL }}' + MMG_API_KEY: '{{ MMG_API_KEY }}' + MMG_INBOUND_SMS_AUTH: '{{ MMG_INBOUND_SMS_AUTH | tojson }}' + MMG_INBOUND_SMS_USERNAME: '{{ MMG_INBOUND_SMS_USERNAME | tojson }}' + + FIRETEXT_API_KEY: '{{ FIRETEXT_API_KEY }}' + LOADTESTING_API_KEY: '{{ LOADTESTING_API_KEY }}' + FIRETEXT_INBOUND_SMS_AUTH: '{{ FIRETEXT_INBOUND_SMS_AUTH | tojson }}' + + REDIS_ENABLED: '{{ REDIS_ENABLED }}' + REDIS_URL: '{{ REDIS_URL }}' + + TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}' + TEMPLATE_PREVIEW_API_KEY: '{{ TEMPLATE_PREVIEW_API_KEY }}' From 7d9cd58e89afaf1202cea7d5045f2e0339db1a97 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 10 Apr 2019 15:19:46 +0100 Subject: [PATCH 3/7] rename public-api to api we don't use the public-api bit anywhere - even cloudwatch overwrites based on CW_APP_NAME (which we can get rid of as this distinction is gone) --- manifest.yml.j2 | 4 ++-- scripts/paas_app_wrapper.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 67bb8bf02..f9a8d77d8 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -1,11 +1,11 @@ {%- set app_vars = { - 'notify-api': {'NOTIFY_APP_NAME': 'public-api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': { + 'notify-api': {'NOTIFY_APP_NAME': 'api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': { 'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'], 'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'], 'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'], } }, - 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'public-api', 'instances': 0}, + 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0}, 'notify-delivery-celery-beat': {'NOTIFY_APP_NAME': 'delivery-celery-beat', 'instances': 1, 'memory': '128M'}, 'notify-delivery-worker-database': {'NOTIFY_APP_NAME': 'delivery-worker-database'}, diff --git a/scripts/paas_app_wrapper.sh b/scripts/paas_app_wrapper.sh index 1bc112b0f..96c0bbade 100755 --- a/scripts/paas_app_wrapper.sh +++ b/scripts/paas_app_wrapper.sh @@ -1,6 +1,6 @@ #!/bin/bash case $NOTIFY_APP_NAME in - public-api) + api) unset GUNICORN_CMD_ARGS scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application ;; From 8cc5d4029187bca2bb846c438968032f41fc0e31 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 10 Apr 2019 15:21:30 +0100 Subject: [PATCH 4/7] remove old manifest files and creation script --- Makefile | 10 --- manifest-api-base.yml | 67 ------------------- manifest-api-preview.yml | 10 --- manifest-api-production.yml | 9 --- manifest-api-sandbox.yml | 6 -- manifest-api-staging.yml | 9 --- manifest-delivery-base.yml | 107 ------------------------------- manifest-delivery-preview.yml | 5 -- manifest-delivery-production.yml | 6 -- manifest-delivery-sandbox.yml | 3 - manifest-delivery-staging.yml | 6 -- scripts/generate_manifest.py | 65 ------------------- 12 files changed, 303 deletions(-) delete mode 100644 manifest-api-base.yml delete mode 100644 manifest-api-preview.yml delete mode 100644 manifest-api-production.yml delete mode 100644 manifest-api-sandbox.yml delete mode 100644 manifest-api-staging.yml delete mode 100644 manifest-delivery-base.yml delete mode 100644 manifest-delivery-preview.yml delete mode 100644 manifest-delivery-production.yml delete mode 100644 manifest-delivery-sandbox.yml delete mode 100644 manifest-delivery-staging.yml delete mode 100755 scripts/generate_manifest.py diff --git a/Makefile b/Makefile index d312b845a..c8a182706 100644 --- a/Makefile +++ b/Makefile @@ -209,16 +209,6 @@ cf-login: ## Log in to Cloud Foundry @echo "Logging in to Cloud Foundry on ${CF_API}" @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" -.PHONY: generate-manifest-old -generate-manifest-old: - $(if ${CF_APP},,$(error Must specify CF_APP)) - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) - $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) - - @./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ - <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) - .PHONY: generate-manifest generate-manifest: $(if ${CF_APP},,$(error Must specify CF_APP)) diff --git a/manifest-api-base.yml b/manifest-api-base.yml deleted file mode 100644 index 5901d45dc..000000000 --- a/manifest-api-base.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- - -buildpack: python_buildpack -command: unset GUNICORN_CMD_ARGS; scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application - -services: - - notify-db - - logit-ssl-syslog-drain - -env: - NOTIFY_APP_NAME: public-api - CW_APP_NAME: api - # required by cf run-task - FLASK_APP: application.py - SQLALCHEMY_POOL_SIZE: 20 - - # Credentials variables - ADMIN_BASE_URL: null - ADMIN_CLIENT_SECRET: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - CRONITOR_KEYS: null - - PERFORMANCE_PLATFORM_ENDPOINTS: null - - NOTIFICATION_QUEUE_PREFIX: null - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - MMG_URL: null - MMG_API_KEY: null - MMG_INBOUND_SMS_AUTH: null - MMG_INBOUND_SMS_USERNAME: null - - FIRETEXT_API_KEY: null - LOADTESTING_API_KEY: null - FIRETEXT_INBOUND_SMS_AUTH: null - - REDIS_ENABLED: null - REDIS_URL: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - - DOCUMENT_DOWNLOAD_API_HOST: null - DOCUMENT_DOWNLOAD_API_KEY: null - -instances: 1 -memory: 1G -disk_quota: 2G - -applications: - - name: notify-api - - - name: notify-api-db-migration - command: sleep infinity - no-route: true - health-check-type: none - instances: 1 - memory: 128M diff --git a/manifest-api-preview.yml b/manifest-api-preview.yml deleted file mode 100644 index 1eb88ef10..000000000 --- a/manifest-api-preview.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-preview.cloudapps.digital - - route: api.notify.works - -instances: 1 -memory: 1G diff --git a/manifest-api-production.yml b/manifest-api-production.yml deleted file mode 100644 index 6dfdbdaf7..000000000 --- a/manifest-api-production.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-production.cloudapps.digital - - route: api.notifications.service.gov.uk -instances: 2 -memory: 1G diff --git a/manifest-api-sandbox.yml b/manifest-api-sandbox.yml deleted file mode 100644 index feecde9ef..000000000 --- a/manifest-api-sandbox.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-sandbox.cloudapps.digital diff --git a/manifest-api-staging.yml b/manifest-api-staging.yml deleted file mode 100644 index ec6602930..000000000 --- a/manifest-api-staging.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-staging.cloudapps.digital - - route: api.staging-notify.works -instances: 2 -memory: 1G diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml deleted file mode 100644 index 2e70b1814..000000000 --- a/manifest-delivery-base.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- - -buildpack: python_buildpack -health-check-type: none -no-route: true - -services: - - notify-db - - logit-ssl-syslog-drain - -instances: 1 -memory: 1G - -env: - # Credentials variables - ADMIN_BASE_URL: null - ADMIN_CLIENT_SECRET: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - CRONITOR_KEYS: null - - PERFORMANCE_PLATFORM_ENDPOINTS: null - - NOTIFICATION_QUEUE_PREFIX: null - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - MMG_URL: null - MMG_API_KEY: null - MMG_INBOUND_SMS_AUTH: null - MMG_INBOUND_SMS_USERNAME: null - - FIRETEXT_API_KEY: null - LOADTESTING_API_KEY: null - FIRETEXT_INBOUND_SMS_AUTH: null - - REDIS_ENABLED: null - REDIS_URL: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - SQLALCHEMY_POOL_SIZE: 1 - -applications: - - name: notify-delivery-celery-beat - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO - instances: 1 - memory: 128M - env: - NOTIFY_APP_NAME: delivery-celery-beat - - - name: notify-delivery-worker-database - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-database - - - name: notify-delivery-worker-research - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-research - - - name: notify-delivery-worker-sender - command: scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO -Q send-sms-tasks,send-email-tasks - memory: 3G - disk_quota: 2G - env: - NOTIFY_APP_NAME: delivery-worker-sender - - - name: notify-delivery-worker-periodic - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks 2> /dev/null - instances: 1 - env: - NOTIFY_APP_NAME: delivery-worker-periodic - - - name: notify-delivery-worker-priority - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-priority - - - name: notify-delivery-worker - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker - - # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority - - name: notify-delivery-worker-internal - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q notify-internal-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-internal - - - name: notify-delivery-worker-receipts - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q ses-callbacks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-receipts - - - name: notify-delivery-worker-service-callbacks - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q service-callbacks 2> /dev/null - disk_quota: 2G - env: - NOTIFY_APP_NAME: delivery-worker-service-callbacks diff --git a/manifest-delivery-preview.yml b/manifest-delivery-preview.yml deleted file mode 100644 index 77fd211c2..000000000 --- a/manifest-delivery-preview.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml - -memory: 1G diff --git a/manifest-delivery-production.yml b/manifest-delivery-production.yml deleted file mode 100644 index 53c8d2f12..000000000 --- a/manifest-delivery-production.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml - -instances: 2 -memory: 1G diff --git a/manifest-delivery-sandbox.yml b/manifest-delivery-sandbox.yml deleted file mode 100644 index d628e5fc9..000000000 --- a/manifest-delivery-sandbox.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml diff --git a/manifest-delivery-staging.yml b/manifest-delivery-staging.yml deleted file mode 100644 index 53c8d2f12..000000000 --- a/manifest-delivery-staging.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml - -instances: 2 -memory: 1G diff --git a/scripts/generate_manifest.py b/scripts/generate_manifest.py deleted file mode 100755 index 8814049c8..000000000 --- a/scripts/generate_manifest.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys - -import json -import yaml - - -def merge_dicts(a, b): - if not (isinstance(a, dict) and isinstance(b, dict)): - raise ValueError("Error merging variables: '{}' and '{}'".format( - type(a).__name__, type(b).__name__ - )) - - result = a.copy() - for key, val in b.items(): - if isinstance(result.get(key), dict): - result[key] = merge_dicts(a[key], b[key]) - else: - result[key] = val - - return result - - -def load_manifest(manifest_file): - with open(manifest_file) as f: - manifest = yaml.load(f) - - if 'inherit' in manifest: - inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit')) - manifest = merge_dicts(load_manifest(inherit_file), manifest) - - return manifest - - -def load_variables(vars_files): - variables = {} - for vars_file in vars_files: - with open(vars_file) as f: - variables = merge_dicts(variables, yaml.load(f)) - - return { - k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v - for k, v in variables.items() - } - - -def paas_manifest(manifest_file, *vars_files): - """Generate a PaaS manifest file from a Jinja2 template""" - - manifest = load_manifest(manifest_file) - variables = load_variables(vars_files) - - for key in manifest.get('env', {}): - if key in variables: - manifest['env'][key] = variables[key] - - return yaml.dump(manifest, default_flow_style=False, allow_unicode=True) - - -if __name__ == "__main__": - print('---') - print(paas_manifest(*sys.argv[1:])) From 575ec96b6b59ff33b8dfb960fa0e8e4cefbb9d8c Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 10 Apr 2019 16:48:21 +0100 Subject: [PATCH 5/7] fix test that checks manifest contains references to all queues --- tests/test_all_queues_used.py | 18 ++++++++++++++++++ tests/test_manifest_delivery_base.py | 23 ----------------------- 2 files changed, 18 insertions(+), 23 deletions(-) create mode 100644 tests/test_all_queues_used.py delete mode 100644 tests/test_manifest_delivery_base.py diff --git a/tests/test_all_queues_used.py b/tests/test_all_queues_used.py new file mode 100644 index 000000000..1a7166b96 --- /dev/null +++ b/tests/test_all_queues_used.py @@ -0,0 +1,18 @@ +from app.config import QueueNames + + +def test_queue_names_set_in_paas_app_wrapper(): + with open("scripts/paas_app_wrapper.sh", 'r') as stream: + search = ' -Q ' + + watched_queues = set() + for line in stream.readlines(): + start_of_queue_arg = line.find(search) + if start_of_queue_arg > 0: + start_of_queue_names = start_of_queue_arg + len(search) + end_of_queue_names = line.find('2>') if '2>' in line else len(line) + watched_queues.update({q.strip() for q in line[start_of_queue_names:end_of_queue_names].split(',')}) + + # ses-callbacks isn't used in api (only used in SNS lambda) + ignored_queues = {'ses-callbacks'} + assert watched_queues == set(QueueNames.all_queues()) | ignored_queues diff --git a/tests/test_manifest_delivery_base.py b/tests/test_manifest_delivery_base.py deleted file mode 100644 index 43c3cfb54..000000000 --- a/tests/test_manifest_delivery_base.py +++ /dev/null @@ -1,23 +0,0 @@ -import yaml - -from app.config import QueueNames - - -def test_queue_names_set_in_manifest_delivery_base_correctly(): - with open("manifest-delivery-base.yml", 'r') as stream: - search = ' -Q ' - yml_commands = [y['command'] for y in yaml.load(stream)['applications']] - - watched_queues = set() - for command in yml_commands: - start_of_queue_arg = command.find(search) - if start_of_queue_arg > 0: - start_of_queue_names = start_of_queue_arg + len(search) - end_of_queue_names = command.find('2>') if '2>' in command else len(command) - watched_queues.update({q.strip() for q in command[start_of_queue_names:end_of_queue_names].split(',')}) - - # ses-callbacks isn't used in api (only used in SNS lambda) - ignored_queues = {'ses-callbacks'} - watched_queues -= ignored_queues - - assert watched_queues == set(QueueNames.all_queues()) From c786258a602200e1967e578be0bb83f8e9df867d Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Thu, 11 Apr 2019 14:57:22 +0100 Subject: [PATCH 6/7] give instances and NOTIFY_APP_NAME sensible defaults NOTIFY_APP_NAME follows precedent and just tries to strip 'notify-' from the beginning of the string. instances is not specified at all if not defined - it'll scale up to the same amount of instances as currently present, and then the autoscaler will take over anyway --- manifest.yml.j2 | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/manifest.yml.j2 b/manifest.yml.j2 index f9a8d77d8..5738b70a4 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -7,22 +7,16 @@ }, 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0}, - 'notify-delivery-celery-beat': {'NOTIFY_APP_NAME': 'delivery-celery-beat', 'instances': 1, 'memory': '128M'}, - 'notify-delivery-worker-database': {'NOTIFY_APP_NAME': 'delivery-worker-database'}, - 'notify-delivery-worker-research': {'NOTIFY_APP_NAME': 'delivery-worker-research'}, - 'notify-delivery-worker-sender': {'NOTIFY_APP_NAME': 'delivery-worker-sender', 'instances': 1, 'disk_quota': '2G', 'memory': '3G'}, - 'notify-delivery-worker-periodic': {'NOTIFY_APP_NAME': 'delivery-worker-periodic', 'instances': 1}, - 'notify-delivery-worker-priority': {'NOTIFY_APP_NAME': 'delivery-worker-priority'}, - 'notify-delivery-worker': {'NOTIFY_APP_NAME': 'delivery-worker'}, - 'notify-delivery-worker-internal': {'NOTIFY_APP_NAME': 'delivery-worker-internal'}, - 'notify-delivery-worker-receipts': {'NOTIFY_APP_NAME': 'delivery-worker-receipts'}, - 'notify-delivery-worker-service-callbacks': {'NOTIFY_APP_NAME': 'delivery-worker-service-callbacks', 'disk_quota': '2G'}, -} -%} - -{%- set env_defaults = { - 'preview': {'instances': 1}, - 'staging': {'instances': 2}, - 'production': {'instances': 2} + 'notify-delivery-celery-beat': {'memory': '128M'}, + 'notify-delivery-worker-database': {}, + 'notify-delivery-worker-research': {}, + 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'}, + 'notify-delivery-worker-periodic': {}, + 'notify-delivery-worker-priority': {}, + 'notify-delivery-worker': {}, + 'notify-delivery-worker-internal': {}, + 'notify-delivery-worker-receipts': {}, + 'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'}, } -%} {%- set app = app_vars[CF_APP] -%} @@ -32,8 +26,9 @@ applications: - name: {{ CF_APP }} buildpack: python_buildpack - - instances: {{ app.get('instances', env_defaults[environment]['instances']) }} + {% if 'instances' in app %} + instances: {{ app['instances'] }} + {%- endif %} memory: {{ app.get('memory', '1G') }} disk_quota: {{ app.get('disk_quota', '1G')}} @@ -52,7 +47,7 @@ applications: - logit-ssl-syslog-drain env: - NOTIFY_APP_NAME: {{ app['NOTIFY_APP_NAME'] }} + NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }} SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }} FLASK_APP: application.py From 3277c65a7ce04a4e7cd1c01239da7936131629c4 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Thu, 11 Apr 2019 15:05:44 +0100 Subject: [PATCH 7/7] add jinja2-cli to dev requirements --- requirements.txt | 10 +++++----- requirements_for_test.txt | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0d1dd3003..c0037b1df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,12 +40,12 @@ alembic==1.0.8 amqp==1.4.9 anyjson==0.3.3 attrs==19.1.0 -awscli==1.16.129 +awscli==1.16.140 bcrypt==3.1.6 billiard==3.3.0.23 bleach==3.1.0 boto3==1.6.16 -botocore==1.12.119 +botocore==1.12.130 certifi==2019.3.9 chardet==3.0.4 Click==7.0 @@ -55,7 +55,7 @@ Flask-Redis==0.3.0 future==0.17.1 greenlet==0.4.15 idna==2.8 -Jinja2==2.10 +Jinja2==2.10.1 jmespath==0.9.4 kombu==3.0.37 Mako==1.0.8 @@ -71,7 +71,7 @@ pyrsistent==0.14.11 python-dateutil==2.8.0 python-editor==1.0.4 python-json-logger==0.1.10 -pytz==2018.9 +pytz==2019.1 PyYAML==3.12 redis==3.2.1 requests==2.21.0 @@ -82,4 +82,4 @@ smartypants==2.0.1 statsd==3.3.0 urllib3==1.24.1 webencodings==0.5.1 -Werkzeug==0.15.1 +Werkzeug==0.15.2 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index aac36b5b0..c0ddbdcff 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -12,3 +12,5 @@ requests-mock==1.5.2 # optional requirements for jsonschema strict-rfc3339==0.7 rfc3987==1.3.8 +# used for creating manifest file locally +jinja2-cli[yaml]==0.6.0