diff --git a/Makefile b/Makefile index 3612540bc..c8a182706 100644 --- a/Makefile +++ b/Makefile @@ -216,8 +216,11 @@ generate-manifest: $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) - @./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ - <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) + @jinja2 --strict manifest.yml.j2 \ + -D environment=${CF_SPACE} \ + -D CF_APP=${CF_APP} \ + --format=yaml \ + <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1 .PHONY: cf-deploy cf-deploy: ## Deploys the app to Cloud Foundry @@ -242,7 +245,7 @@ cf-deploy-api-db-migration: cf unbind-service notify-api-db-migration notify-db cf unbind-service notify-api-db-migration notify-config cf unbind-service notify-api-db-migration notify-aws - cf push notify-api-db-migration -f <(make -s CF_APP=api generate-manifest) + cf push notify-api-db-migration -f <(make -s CF_APP=notify-api-db-migration generate-manifest) cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task diff --git a/Procfile b/Procfile new file mode 100644 index 000000000..8662d896e --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: ./scripts/paas_app_wrapper.sh diff --git a/manifest-api-base.yml b/manifest-api-base.yml deleted file mode 100644 index 5901d45dc..000000000 --- a/manifest-api-base.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- - -buildpack: python_buildpack -command: unset GUNICORN_CMD_ARGS; scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application - -services: - - notify-db - - logit-ssl-syslog-drain - -env: - NOTIFY_APP_NAME: public-api - CW_APP_NAME: api - # required by cf run-task - FLASK_APP: application.py - SQLALCHEMY_POOL_SIZE: 20 - - # Credentials variables - ADMIN_BASE_URL: null - ADMIN_CLIENT_SECRET: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - CRONITOR_KEYS: null - - PERFORMANCE_PLATFORM_ENDPOINTS: null - - NOTIFICATION_QUEUE_PREFIX: null - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - MMG_URL: null - MMG_API_KEY: null - MMG_INBOUND_SMS_AUTH: null - MMG_INBOUND_SMS_USERNAME: null - - FIRETEXT_API_KEY: null - LOADTESTING_API_KEY: null - FIRETEXT_INBOUND_SMS_AUTH: null - - REDIS_ENABLED: null - REDIS_URL: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - - DOCUMENT_DOWNLOAD_API_HOST: null - DOCUMENT_DOWNLOAD_API_KEY: null - -instances: 1 -memory: 1G -disk_quota: 2G - -applications: - - name: notify-api - - - name: notify-api-db-migration - command: sleep infinity - no-route: true - health-check-type: none - instances: 1 - memory: 128M diff --git a/manifest-api-preview.yml b/manifest-api-preview.yml deleted file mode 100644 index 1eb88ef10..000000000 --- a/manifest-api-preview.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-preview.cloudapps.digital - - route: api.notify.works - -instances: 1 -memory: 1G diff --git a/manifest-api-production.yml b/manifest-api-production.yml deleted file mode 100644 index 6dfdbdaf7..000000000 --- a/manifest-api-production.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-production.cloudapps.digital - - route: api.notifications.service.gov.uk -instances: 2 -memory: 1G diff --git a/manifest-api-sandbox.yml b/manifest-api-sandbox.yml deleted file mode 100644 index feecde9ef..000000000 --- a/manifest-api-sandbox.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-sandbox.cloudapps.digital diff --git a/manifest-api-staging.yml b/manifest-api-staging.yml deleted file mode 100644 index ec6602930..000000000 --- a/manifest-api-staging.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -inherit: manifest-api-base.yml - -routes: - - route: notify-api-staging.cloudapps.digital - - route: api.staging-notify.works -instances: 2 -memory: 1G diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml deleted file mode 100644 index 2e70b1814..000000000 --- a/manifest-delivery-base.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- - -buildpack: python_buildpack -health-check-type: none -no-route: true - -services: - - notify-db - - logit-ssl-syslog-drain - -instances: 1 -memory: 1G - -env: - # Credentials variables - ADMIN_BASE_URL: null - ADMIN_CLIENT_SECRET: null - API_HOST_NAME: null - DANGEROUS_SALT: null - SECRET_KEY: null - ROUTE_SECRET_KEY_1: null - ROUTE_SECRET_KEY_2: null - CRONITOR_KEYS: null - - PERFORMANCE_PLATFORM_ENDPOINTS: null - - NOTIFICATION_QUEUE_PREFIX: null - AWS_ACCESS_KEY_ID: null - AWS_SECRET_ACCESS_KEY: null - - STATSD_PREFIX: null - - ZENDESK_API_KEY: null - - MMG_URL: null - MMG_API_KEY: null - MMG_INBOUND_SMS_AUTH: null - MMG_INBOUND_SMS_USERNAME: null - - FIRETEXT_API_KEY: null - LOADTESTING_API_KEY: null - FIRETEXT_INBOUND_SMS_AUTH: null - - REDIS_ENABLED: null - REDIS_URL: null - - TEMPLATE_PREVIEW_API_HOST: null - TEMPLATE_PREVIEW_API_KEY: null - SQLALCHEMY_POOL_SIZE: 1 - -applications: - - name: notify-delivery-celery-beat - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO - instances: 1 - memory: 128M - env: - NOTIFY_APP_NAME: delivery-celery-beat - - - name: notify-delivery-worker-database - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-database - - - name: notify-delivery-worker-research - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-research - - - name: notify-delivery-worker-sender - command: scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO -Q send-sms-tasks,send-email-tasks - memory: 3G - disk_quota: 2G - env: - NOTIFY_APP_NAME: delivery-worker-sender - - - name: notify-delivery-worker-periodic - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks 2> /dev/null - instances: 1 - env: - NOTIFY_APP_NAME: delivery-worker-periodic - - - name: notify-delivery-worker-priority - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-priority - - - name: notify-delivery-worker - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker - - # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority - - name: notify-delivery-worker-internal - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q notify-internal-tasks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-internal - - - name: notify-delivery-worker-receipts - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q ses-callbacks 2> /dev/null - env: - NOTIFY_APP_NAME: delivery-worker-receipts - - - name: notify-delivery-worker-service-callbacks - command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q service-callbacks 2> /dev/null - disk_quota: 2G - env: - NOTIFY_APP_NAME: delivery-worker-service-callbacks diff --git a/manifest-delivery-preview.yml b/manifest-delivery-preview.yml deleted file mode 100644 index 77fd211c2..000000000 --- a/manifest-delivery-preview.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml - -memory: 1G diff --git a/manifest-delivery-production.yml b/manifest-delivery-production.yml deleted file mode 100644 index 53c8d2f12..000000000 --- a/manifest-delivery-production.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml - -instances: 2 -memory: 1G diff --git a/manifest-delivery-sandbox.yml b/manifest-delivery-sandbox.yml deleted file mode 100644 index d628e5fc9..000000000 --- a/manifest-delivery-sandbox.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml diff --git a/manifest-delivery-staging.yml b/manifest-delivery-staging.yml deleted file mode 100644 index 53c8d2f12..000000000 --- a/manifest-delivery-staging.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -inherit: manifest-delivery-base.yml - -instances: 2 -memory: 1G diff --git a/manifest.yml.j2 b/manifest.yml.j2 new file mode 100644 index 000000000..5738b70a4 --- /dev/null +++ b/manifest.yml.j2 @@ -0,0 +1,90 @@ +{%- set app_vars = { + 'notify-api': {'NOTIFY_APP_NAME': 'api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': { + 'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'], + 'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'], + 'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'], + } + }, + 'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0}, + + 'notify-delivery-celery-beat': {'memory': '128M'}, + 'notify-delivery-worker-database': {}, + 'notify-delivery-worker-research': {}, + 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'}, + 'notify-delivery-worker-periodic': {}, + 'notify-delivery-worker-priority': {}, + 'notify-delivery-worker': {}, + 'notify-delivery-worker-internal': {}, + 'notify-delivery-worker-receipts': {}, + 'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'}, +} -%} + +{%- set app = app_vars[CF_APP] -%} + +--- + +applications: + - name: {{ CF_APP }} + buildpack: python_buildpack + {% if 'instances' in app %} + instances: {{ app['instances'] }} + {%- endif %} + memory: {{ app.get('memory', '1G') }} + disk_quota: {{ app.get('disk_quota', '1G')}} + + {% if 'routes' in app -%} + routes: + {%- for route in app['routes'][environment] %} + - route: {{ route }} + {%- endfor -%} + {%- else -%} + health-check-type: none + no-route: true + {% endif %} + + services: + - notify-db + - logit-ssl-syslog-drain + + env: + NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }} + SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }} + FLASK_APP: application.py + + # Credentials variables + ADMIN_BASE_URL: '{{ ADMIN_BASE_URL }}' + ADMIN_CLIENT_SECRET: '{{ ADMIN_CLIENT_SECRET }}' + API_HOST_NAME: '{{ API_HOST_NAME }}' + DANGEROUS_SALT: '{{ DANGEROUS_SALT }}' + SECRET_KEY: '{{ SECRET_KEY }}' + ROUTE_SECRET_KEY_1: '{{ ROUTE_SECRET_KEY_1 }}' + ROUTE_SECRET_KEY_2: '{{ ROUTE_SECRET_KEY_2 }}' + CRONITOR_KEYS: '{{ CRONITOR_KEYS | tojson }}' + + PERFORMANCE_PLATFORM_ENDPOINTS: '{{ PERFORMANCE_PLATFORM_ENDPOINTS | tojson }}' + + DOCUMENT_DOWNLOAD_API_HOST: '{{ DOCUMENT_DOWNLOAD_API_HOST }}' + DOCUMENT_DOWNLOAD_API_KEY: '{{ DOCUMENT_DOWNLOAD_API_KEY }}' + + NOTIFICATION_QUEUE_PREFIX: '{{ NOTIFICATION_QUEUE_PREFIX }}' + AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}' + AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}' + + STATSD_PREFIX: '{{ STATSD_PREFIX }}' + + ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}' + + MMG_URL: '{{ MMG_URL }}' + MMG_API_KEY: '{{ MMG_API_KEY }}' + MMG_INBOUND_SMS_AUTH: '{{ MMG_INBOUND_SMS_AUTH | tojson }}' + MMG_INBOUND_SMS_USERNAME: '{{ MMG_INBOUND_SMS_USERNAME | tojson }}' + + FIRETEXT_API_KEY: '{{ FIRETEXT_API_KEY }}' + LOADTESTING_API_KEY: '{{ LOADTESTING_API_KEY }}' + FIRETEXT_INBOUND_SMS_AUTH: '{{ FIRETEXT_INBOUND_SMS_AUTH | tojson }}' + + REDIS_ENABLED: '{{ REDIS_ENABLED }}' + REDIS_URL: '{{ REDIS_URL }}' + + TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}' + TEMPLATE_PREVIEW_API_KEY: '{{ TEMPLATE_PREVIEW_API_KEY }}' diff --git a/requirements.txt b/requirements.txt index 57d2d04f9..213a16d81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,12 +40,12 @@ alembic==1.0.8 amqp==1.4.9 anyjson==0.3.3 attrs==19.1.0 -awscli==1.16.129 +awscli==1.16.140 bcrypt==3.1.6 billiard==3.3.0.23 bleach==3.1.0 boto3==1.6.16 -botocore==1.12.119 +botocore==1.12.130 certifi==2019.3.9 chardet==3.0.4 Click==7.0 @@ -55,7 +55,7 @@ Flask-Redis==0.3.0 future==0.17.1 greenlet==0.4.15 idna==2.8 -Jinja2==2.10 +Jinja2==2.10.1 jmespath==0.9.4 kombu==3.0.37 Mako==1.0.8 @@ -71,7 +71,7 @@ pyrsistent==0.14.11 python-dateutil==2.8.0 python-editor==1.0.4 python-json-logger==0.1.10 -pytz==2018.9 +pytz==2019.1 PyYAML==3.12 redis==3.2.1 requests==2.21.0 @@ -82,4 +82,4 @@ smartypants==2.0.1 statsd==3.3.0 urllib3==1.24.1 webencodings==0.5.1 -Werkzeug==0.15.1 +Werkzeug==0.15.2 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index aac36b5b0..c0ddbdcff 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -12,3 +12,5 @@ requests-mock==1.5.2 # optional requirements for jsonschema strict-rfc3339==0.7 rfc3987==1.3.8 +# used for creating manifest file locally +jinja2-cli[yaml]==0.6.0 diff --git a/scripts/generate_manifest.py b/scripts/generate_manifest.py deleted file mode 100755 index 8814049c8..000000000 --- a/scripts/generate_manifest.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys - -import json -import yaml - - -def merge_dicts(a, b): - if not (isinstance(a, dict) and isinstance(b, dict)): - raise ValueError("Error merging variables: '{}' and '{}'".format( - type(a).__name__, type(b).__name__ - )) - - result = a.copy() - for key, val in b.items(): - if isinstance(result.get(key), dict): - result[key] = merge_dicts(a[key], b[key]) - else: - result[key] = val - - return result - - -def load_manifest(manifest_file): - with open(manifest_file) as f: - manifest = yaml.load(f) - - if 'inherit' in manifest: - inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit')) - manifest = merge_dicts(load_manifest(inherit_file), manifest) - - return manifest - - -def load_variables(vars_files): - variables = {} - for vars_file in vars_files: - with open(vars_file) as f: - variables = merge_dicts(variables, yaml.load(f)) - - return { - k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v - for k, v in variables.items() - } - - -def paas_manifest(manifest_file, *vars_files): - """Generate a PaaS manifest file from a Jinja2 template""" - - manifest = load_manifest(manifest_file) - variables = load_variables(vars_files) - - for key in manifest.get('env', {}): - if key in variables: - manifest['env'][key] = variables[key] - - return yaml.dump(manifest, default_flow_style=False, allow_unicode=True) - - -if __name__ == "__main__": - print('---') - print(paas_manifest(*sys.argv[1:])) diff --git a/scripts/paas_app_wrapper.sh b/scripts/paas_app_wrapper.sh new file mode 100755 index 000000000..96c0bbade --- /dev/null +++ b/scripts/paas_app_wrapper.sh @@ -0,0 +1,48 @@ +#!/bin/bash +case $NOTIFY_APP_NAME in + api) + unset GUNICORN_CMD_ARGS + scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application + ;; + delivery-worker) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null + ;; + delivery-worker-database) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q database-tasks 2> /dev/null + ;; + delivery-worker-research) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \ + -Q research-mode-tasks 2> /dev/null + ;; + delivery-worker-sender) + scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO \ + -Q send-sms-tasks,send-email-tasks + ;; + delivery-worker-periodic) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \ + -Q periodic-tasks,statistics-tasks 2> /dev/null + ;; + delivery-worker-priority) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \ + -Q priority-tasks 2> /dev/null + ;; + # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority + delivery-worker-internal) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q notify-internal-tasks 2> /dev/null + ;; + delivery-worker-receipts) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q ses-callbacks 2> /dev/null + ;; + delivery-worker-service-callbacks) + scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \ + -Q service-callbacks 2> /dev/null + ;; + *) + echo "Unknown notify_app_name $NOTIFY_APP_NAME" + exit 1 + ;; +esac diff --git a/tests/test_all_queues_used.py b/tests/test_all_queues_used.py new file mode 100644 index 000000000..1a7166b96 --- /dev/null +++ b/tests/test_all_queues_used.py @@ -0,0 +1,18 @@ +from app.config import QueueNames + + +def test_queue_names_set_in_paas_app_wrapper(): + with open("scripts/paas_app_wrapper.sh", 'r') as stream: + search = ' -Q ' + + watched_queues = set() + for line in stream.readlines(): + start_of_queue_arg = line.find(search) + if start_of_queue_arg > 0: + start_of_queue_names = start_of_queue_arg + len(search) + end_of_queue_names = line.find('2>') if '2>' in line else len(line) + watched_queues.update({q.strip() for q in line[start_of_queue_names:end_of_queue_names].split(',')}) + + # ses-callbacks isn't used in api (only used in SNS lambda) + ignored_queues = {'ses-callbacks'} + assert watched_queues == set(QueueNames.all_queues()) | ignored_queues diff --git a/tests/test_manifest_delivery_base.py b/tests/test_manifest_delivery_base.py deleted file mode 100644 index 43c3cfb54..000000000 --- a/tests/test_manifest_delivery_base.py +++ /dev/null @@ -1,23 +0,0 @@ -import yaml - -from app.config import QueueNames - - -def test_queue_names_set_in_manifest_delivery_base_correctly(): - with open("manifest-delivery-base.yml", 'r') as stream: - search = ' -Q ' - yml_commands = [y['command'] for y in yaml.load(stream)['applications']] - - watched_queues = set() - for command in yml_commands: - start_of_queue_arg = command.find(search) - if start_of_queue_arg > 0: - start_of_queue_names = start_of_queue_arg + len(search) - end_of_queue_names = command.find('2>') if '2>' in command else len(command) - watched_queues.update({q.strip() for q in command[start_of_queue_names:end_of_queue_names].split(',')}) - - # ses-callbacks isn't used in api (only used in SNS lambda) - ignored_queues = {'ses-callbacks'} - watched_queues -= ignored_queues - - assert watched_queues == set(QueueNames.all_queues())