Merge pull request #2456 from alphagov/remove-multi-app-manifests

Remove multi app manifests
2026-01-31 06:52:06 -05:00 · 2019-04-12 10:34:49 +01:00
parent 622b3e76f7 3277c65a7c
commit 5d26724022
19 changed files with 170 additions and 324 deletions
--- a/9
+++ b/9
@@ -216,8 +216,11 @@ generate-manifest:
 	$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
 	$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))

-	@./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \
-	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg)
+	@jinja2 --strict manifest.yml.j2 \
+	    -D environment=${CF_SPACE} \
+	    -D CF_APP=${CF_APP} \
+	    --format=yaml \
+	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1

 .PHONY: cf-deploy
 cf-deploy: ## Deploys the app to Cloud Foundry
@@ -242,7 +245,7 @@ cf-deploy-api-db-migration:
 	cf unbind-service notify-api-db-migration notify-db
 	cf unbind-service notify-api-db-migration notify-config
 	cf unbind-service notify-api-db-migration notify-aws
-	cf push notify-api-db-migration -f <(make -s CF_APP=api generate-manifest)
+	cf push notify-api-db-migration -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
 	cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration

 .PHONY: cf-check-api-db-migration-task
--- a/1
+++ b/1
@@ -0,0 +1 @@
+web: ./scripts/paas_app_wrapper.sh
--- a/manifest-api-base.yml
+++ b/manifest-api-base.yml
@@ -1,67 +0,0 @@
---
-
-buildpack: python_buildpack
-command: unset GUNICORN_CMD_ARGS; scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
-
-services:
-  - notify-db
-  - logit-ssl-syslog-drain
-
-env:
-  NOTIFY_APP_NAME: public-api
-  CW_APP_NAME: api
-  # required by cf run-task
-  FLASK_APP: application.py
-  SQLALCHEMY_POOL_SIZE: 20
-
-  # Credentials variables
-  ADMIN_BASE_URL: null
-  ADMIN_CLIENT_SECRET: null
-  API_HOST_NAME: null
-  DANGEROUS_SALT: null
-  SECRET_KEY: null
-  ROUTE_SECRET_KEY_1: null
-  ROUTE_SECRET_KEY_2: null
-  CRONITOR_KEYS: null
-
-  PERFORMANCE_PLATFORM_ENDPOINTS: null
-
-  NOTIFICATION_QUEUE_PREFIX: null
-  AWS_ACCESS_KEY_ID: null
-  AWS_SECRET_ACCESS_KEY: null
-
-  STATSD_PREFIX: null
-
-  ZENDESK_API_KEY: null
-
-  MMG_URL: null
-  MMG_API_KEY: null
-  MMG_INBOUND_SMS_AUTH: null
-  MMG_INBOUND_SMS_USERNAME: null
-
-  FIRETEXT_API_KEY: null
-  LOADTESTING_API_KEY: null
-  FIRETEXT_INBOUND_SMS_AUTH: null
-
-  REDIS_ENABLED: null
-  REDIS_URL: null
-
-  TEMPLATE_PREVIEW_API_HOST: null
-  TEMPLATE_PREVIEW_API_KEY: null
-
-  DOCUMENT_DOWNLOAD_API_HOST: null
-  DOCUMENT_DOWNLOAD_API_KEY: null
-
-instances: 1
-memory: 1G
-disk_quota: 2G
-
-applications:
-  - name: notify-api
-
-  - name: notify-api-db-migration
-    command: sleep infinity
-    no-route: true
-    health-check-type: none
-    instances: 1
-    memory: 128M
--- a/manifest-api-preview.yml
+++ b/manifest-api-preview.yml
@@ -1,10 +0,0 @@
---
-
-inherit: manifest-api-base.yml
-
-routes:
-  - route: notify-api-preview.cloudapps.digital
-  - route: api.notify.works
-
-instances: 1
-memory: 1G
--- a/manifest-api-production.yml
+++ b/manifest-api-production.yml
@@ -1,9 +0,0 @@
---
-
-inherit: manifest-api-base.yml
-
-routes:
-  - route: notify-api-production.cloudapps.digital
-  - route: api.notifications.service.gov.uk
-instances: 2
-memory: 1G
--- a/manifest-api-sandbox.yml
+++ b/manifest-api-sandbox.yml
@@ -1,6 +0,0 @@
---
-
-inherit: manifest-api-base.yml
-
-routes:
-  - route: notify-api-sandbox.cloudapps.digital
--- a/manifest-api-staging.yml
+++ b/manifest-api-staging.yml
@@ -1,9 +0,0 @@
---
-
-inherit: manifest-api-base.yml
-
-routes:
-  - route: notify-api-staging.cloudapps.digital
-  - route: api.staging-notify.works
-instances: 2
-memory: 1G
--- a/manifest-delivery-base.yml
+++ b/manifest-delivery-base.yml
@@ -1,107 +0,0 @@
---
-
-buildpack: python_buildpack
-health-check-type: none
-no-route: true
-
-services:
-  - notify-db
-  - logit-ssl-syslog-drain
-
-instances: 1
-memory: 1G
-
-env:
-  # Credentials variables
-  ADMIN_BASE_URL: null
-  ADMIN_CLIENT_SECRET: null
-  API_HOST_NAME: null
-  DANGEROUS_SALT: null
-  SECRET_KEY: null
-  ROUTE_SECRET_KEY_1: null
-  ROUTE_SECRET_KEY_2: null
-  CRONITOR_KEYS: null
-
-  PERFORMANCE_PLATFORM_ENDPOINTS: null
-
-  NOTIFICATION_QUEUE_PREFIX: null
-  AWS_ACCESS_KEY_ID: null
-  AWS_SECRET_ACCESS_KEY: null
-
-  STATSD_PREFIX: null
-
-  ZENDESK_API_KEY: null
-
-  MMG_URL: null
-  MMG_API_KEY: null
-  MMG_INBOUND_SMS_AUTH: null
-  MMG_INBOUND_SMS_USERNAME: null
-
-  FIRETEXT_API_KEY: null
-  LOADTESTING_API_KEY: null
-  FIRETEXT_INBOUND_SMS_AUTH: null
-
-  REDIS_ENABLED: null
-  REDIS_URL: null
-
-  TEMPLATE_PREVIEW_API_HOST: null
-  TEMPLATE_PREVIEW_API_KEY: null
-  SQLALCHEMY_POOL_SIZE: 1
-
-applications:
-  - name: notify-delivery-celery-beat
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO
-    instances: 1
-    memory: 128M
-    env:
-      NOTIFY_APP_NAME: delivery-celery-beat
-
-  - name: notify-delivery-worker-database
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks 2> /dev/null
-    env:
-      NOTIFY_APP_NAME: delivery-worker-database
-
-  - name: notify-delivery-worker-research
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks 2> /dev/null
-    env:
-      NOTIFY_APP_NAME: delivery-worker-research
-
-  - name: notify-delivery-worker-sender
-    command: scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO -Q send-sms-tasks,send-email-tasks
-    memory: 3G
-    disk_quota: 2G
-    env:
-      NOTIFY_APP_NAME: delivery-worker-sender
-
-  - name: notify-delivery-worker-periodic
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks 2> /dev/null
-    instances: 1
-    env:
-      NOTIFY_APP_NAME: delivery-worker-periodic
-
-  - name: notify-delivery-worker-priority
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks 2> /dev/null
-    env:
-      NOTIFY_APP_NAME: delivery-worker-priority
-
-  - name: notify-delivery-worker
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null
-    env:
-      NOTIFY_APP_NAME: delivery-worker
-
-  # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority
-  - name: notify-delivery-worker-internal
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q notify-internal-tasks 2> /dev/null
-    env:
-      NOTIFY_APP_NAME: delivery-worker-internal
-
-  - name: notify-delivery-worker-receipts
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q ses-callbacks 2> /dev/null
-    env:
-      NOTIFY_APP_NAME: delivery-worker-receipts
-
-  - name: notify-delivery-worker-service-callbacks
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q service-callbacks 2> /dev/null
-    disk_quota: 2G
-    env:
-      NOTIFY_APP_NAME: delivery-worker-service-callbacks
--- a/manifest-delivery-preview.yml
+++ b/manifest-delivery-preview.yml
@@ -1,5 +0,0 @@
---
-
-inherit: manifest-delivery-base.yml
-
-memory: 1G
--- a/manifest-delivery-production.yml
+++ b/manifest-delivery-production.yml
@@ -1,6 +0,0 @@
---
-
-inherit: manifest-delivery-base.yml
-
-instances: 2
-memory: 1G
--- a/manifest-delivery-sandbox.yml
+++ b/manifest-delivery-sandbox.yml
@@ -1,3 +0,0 @@
---
-
-inherit: manifest-delivery-base.yml
--- a/manifest-delivery-staging.yml
+++ b/manifest-delivery-staging.yml
@@ -1,6 +0,0 @@
---
-
-inherit: manifest-delivery-base.yml
-
-instances: 2
-memory: 1G
--- a/manifest.yml.j2
+++ b/manifest.yml.j2
@@ -0,0 +1,90 @@
+{%- set app_vars = {
+  'notify-api': {'NOTIFY_APP_NAME': 'api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': {
+      'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'],
+      'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'],
+      'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'],
+    }
+  },
+  'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0},
+
+  'notify-delivery-celery-beat': {'memory': '128M'},
+  'notify-delivery-worker-database': {},
+  'notify-delivery-worker-research': {},
+  'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'},
+  'notify-delivery-worker-periodic': {},
+  'notify-delivery-worker-priority': {},
+  'notify-delivery-worker': {},
+  'notify-delivery-worker-internal': {},
+  'notify-delivery-worker-receipts': {},
+  'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'},
+} -%}
+
+{%- set app = app_vars[CF_APP] -%}
+
+---
+
+applications:
+  - name: {{ CF_APP }}
+    buildpack: python_buildpack
+    {% if 'instances' in app %}
+    instances: {{ app['instances'] }}
+    {%- endif %}
+    memory: {{ app.get('memory', '1G') }}
+    disk_quota: {{ app.get('disk_quota', '1G')}}
+
+    {% if 'routes' in app -%}
+    routes:
+      {%- for route in app['routes'][environment] %}
+      - route: {{ route }}
+      {%- endfor -%}
+    {%- else -%}
+    health-check-type: none
+    no-route: true
+    {% endif %}
+
+    services:
+      - notify-db
+      - logit-ssl-syslog-drain
+
+    env:
+      NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }}
+      SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }}
+      FLASK_APP: application.py
+
+      # Credentials variables
+      ADMIN_BASE_URL: '{{ ADMIN_BASE_URL }}'
+      ADMIN_CLIENT_SECRET: '{{ ADMIN_CLIENT_SECRET }}'
+      API_HOST_NAME: '{{ API_HOST_NAME }}'
+      DANGEROUS_SALT: '{{ DANGEROUS_SALT }}'
+      SECRET_KEY: '{{ SECRET_KEY }}'
+      ROUTE_SECRET_KEY_1: '{{ ROUTE_SECRET_KEY_1 }}'
+      ROUTE_SECRET_KEY_2: '{{ ROUTE_SECRET_KEY_2 }}'
+      CRONITOR_KEYS: '{{ CRONITOR_KEYS | tojson }}'
+
+      PERFORMANCE_PLATFORM_ENDPOINTS: '{{ PERFORMANCE_PLATFORM_ENDPOINTS | tojson }}'
+
+      DOCUMENT_DOWNLOAD_API_HOST: '{{ DOCUMENT_DOWNLOAD_API_HOST }}'
+      DOCUMENT_DOWNLOAD_API_KEY: '{{ DOCUMENT_DOWNLOAD_API_KEY }}'
+
+      NOTIFICATION_QUEUE_PREFIX: '{{ NOTIFICATION_QUEUE_PREFIX }}'
+      AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}'
+      AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}'
+
+      STATSD_PREFIX: '{{ STATSD_PREFIX }}'
+
+      ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}'
+
+      MMG_URL: '{{ MMG_URL }}'
+      MMG_API_KEY: '{{ MMG_API_KEY }}'
+      MMG_INBOUND_SMS_AUTH: '{{ MMG_INBOUND_SMS_AUTH | tojson }}'
+      MMG_INBOUND_SMS_USERNAME: '{{ MMG_INBOUND_SMS_USERNAME | tojson }}'
+
+      FIRETEXT_API_KEY: '{{ FIRETEXT_API_KEY }}'
+      LOADTESTING_API_KEY: '{{ LOADTESTING_API_KEY }}'
+      FIRETEXT_INBOUND_SMS_AUTH: '{{ FIRETEXT_INBOUND_SMS_AUTH | tojson }}'
+
+      REDIS_ENABLED: '{{ REDIS_ENABLED }}'
+      REDIS_URL: '{{ REDIS_URL }}'
+
+      TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}'
+      TEMPLATE_PREVIEW_API_KEY: '{{ TEMPLATE_PREVIEW_API_KEY }}'
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,12 +40,12 @@ alembic==1.0.8
 amqp==1.4.9
 anyjson==0.3.3
 attrs==19.1.0
-awscli==1.16.129
+awscli==1.16.140
 bcrypt==3.1.6
 billiard==3.3.0.23
 bleach==3.1.0
 boto3==1.6.16
-botocore==1.12.119
+botocore==1.12.130
 certifi==2019.3.9
 chardet==3.0.4
 Click==7.0
@@ -55,7 +55,7 @@ Flask-Redis==0.3.0
 future==0.17.1
 greenlet==0.4.15
 idna==2.8
-Jinja2==2.10
+Jinja2==2.10.1
 jmespath==0.9.4
 kombu==3.0.37
 Mako==1.0.8
@@ -71,7 +71,7 @@ pyrsistent==0.14.11
 python-dateutil==2.8.0
 python-editor==1.0.4
 python-json-logger==0.1.10
-pytz==2018.9
+pytz==2019.1
 PyYAML==3.12
 redis==3.2.1
 requests==2.21.0
@@ -82,4 +82,4 @@ smartypants==2.0.1
 statsd==3.3.0
 urllib3==1.24.1
 webencodings==0.5.1
-Werkzeug==0.15.1
+Werkzeug==0.15.2
--- a/requirements_for_test.txt
+++ b/requirements_for_test.txt
@@ -12,3 +12,5 @@ requests-mock==1.5.2
 # optional requirements for jsonschema
 strict-rfc3339==0.7
 rfc3987==1.3.8
+# used for creating manifest file locally
+jinja2-cli[yaml]==0.6.0
--- a/scripts/generate_manifest.py
+++ b/scripts/generate_manifest.py
@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import sys
-
-import json
-import yaml
-
-
-def merge_dicts(a, b):
-    if not (isinstance(a, dict) and isinstance(b, dict)):
-        raise ValueError("Error merging variables: '{}' and '{}'".format(
-            type(a).__name__, type(b).__name__
-        ))
-
-    result = a.copy()
-    for key, val in b.items():
-        if isinstance(result.get(key), dict):
-            result[key] = merge_dicts(a[key], b[key])
-        else:
-            result[key] = val
-
-    return result
-
-
-def load_manifest(manifest_file):
-    with open(manifest_file) as f:
-        manifest = yaml.load(f)
-
-    if 'inherit' in manifest:
-        inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit'))
-        manifest = merge_dicts(load_manifest(inherit_file), manifest)
-
-    return manifest
-
-
-def load_variables(vars_files):
-    variables = {}
-    for vars_file in vars_files:
-        with open(vars_file) as f:
-            variables = merge_dicts(variables, yaml.load(f))
-
-    return {
-        k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v
-        for k, v in variables.items()
-    }
-
-
-def paas_manifest(manifest_file, *vars_files):
-    """Generate a PaaS manifest file from a Jinja2 template"""
-
-    manifest = load_manifest(manifest_file)
-    variables = load_variables(vars_files)
-
-    for key in manifest.get('env', {}):
-        if key in variables:
-            manifest['env'][key] = variables[key]
-
-    return yaml.dump(manifest, default_flow_style=False, allow_unicode=True)
-
-
-if __name__ == "__main__":
-    print('---')
-    print(paas_manifest(*sys.argv[1:]))
--- a/scripts/paas_app_wrapper.sh
+++ b/scripts/paas_app_wrapper.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+case $NOTIFY_APP_NAME in
+  api)
+    unset GUNICORN_CMD_ARGS
+    scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
+    ;;
+  delivery-worker)
+    scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
+    -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null
+    ;;
+  delivery-worker-database)
+    scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
+    -Q database-tasks 2> /dev/null
+    ;;
+  delivery-worker-research)
+    scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \
+    -Q research-mode-tasks 2> /dev/null
+    ;;
+  delivery-worker-sender)
+    scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO \
+    -Q send-sms-tasks,send-email-tasks
+    ;;
+  delivery-worker-periodic)
+    scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \
+    -Q periodic-tasks,statistics-tasks 2> /dev/null
+    ;;
+  delivery-worker-priority)
+    scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \
+    -Q priority-tasks 2> /dev/null
+    ;;
+  # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority
+  delivery-worker-internal)
+    scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
+    -Q notify-internal-tasks 2> /dev/null
+    ;;
+  delivery-worker-receipts)
+    scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
+    -Q ses-callbacks 2> /dev/null
+    ;;
+  delivery-worker-service-callbacks)
+    scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
+    -Q service-callbacks 2> /dev/null
+    ;;
+  *)
+    echo "Unknown notify_app_name $NOTIFY_APP_NAME"
+    exit 1
+    ;;
+esac
--- a/tests/test_all_queues_used.py
+++ b/tests/test_all_queues_used.py
@@ -0,0 +1,18 @@
+from app.config import QueueNames
+
+
+def test_queue_names_set_in_paas_app_wrapper():
+    with open("scripts/paas_app_wrapper.sh", 'r') as stream:
+        search = ' -Q '
+
+        watched_queues = set()
+        for line in stream.readlines():
+            start_of_queue_arg = line.find(search)
+            if start_of_queue_arg > 0:
+                start_of_queue_names = start_of_queue_arg + len(search)
+                end_of_queue_names = line.find('2>') if '2>' in line else len(line)
+                watched_queues.update({q.strip() for q in line[start_of_queue_names:end_of_queue_names].split(',')})
+
+        # ses-callbacks isn't used in api (only used in SNS lambda)
+        ignored_queues = {'ses-callbacks'}
+        assert watched_queues == set(QueueNames.all_queues()) | ignored_queues
--- a/tests/test_manifest_delivery_base.py
+++ b/tests/test_manifest_delivery_base.py
@@ -1,23 +0,0 @@
-import yaml
-
-from app.config import QueueNames
-
-
-def test_queue_names_set_in_manifest_delivery_base_correctly():
-    with open("manifest-delivery-base.yml", 'r') as stream:
-        search = ' -Q '
-        yml_commands = [y['command'] for y in yaml.load(stream)['applications']]
-
-        watched_queues = set()
-        for command in yml_commands:
-            start_of_queue_arg = command.find(search)
-            if start_of_queue_arg > 0:
-                start_of_queue_names = start_of_queue_arg + len(search)
-                end_of_queue_names = command.find('2>') if '2>' in command else len(command)
-                watched_queues.update({q.strip() for q in command[start_of_queue_names:end_of_queue_names].split(',')})
-
-        # ses-callbacks isn't used in api (only used in SNS lambda)
-        ignored_queues = {'ses-callbacks'}
-        watched_queues -= ignored_queues
-
-        assert watched_queues == set(QueueNames.all_queues())