darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-07 13:38:24 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2456 from alphagov/remove-multi-app-manifests

Browse Source 
Remove multi app manifests
This commit is contained in:
Leo Hemsted
2019-04-12 10:34:49 +01:00
committed by GitHub
parent 622b3e76f7 3277c65a7c
commit 5d26724022
19 changed files with 170 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
Makefile
View File

@@ -216,8 +216,11 @@ generate-manifest:
$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
@./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ @jinja2 --strict manifest.yml.j2 \
<(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) -D environment=${CF_SPACE} \
-D CF_APP=${CF_APP} \
--format=yaml \
<(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
.PHONY: cf-deploy .PHONY: cf-deploy
cf-deploy: ## Deploys the app to Cloud Foundry cf-deploy: ## Deploys the app to Cloud Foundry
@@ -242,7 +245,7 @@ cf-deploy-api-db-migration:
cf unbind-service notify-api-db-migration notify-db cf unbind-service notify-api-db-migration notify-db
cf unbind-service notify-api-db-migration notify-config cf unbind-service notify-api-db-migration notify-config
cf unbind-service notify-api-db-migration notify-aws cf unbind-service notify-api-db-migration notify-aws
cf push notify-api-db-migration -f <(make -s CF_APP=api generate-manifest) cf push notify-api-db-migration -f <(make -s CF_APP=notify-api-db-migration generate-manifest)
cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
.PHONY: cf-check-api-db-migration-task .PHONY: cf-check-api-db-migration-task

1
Procfile Normal file
View File

@@ -0,0 +1 @@
web: ./scripts/paas_app_wrapper.sh

67
manifest-api-base.yml
View File

@@ -1,67 +0,0 @@
---
buildpack: python_buildpack
command: unset GUNICORN_CMD_ARGS; scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
services:
- notify-db
- logit-ssl-syslog-drain
env:
NOTIFY_APP_NAME: public-api
CW_APP_NAME: api
# required by cf run-task
FLASK_APP: application.py
SQLALCHEMY_POOL_SIZE: 20
# Credentials variables
ADMIN_BASE_URL: null
ADMIN_CLIENT_SECRET: null
API_HOST_NAME: null
DANGEROUS_SALT: null
SECRET_KEY: null
ROUTE_SECRET_KEY_1: null
ROUTE_SECRET_KEY_2: null
CRONITOR_KEYS: null
PERFORMANCE_PLATFORM_ENDPOINTS: null
NOTIFICATION_QUEUE_PREFIX: null
AWS_ACCESS_KEY_ID: null
AWS_SECRET_ACCESS_KEY: null
STATSD_PREFIX: null
ZENDESK_API_KEY: null
MMG_URL: null
MMG_API_KEY: null
MMG_INBOUND_SMS_AUTH: null
MMG_INBOUND_SMS_USERNAME: null
FIRETEXT_API_KEY: null
LOADTESTING_API_KEY: null
FIRETEXT_INBOUND_SMS_AUTH: null
REDIS_ENABLED: null
REDIS_URL: null
TEMPLATE_PREVIEW_API_HOST: null
TEMPLATE_PREVIEW_API_KEY: null
DOCUMENT_DOWNLOAD_API_HOST: null
DOCUMENT_DOWNLOAD_API_KEY: null
instances: 1
memory: 1G
disk_quota: 2G
applications:
- name: notify-api
- name: notify-api-db-migration
command: sleep infinity
no-route: true
health-check-type: none
instances: 1
memory: 128M

10
manifest-api-preview.yml
View File

@@ -1,10 +0,0 @@
---
inherit: manifest-api-base.yml
routes:
- route: notify-api-preview.cloudapps.digital
- route: api.notify.works
instances: 1
memory: 1G

9
manifest-api-production.yml
View File

@@ -1,9 +0,0 @@
---
inherit: manifest-api-base.yml
routes:
- route: notify-api-production.cloudapps.digital
- route: api.notifications.service.gov.uk
instances: 2
memory: 1G

6
manifest-api-sandbox.yml
View File

@@ -1,6 +0,0 @@
---
inherit: manifest-api-base.yml
routes:
- route: notify-api-sandbox.cloudapps.digital

9
manifest-api-staging.yml
View File

@@ -1,9 +0,0 @@
---
inherit: manifest-api-base.yml
routes:
- route: notify-api-staging.cloudapps.digital
- route: api.staging-notify.works
instances: 2
memory: 1G

107
manifest-delivery-base.yml
View File

@@ -1,107 +0,0 @@
---
buildpack: python_buildpack
health-check-type: none
no-route: true
services:
- notify-db
- logit-ssl-syslog-drain
instances: 1
memory: 1G
env:
# Credentials variables
ADMIN_BASE_URL: null
ADMIN_CLIENT_SECRET: null
API_HOST_NAME: null
DANGEROUS_SALT: null
SECRET_KEY: null
ROUTE_SECRET_KEY_1: null
ROUTE_SECRET_KEY_2: null
CRONITOR_KEYS: null
PERFORMANCE_PLATFORM_ENDPOINTS: null
NOTIFICATION_QUEUE_PREFIX: null
AWS_ACCESS_KEY_ID: null
AWS_SECRET_ACCESS_KEY: null
STATSD_PREFIX: null
ZENDESK_API_KEY: null
MMG_URL: null
MMG_API_KEY: null
MMG_INBOUND_SMS_AUTH: null
MMG_INBOUND_SMS_USERNAME: null
FIRETEXT_API_KEY: null
LOADTESTING_API_KEY: null
FIRETEXT_INBOUND_SMS_AUTH: null
REDIS_ENABLED: null
REDIS_URL: null
TEMPLATE_PREVIEW_API_HOST: null
TEMPLATE_PREVIEW_API_KEY: null
SQLALCHEMY_POOL_SIZE: 1
applications:
- name: notify-delivery-celery-beat
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO
instances: 1
memory: 128M
env:
NOTIFY_APP_NAME: delivery-celery-beat
- name: notify-delivery-worker-database
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks 2> /dev/null
env:
NOTIFY_APP_NAME: delivery-worker-database
- name: notify-delivery-worker-research
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks 2> /dev/null
env:
NOTIFY_APP_NAME: delivery-worker-research
- name: notify-delivery-worker-sender
command: scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO -Q send-sms-tasks,send-email-tasks
memory: 3G
disk_quota: 2G
env:
NOTIFY_APP_NAME: delivery-worker-sender
- name: notify-delivery-worker-periodic
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks 2> /dev/null
instances: 1
env:
NOTIFY_APP_NAME: delivery-worker-periodic
- name: notify-delivery-worker-priority
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks 2> /dev/null
env:
NOTIFY_APP_NAME: delivery-worker-priority
- name: notify-delivery-worker
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null
env:
NOTIFY_APP_NAME: delivery-worker
# Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority
- name: notify-delivery-worker-internal
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q notify-internal-tasks 2> /dev/null
env:
NOTIFY_APP_NAME: delivery-worker-internal
- name: notify-delivery-worker-receipts
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q ses-callbacks 2> /dev/null
env:
NOTIFY_APP_NAME: delivery-worker-receipts
- name: notify-delivery-worker-service-callbacks
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q service-callbacks 2> /dev/null
disk_quota: 2G
env:
NOTIFY_APP_NAME: delivery-worker-service-callbacks

5
manifest-delivery-preview.yml
View File

@@ -1,5 +0,0 @@
---
inherit: manifest-delivery-base.yml
memory: 1G

6
manifest-delivery-production.yml
View File

@@ -1,6 +0,0 @@
---
inherit: manifest-delivery-base.yml
instances: 2
memory: 1G

3
manifest-delivery-sandbox.yml
View File

@@ -1,3 +0,0 @@
---
inherit: manifest-delivery-base.yml

6
manifest-delivery-staging.yml
View File

@@ -1,6 +0,0 @@
---
inherit: manifest-delivery-base.yml
instances: 2
memory: 1G

90
manifest.yml.j2 Normal file
View File

@@ -0,0 +1,90 @@
{%- set app_vars = {
'notify-api': {'NOTIFY_APP_NAME': 'api', 'disk_quota': '2G', 'sqlalchemy_pool_size': 20, 'routes': {
'preview': ['notify-api-preview.cloudapps.digital', 'api.notify.works'],
'staging': ['notify-api-staging.cloudapps.digital', 'api.staging-notify.works'],
'production': ['notify-api-production.cloudapps.digital', 'api.notifications.service.gov.uk'],
}
},
'notify-api-db-migration': {'NOTIFY_APP_NAME': 'api', 'instances': 0},
'notify-delivery-celery-beat': {'memory': '128M'},
'notify-delivery-worker-database': {},
'notify-delivery-worker-research': {},
'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'},
'notify-delivery-worker-periodic': {},
'notify-delivery-worker-priority': {},
'notify-delivery-worker': {},
'notify-delivery-worker-internal': {},
'notify-delivery-worker-receipts': {},
'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'},
} -%}
{%- set app = app_vars[CF_APP] -%}
---
applications:
- name: {{ CF_APP }}
buildpack: python_buildpack
{% if 'instances' in app %}
instances: {{ app['instances'] }}
{%- endif %}
memory: {{ app.get('memory', '1G') }}
disk_quota: {{ app.get('disk_quota', '1G')}}
{% if 'routes' in app -%}
routes:
{%- for route in app['routes'][environment] %}
- route: {{ route }}
{%- endfor -%}
{%- else -%}
health-check-type: none
no-route: true
{% endif %}
services:
- notify-db
- logit-ssl-syslog-drain
env:
NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }}
SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }}
FLASK_APP: application.py
# Credentials variables
ADMIN_BASE_URL: '{{ ADMIN_BASE_URL }}'
ADMIN_CLIENT_SECRET: '{{ ADMIN_CLIENT_SECRET }}'
API_HOST_NAME: '{{ API_HOST_NAME }}'
DANGEROUS_SALT: '{{ DANGEROUS_SALT }}'
SECRET_KEY: '{{ SECRET_KEY }}'
ROUTE_SECRET_KEY_1: '{{ ROUTE_SECRET_KEY_1 }}'
ROUTE_SECRET_KEY_2: '{{ ROUTE_SECRET_KEY_2 }}'
CRONITOR_KEYS: '{{ CRONITOR_KEYS | tojson }}'
PERFORMANCE_PLATFORM_ENDPOINTS: '{{ PERFORMANCE_PLATFORM_ENDPOINTS | tojson }}'
DOCUMENT_DOWNLOAD_API_HOST: '{{ DOCUMENT_DOWNLOAD_API_HOST }}'
DOCUMENT_DOWNLOAD_API_KEY: '{{ DOCUMENT_DOWNLOAD_API_KEY }}'
NOTIFICATION_QUEUE_PREFIX: '{{ NOTIFICATION_QUEUE_PREFIX }}'
AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}'
AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}'
STATSD_PREFIX: '{{ STATSD_PREFIX }}'
ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}'
MMG_URL: '{{ MMG_URL }}'
MMG_API_KEY: '{{ MMG_API_KEY }}'
MMG_INBOUND_SMS_AUTH: '{{ MMG_INBOUND_SMS_AUTH | tojson }}'
MMG_INBOUND_SMS_USERNAME: '{{ MMG_INBOUND_SMS_USERNAME | tojson }}'
FIRETEXT_API_KEY: '{{ FIRETEXT_API_KEY }}'
LOADTESTING_API_KEY: '{{ LOADTESTING_API_KEY }}'
FIRETEXT_INBOUND_SMS_AUTH: '{{ FIRETEXT_INBOUND_SMS_AUTH | tojson }}'
REDIS_ENABLED: '{{ REDIS_ENABLED }}'
REDIS_URL: '{{ REDIS_URL }}'
TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}'
TEMPLATE_PREVIEW_API_KEY: '{{ TEMPLATE_PREVIEW_API_KEY }}'

10
requirements.txt
View File

@@ -40,12 +40,12 @@ alembic==1.0.8
amqp==1.4.9 amqp==1.4.9
anyjson==0.3.3 anyjson==0.3.3
attrs==19.1.0 attrs==19.1.0
awscli==1.16.129 awscli==1.16.140
bcrypt==3.1.6 bcrypt==3.1.6
billiard==3.3.0.23 billiard==3.3.0.23
bleach==3.1.0 bleach==3.1.0
boto3==1.6.16 boto3==1.6.16
botocore==1.12.119 botocore==1.12.130
certifi==2019.3.9 certifi==2019.3.9
chardet==3.0.4 chardet==3.0.4
Click==7.0 Click==7.0
@@ -55,7 +55,7 @@ Flask-Redis==0.3.0
future==0.17.1 future==0.17.1
greenlet==0.4.15 greenlet==0.4.15
idna==2.8 idna==2.8
Jinja2==2.10 Jinja2==2.10.1
jmespath==0.9.4 jmespath==0.9.4
kombu==3.0.37 kombu==3.0.37
Mako==1.0.8 Mako==1.0.8
@@ -71,7 +71,7 @@ pyrsistent==0.14.11
python-dateutil==2.8.0 python-dateutil==2.8.0
python-editor==1.0.4 python-editor==1.0.4
python-json-logger==0.1.10 python-json-logger==0.1.10
pytz==2018.9 pytz==2019.1
PyYAML==3.12 PyYAML==3.12
redis==3.2.1 redis==3.2.1
requests==2.21.0 requests==2.21.0
@@ -82,4 +82,4 @@ smartypants==2.0.1
statsd==3.3.0 statsd==3.3.0
urllib3==1.24.1 urllib3==1.24.1
webencodings==0.5.1 webencodings==0.5.1
Werkzeug==0.15.1 Werkzeug==0.15.2

2
requirements_for_test.txt
View File

@@ -12,3 +12,5 @@ requests-mock==1.5.2
# optional requirements for jsonschema # optional requirements for jsonschema
strict-rfc3339==0.7 strict-rfc3339==0.7
rfc3987==1.3.8 rfc3987==1.3.8
# used for creating manifest file locally
jinja2-cli[yaml]==0.6.0

65
scripts/generate_manifest.py
View File

@@ -1,65 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import json
import yaml
def merge_dicts(a, b):
if not (isinstance(a, dict) and isinstance(b, dict)):
raise ValueError("Error merging variables: '{}' and '{}'".format(
type(a).__name__, type(b).__name__
))
result = a.copy()
for key, val in b.items():
if isinstance(result.get(key), dict):
result[key] = merge_dicts(a[key], b[key])
else:
result[key] = val
return result
def load_manifest(manifest_file):
with open(manifest_file) as f:
manifest = yaml.load(f)
if 'inherit' in manifest:
inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit'))
manifest = merge_dicts(load_manifest(inherit_file), manifest)
return manifest
def load_variables(vars_files):
variables = {}
for vars_file in vars_files:
with open(vars_file) as f:
variables = merge_dicts(variables, yaml.load(f))
return {
k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v
for k, v in variables.items()
}
def paas_manifest(manifest_file, *vars_files):
"""Generate a PaaS manifest file from a Jinja2 template"""
manifest = load_manifest(manifest_file)
variables = load_variables(vars_files)
for key in manifest.get('env', {}):
if key in variables:
manifest['env'][key] = variables[key]
return yaml.dump(manifest, default_flow_style=False, allow_unicode=True)
if __name__ == "__main__":
print('---')
print(paas_manifest(*sys.argv[1:]))

48
scripts/paas_app_wrapper.sh Executable file
View File

@@ -0,0 +1,48 @@
#!/bin/bash
case $NOTIFY_APP_NAME in
api)
unset GUNICORN_CMD_ARGS
scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
;;
delivery-worker)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
-Q job-tasks,retry-tasks,create-letters-pdf-tasks,letter-tasks 2> /dev/null
;;
delivery-worker-database)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
-Q database-tasks 2> /dev/null
;;
delivery-worker-research)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \
-Q research-mode-tasks 2> /dev/null
;;
delivery-worker-sender)
scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO \
-Q send-sms-tasks,send-email-tasks
;;
delivery-worker-periodic)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \
-Q periodic-tasks,statistics-tasks 2> /dev/null
;;
delivery-worker-priority)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=5 \
-Q priority-tasks 2> /dev/null
;;
# Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority
delivery-worker-internal)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
-Q notify-internal-tasks 2> /dev/null
;;
delivery-worker-receipts)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
-Q ses-callbacks 2> /dev/null
;;
delivery-worker-service-callbacks)
scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 \
-Q service-callbacks 2> /dev/null
;;
*)
echo "Unknown notify_app_name $NOTIFY_APP_NAME"
exit 1
;;
esac

18
tests/test_all_queues_used.py Normal file
View File

@@ -0,0 +1,18 @@
from app.config import QueueNames
def test_queue_names_set_in_paas_app_wrapper():
with open("scripts/paas_app_wrapper.sh", 'r') as stream:
search = ' -Q '
watched_queues = set()
for line in stream.readlines():
start_of_queue_arg = line.find(search)
if start_of_queue_arg > 0:
start_of_queue_names = start_of_queue_arg + len(search)
end_of_queue_names = line.find('2>') if '2>' in line else len(line)
watched_queues.update({q.strip() for q in line[start_of_queue_names:end_of_queue_names].split(',')})
# ses-callbacks isn't used in api (only used in SNS lambda)
ignored_queues = {'ses-callbacks'}
assert watched_queues == set(QueueNames.all_queues()) | ignored_queues

23
tests/test_manifest_delivery_base.py
View File

@@ -1,23 +0,0 @@
import yaml
from app.config import QueueNames
def test_queue_names_set_in_manifest_delivery_base_correctly():
with open("manifest-delivery-base.yml", 'r') as stream:
search = ' -Q '
yml_commands = [y['command'] for y in yaml.load(stream)['applications']]
watched_queues = set()
for command in yml_commands:
start_of_queue_arg = command.find(search)
if start_of_queue_arg > 0:
start_of_queue_names = start_of_queue_arg + len(search)
end_of_queue_names = command.find('2>') if '2>' in command else len(command)
watched_queues.update({q.strip() for q in command[start_of_queue_names:end_of_queue_names].split(',')})
# ses-callbacks isn't used in api (only used in SNS lambda)
ignored_queues = {'ses-callbacks'}
watched_queues -= ignored_queues
assert watched_queues == set(QueueNames.all_queues())