darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-05-06 17:18:43 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1543 from alphagov/generate-manifest-env

Browse Source 
Replaces user-provided services with manifest environment variables
This commit is contained in:
Alexey Bezhan
2018-01-09 11:02:16 +00:00
committed by GitHub
parent d1da39c388 40c27c6e70
commit 18c60c1418
12 changed files with 148 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
Makefile
View File

@@ -27,6 +27,8 @@ $(eval export CF_HOME)
CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml
NOTIFY_CREDENTIALS ?= ~/.notify-credentials
.PHONY: help
help:
@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -188,13 +190,23 @@ cf-login: ## Log in to Cloud Foundry
@echo "Logging in to Cloud Foundry on ${CF_API}"
@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
.PHONY: generate-manifest
generate-manifest:
$(if ${CF_APP},,$(error Must specify CF_APP))
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
$(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg))
$(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d))
@./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \
<(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg)
.PHONY: cf-deploy
cf-deploy: ## Deploys the app to Cloud Foundry
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
$(if ${CF_APP},,$(error Must specify CF_APP))
@cf app --guid ${CF_APP} || exit 1
cf rename ${CF_APP} ${CF_APP}-rollback
cf push ${CF_APP} -f ${CF_MANIFEST_FILE}
cf push ${CF_APP} -f <(make -s generate-manifest)
cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP}
cf stop ${CF_APP}-rollback
# sleep for 10 seconds to try and make sure that all worker threads (either web api or celery) have finished before we delete
@@ -211,7 +223,7 @@ cf-deploy-api-db-migration:
cf unbind-service notify-api-db-migration notify-db
cf unbind-service notify-api-db-migration notify-config
cf unbind-service notify-api-db-migration notify-aws
cf push notify-api-db-migration -f manifest-api-${CF_SPACE}.yml
cf push notify-api-db-migration -f -f <(make -s CF_APP=api generate-manifest)
cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
.PHONY: cf-check-api-db-migration-task
@@ -230,7 +242,7 @@ cf-rollback: ## Rollbacks the app to the previous release
cf-push:
$(if ${CF_APP},,$(error Must specify CF_APP))
cf target -o ${CF_ORG} -s ${CF_SPACE}
cf push ${CF_APP} -f ${CF_MANIFEST_FILE}
cf push ${CF_APP} -f <(make -s generate-manifest)
.PHONY: check-if-migrations-to-run
check-if-migrations-to-run:

67
app/cloudfoundry_config.py
View File

@@ -19,70 +19,3 @@ def set_config_env_vars(vcap_services):
vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'
# Notify common config
for s in vcap_services['user-provided']:
if s['name'] == 'notify-config':
extract_notify_config(s)
elif s['name'] == 'notify-aws':
extract_notify_aws_config(s)
elif s['name'] == 'hosted-graphite':
extract_hosted_graphite_config(s)
elif s['name'] == 'mmg':
extract_mmg_config(s)
elif s['name'] == 'firetext':
extract_firetext_config(s)
elif s['name'] == 'redis':
extract_redis_config(s)
elif s['name'] == 'performance-platform':
extract_performance_platform_config(s)
elif s['name'] == 'notify-template-preview':
extract_template_preview_config(s)
def extract_notify_config(notify_config):
os.environ['ADMIN_BASE_URL'] = notify_config['credentials']['admin_base_url']
os.environ['API_HOST_NAME'] = notify_config['credentials']['api_host_name']
os.environ['ADMIN_CLIENT_SECRET'] = notify_config['credentials']['admin_client_secret']
os.environ['SECRET_KEY'] = notify_config['credentials']['secret_key']
os.environ['DANGEROUS_SALT'] = notify_config['credentials']['dangerous_salt']
os.environ['SMS_INBOUND_WHITELIST'] = json.dumps(notify_config['credentials']['allow_ip_inbound_sms'])
os.environ['FIRETEXT_INBOUND_SMS_AUTH'] = json.dumps(notify_config['credentials']['firetext_inbound_sms_auth'])
os.environ['MMG_INBOUND_SMS_AUTH'] = json.dumps(notify_config['credentials']['mmg_inbound_sms_auth'])
os.environ['MMG_INBOUND_SMS_USERNAME'] = json.dumps(notify_config['credentials']['mmg_inbound_sms_username'])
os.environ['ROUTE_SECRET_KEY_1'] = notify_config['credentials']['route_secret_key_1']
os.environ['ROUTE_SECRET_KEY_2'] = notify_config['credentials']['route_secret_key_2']
def extract_performance_platform_config(performance_platform_config):
os.environ['PERFORMANCE_PLATFORM_ENDPOINTS'] = json.dumps(performance_platform_config['credentials'])
def extract_notify_aws_config(aws_config):
os.environ['NOTIFICATION_QUEUE_PREFIX'] = aws_config['credentials']['sqs_queue_prefix']
os.environ['AWS_ACCESS_KEY_ID'] = aws_config['credentials']['aws_access_key_id']
os.environ['AWS_SECRET_ACCESS_KEY'] = aws_config['credentials']['aws_secret_access_key']
def extract_hosted_graphite_config(hosted_graphite_config):
os.environ['STATSD_PREFIX'] = hosted_graphite_config['credentials']['statsd_prefix']
def extract_mmg_config(mmg_config):
os.environ['MMG_URL'] = mmg_config['credentials']['api_url']
os.environ['MMG_API_KEY'] = mmg_config['credentials']['api_key']
def extract_firetext_config(firetext_config):
os.environ['FIRETEXT_API_KEY'] = firetext_config['credentials']['api_key']
os.environ['LOADTESTING_API_KEY'] = firetext_config['credentials']['loadtesting_api_key']
def extract_redis_config(redis_config):
os.environ['REDIS_ENABLED'] = redis_config['credentials']['redis_enabled']
os.environ['REDIS_URL'] = redis_config['credentials']['redis_url']
def extract_template_preview_config(template_preview_config):
os.environ['TEMPLATE_PREVIEW_API_HOST'] = template_preview_config['credentials']['api_host']
os.environ['TEMPLATE_PREVIEW_API_KEY'] = template_preview_config['credentials']['api_key']

1
app/config.py
View File

@@ -103,7 +103,6 @@ class Config(object):
# Performance platform
PERFORMANCE_PLATFORM_ENABLED = False
PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/'
PERFORMANCE_PLATFORM_TOKEN = os.getenv('PERFORMANCE_PLATFORM_TOKEN')
# Logging
DEBUG = False

40
manifest-api-base.yml
View File

@@ -3,19 +3,45 @@
buildpack: python_buildpack
command: scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py --error-logfile /home/vcap/logs/gunicorn_error.log -w 5 -b 0.0.0.0:$PORT application
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
- redis
- performance-platform
env:
NOTIFY_APP_NAME: public-api
CW_APP_NAME: api
# required by cf run-task
FLASK_APP: application.py
# Credentials variables
ADMIN_BASE_URL: null
ADMIN_CLIENT_SECRET: null
API_HOST_NAME: null
DANGEROUS_SALT: null
SECRET_KEY: null
ROUTE_SECRET_KEY_1: null
ROUTE_SECRET_KEY_2: null
PERFORMANCE_PLATFORM_ENDPOINTS: null
NOTIFICATION_QUEUE_PREFIX: null
AWS_ACCESS_KEY_ID: null
AWS_SECRET_ACCESS_KEY: null
STATSD_PREFIX: null
MMG_URL: null
MMG_API_KEY: null
MMG_INBOUND_SMS_AUTH: null
MMG_INBOUND_SMS_USERNAME: null
FIRETEXT_API_KEY: null
LOADTESTING_API_KEY: null
FIRETEXT_INBOUND_SMS_AUTH: null
REDIS_ENABLED: null
REDIS_URL: null
TEMPLATE_PREVIEW_API_HOST: null
TEMPLATE_PREVIEW_API_KEY: null
instances: 1
memory: 1G

7
manifest-api-preview.yml
View File

@@ -3,14 +3,7 @@
inherit: manifest-api-base.yml
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
- redis
- performance-platform
- logit-ssl-syslog-drain
routes:

7
manifest-api-staging.yml
View File

@@ -2,14 +2,7 @@
inherit: manifest-api-base.yml
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
- redis
- performance-platform
- logit-ssl-syslog-drain
routes:

41
manifest-delivery-base.yml
View File

@@ -4,18 +4,43 @@ buildpack: python_buildpack
health-check-type: none
no-route: true
services:
- notify-aws
- notify-config
- notify-db
- notify-template-preview
- mmg
- firetext
- hosted-graphite
- redis
- performance-platform
instances: 1
memory: 1G
env:
# Credentials variables
ADMIN_BASE_URL: null
ADMIN_CLIENT_SECRET: null
API_HOST_NAME: null
DANGEROUS_SALT: null
SECRET_KEY: null
ROUTE_SECRET_KEY_1: null
ROUTE_SECRET_KEY_2: null
PERFORMANCE_PLATFORM_ENDPOINTS: null
NOTIFICATION_QUEUE_PREFIX: null
AWS_ACCESS_KEY_ID: null
AWS_SECRET_ACCESS_KEY: null
STATSD_PREFIX: null
MMG_URL: null
MMG_API_KEY: null
MMG_INBOUND_SMS_AUTH: null
MMG_INBOUND_SMS_USERNAME: null
FIRETEXT_API_KEY: null
LOADTESTING_API_KEY: null
FIRETEXT_INBOUND_SMS_AUTH: null
REDIS_ENABLED: null
REDIS_URL: null
TEMPLATE_PREVIEW_API_HOST: null
TEMPLATE_PREVIEW_API_KEY: null
applications:
- name: notify-delivery-celery-beat
command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO

8
manifest-delivery-preview.yml
View File

@@ -3,15 +3,7 @@
inherit: manifest-delivery-base.yml
services:
- notify-aws
- notify-config
- notify-db
- notify-template-preview
- mmg
- firetext
- hosted-graphite
- redis
- performance-platform
- logit-ssl-syslog-drain
memory: 1G

8
manifest-delivery-staging.yml
View File

@@ -3,15 +3,7 @@
inherit: manifest-delivery-base.yml
services:
- notify-aws
- notify-config
- notify-db
- notify-template-preview
- mmg
- firetext
- hosted-graphite
- redis
- performance-platform
- logit-ssl-syslog-drain
instances: 2

65
scripts/generate_manifest.py Executable file
View File

@@ -0,0 +1,65 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import json
import yaml
def merge_dicts(a, b):
if not (isinstance(a, dict) and isinstance(b, dict)):
raise ValueError("Error merging variables: '{}' and '{}'".format(
type(a).__name__, type(b).__name__
))
result = a.copy()
for key, val in b.items():
if isinstance(result.get(key), dict):
result[key] = merge_dicts(a[key], b[key])
else:
result[key] = val
return result
def load_manifest(manifest_file):
with open(manifest_file) as f:
manifest = yaml.load(f)
if 'inherit' in manifest:
inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit'))
manifest = merge_dicts(load_manifest(inherit_file), manifest)
return manifest
def load_variables(vars_files):
variables = {}
for vars_file in vars_files:
with open(vars_file) as f:
variables = merge_dicts(variables, yaml.load(f))
return {
k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v
for k, v in variables.items()
}
def paas_manifest(manifest_file, *vars_files):
"""Generate a PaaS manifest file from a Jinja2 template"""
manifest = load_manifest(manifest_file)
variables = load_variables(vars_files)
for key in manifest.get('env', {}):
if key in variables:
manifest['env'][key] = variables[key]
return yaml.dump(manifest, default_flow_style=False, allow_unicode=True)
if __name__ == "__main__":
print('---')
print(paas_manifest(*sys.argv[1:]))

3
scripts/run_app_paas.sh
View File

@@ -22,9 +22,6 @@ function configure_aws_logs {
aws configure set plugins.cwlogs cwlogs
export AWS_ACCESS_KEY_ID=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_access_key_id')
export AWS_SECRET_ACCESS_KEY=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_secret_access_key')
cat > /home/vcap/app/awslogs.conf << EOF
[general]
state_file = /home/vcap/logs/awslogs-state

217
tests/app/test_cloudfoundry_config.py
View File

@@ -6,70 +6,6 @@ import pytest
from app.cloudfoundry_config import extract_cloudfoundry_config, set_config_env_vars
@pytest.fixture
def notify_config():
return {
'name': 'notify-config',
'credentials': {
'admin_base_url': 'admin base url',
'api_host_name': 'api host name',
'admin_client_secret': 'admin client secret',
'secret_key': 'secret key',
'dangerous_salt': 'dangerous salt',
'allow_ip_inbound_sms': ['111.111.111.111', '100.100.100.100'],
'firetext_inbound_sms_auth': ['testkey'],
'mmg_inbound_sms_auth': ['testkey'],
'mmg_inbound_sms_username': ['username'],
'route_secret_key_1': "key_1",
'route_secret_key_2': ""
}
}
@pytest.fixture
def aws_config():
return {
'name': 'notify-aws',
'credentials': {
'sqs_queue_prefix': 'sqs queue prefix',
'aws_access_key_id': 'aws access key id',
'aws_secret_access_key': 'aws secret access key',
}
}
@pytest.fixture
def hosted_graphite_config():
return {
'name': 'hosted-graphite',
'credentials': {
'statsd_prefix': 'statsd prefix'
}
}
@pytest.fixture
def mmg_config():
return {
'name': 'mmg',
'credentials': {
'api_url': 'mmg api url',
'api_key': 'mmg api key'
}
}
@pytest.fixture
def firetext_config():
return {
'name': 'firetext',
'credentials': {
'api_key': 'firetext api key',
'loadtesting_api_key': 'loadtesting api key'
}
}
@pytest.fixture
def postgres_config():
return [
@@ -82,62 +18,10 @@ def postgres_config():
@pytest.fixture
def redis_config():
return {
'name': 'redis',
'credentials': {
'redis_enabled': '1',
'redis_url': 'redis url'
}
}
@pytest.fixture
def performance_platform_config():
return {
'name': 'performance-platform',
'credentials': {
'foo': 'my_token',
'bar': 'other_token'
}
}
@pytest.fixture
def template_preview_config():
return {
'name': 'notify-template-preview',
'credentials': {
'api_host': 'template-preview api host',
'api_key': 'template-preview api key'
}
}
@pytest.fixture
def cloudfoundry_config(
postgres_config,
notify_config,
aws_config,
hosted_graphite_config,
mmg_config,
firetext_config,
redis_config,
performance_platform_config,
template_preview_config
):
def cloudfoundry_config(postgres_config):
return {
'postgres': postgres_config,
'user-provided': [
notify_config,
aws_config,
hosted_graphite_config,
mmg_config,
firetext_config,
redis_config,
performance_platform_config,
template_preview_config
]
'user-provided': []
}
@@ -167,100 +51,3 @@ def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config):
assert 'foo' not in os.environ
assert 'bar' not in os.environ
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_notify_config():
extract_cloudfoundry_config()
assert os.environ['ADMIN_BASE_URL'] == 'admin base url'
assert os.environ['API_HOST_NAME'] == 'api host name'
assert os.environ['ADMIN_CLIENT_SECRET'] == 'admin client secret'
assert os.environ['SECRET_KEY'] == 'secret key'
assert os.environ['DANGEROUS_SALT'] == 'dangerous salt'
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_aws_config():
extract_cloudfoundry_config()
assert os.environ['NOTIFICATION_QUEUE_PREFIX'] == 'sqs queue prefix'
assert os.environ['AWS_ACCESS_KEY_ID'] == 'aws access key id'
assert os.environ['AWS_SECRET_ACCESS_KEY'] == 'aws secret access key'
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_hosted_graphite_config():
extract_cloudfoundry_config()
assert os.environ['STATSD_PREFIX'] == 'statsd prefix'
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_mmg_config():
extract_cloudfoundry_config()
assert os.environ['MMG_URL'] == 'mmg api url'
assert os.environ['MMG_API_KEY'] == 'mmg api key'
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_firetext_config():
extract_cloudfoundry_config()
assert os.environ['FIRETEXT_API_KEY'] == 'firetext api key'
assert os.environ['LOADTESTING_API_KEY'] == 'loadtesting api key'
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_redis_config():
extract_cloudfoundry_config()
assert os.environ['REDIS_ENABLED'] == '1'
assert os.environ['REDIS_URL'] == 'redis url'
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_sms_inbound_config():
extract_cloudfoundry_config()
assert os.environ['SMS_INBOUND_WHITELIST'] == json.dumps(['111.111.111.111', '100.100.100.100'])
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_firetext_inbound_sms_auth_config():
extract_cloudfoundry_config()
assert os.environ['FIRETEXT_INBOUND_SMS_AUTH'] == json.dumps(['testkey'])
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_mmg_inbound_sms_auth_config():
extract_cloudfoundry_config()
assert os.environ['MMG_INBOUND_SMS_AUTH'] == json.dumps(['testkey'])
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_mmg_inbound_sms_username_config():
extract_cloudfoundry_config()
assert os.environ['MMG_INBOUND_SMS_USERNAME'] == json.dumps(['username'])
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_performance_platform_config():
extract_cloudfoundry_config()
assert json.loads(os.environ['PERFORMANCE_PLATFORM_ENDPOINTS']) == {
'foo': 'my_token',
'bar': 'other_token'
}
@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
def test_template_preview_config():
extract_cloudfoundry_config()
assert os.environ['TEMPLATE_PREVIEW_API_HOST'] == 'template-preview api host'
assert os.environ['TEMPLATE_PREVIEW_API_KEY'] == 'template-preview api key'