diff --git a/Makefile b/Makefile index 3b2d46d16..a09f929dd 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,8 @@ $(eval export CF_HOME) CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml +NOTIFY_CREDENTIALS ?= ~/.notify-credentials + .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -188,13 +190,23 @@ cf-login: ## Log in to Cloud Foundry @echo "Logging in to Cloud Foundry on ${CF_API}" @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" +.PHONY: generate-manifest +generate-manifest: + $(if ${CF_APP},,$(error Must specify CF_APP)) + $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) + $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) + $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) + + @./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ + <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) + .PHONY: cf-deploy cf-deploy: ## Deploys the app to Cloud Foundry $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) $(if ${CF_APP},,$(error Must specify CF_APP)) @cf app --guid ${CF_APP} || exit 1 cf rename ${CF_APP} ${CF_APP}-rollback - cf push ${CF_APP} -f ${CF_MANIFEST_FILE} + cf push ${CF_APP} -f <(make -s generate-manifest) cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP} cf stop ${CF_APP}-rollback # sleep for 10 seconds to try and make sure that all worker threads (either web api or celery) have finished before we delete @@ -211,7 +223,7 @@ cf-deploy-api-db-migration: cf unbind-service notify-api-db-migration notify-db cf unbind-service notify-api-db-migration notify-config cf unbind-service notify-api-db-migration notify-aws - cf push notify-api-db-migration -f manifest-api-${CF_SPACE}.yml + cf push notify-api-db-migration -f -f <(make -s CF_APP=api generate-manifest) cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task @@ -230,7 +242,7 @@ cf-rollback: ## Rollbacks the app to the previous release cf-push: $(if ${CF_APP},,$(error Must specify CF_APP)) cf target -o ${CF_ORG} -s ${CF_SPACE} - cf push ${CF_APP} -f ${CF_MANIFEST_FILE} + cf push ${CF_APP} -f <(make -s generate-manifest) .PHONY: check-if-migrations-to-run check-if-migrations-to-run: diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 3a21a7a29..5cb7c437f 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -19,70 +19,3 @@ def set_config_env_vars(vcap_services): vcap_application = json.loads(os.environ['VCAP_APPLICATION']) os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name'] os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log' - - # Notify common config - for s in vcap_services['user-provided']: - if s['name'] == 'notify-config': - extract_notify_config(s) - elif s['name'] == 'notify-aws': - extract_notify_aws_config(s) - elif s['name'] == 'hosted-graphite': - extract_hosted_graphite_config(s) - elif s['name'] == 'mmg': - extract_mmg_config(s) - elif s['name'] == 'firetext': - extract_firetext_config(s) - elif s['name'] == 'redis': - extract_redis_config(s) - elif s['name'] == 'performance-platform': - extract_performance_platform_config(s) - elif s['name'] == 'notify-template-preview': - extract_template_preview_config(s) - - -def extract_notify_config(notify_config): - os.environ['ADMIN_BASE_URL'] = notify_config['credentials']['admin_base_url'] - os.environ['API_HOST_NAME'] = notify_config['credentials']['api_host_name'] - os.environ['ADMIN_CLIENT_SECRET'] = notify_config['credentials']['admin_client_secret'] - os.environ['SECRET_KEY'] = notify_config['credentials']['secret_key'] - os.environ['DANGEROUS_SALT'] = notify_config['credentials']['dangerous_salt'] - os.environ['SMS_INBOUND_WHITELIST'] = json.dumps(notify_config['credentials']['allow_ip_inbound_sms']) - os.environ['FIRETEXT_INBOUND_SMS_AUTH'] = json.dumps(notify_config['credentials']['firetext_inbound_sms_auth']) - os.environ['MMG_INBOUND_SMS_AUTH'] = json.dumps(notify_config['credentials']['mmg_inbound_sms_auth']) - os.environ['MMG_INBOUND_SMS_USERNAME'] = json.dumps(notify_config['credentials']['mmg_inbound_sms_username']) - os.environ['ROUTE_SECRET_KEY_1'] = notify_config['credentials']['route_secret_key_1'] - os.environ['ROUTE_SECRET_KEY_2'] = notify_config['credentials']['route_secret_key_2'] - - -def extract_performance_platform_config(performance_platform_config): - os.environ['PERFORMANCE_PLATFORM_ENDPOINTS'] = json.dumps(performance_platform_config['credentials']) - - -def extract_notify_aws_config(aws_config): - os.environ['NOTIFICATION_QUEUE_PREFIX'] = aws_config['credentials']['sqs_queue_prefix'] - os.environ['AWS_ACCESS_KEY_ID'] = aws_config['credentials']['aws_access_key_id'] - os.environ['AWS_SECRET_ACCESS_KEY'] = aws_config['credentials']['aws_secret_access_key'] - - -def extract_hosted_graphite_config(hosted_graphite_config): - os.environ['STATSD_PREFIX'] = hosted_graphite_config['credentials']['statsd_prefix'] - - -def extract_mmg_config(mmg_config): - os.environ['MMG_URL'] = mmg_config['credentials']['api_url'] - os.environ['MMG_API_KEY'] = mmg_config['credentials']['api_key'] - - -def extract_firetext_config(firetext_config): - os.environ['FIRETEXT_API_KEY'] = firetext_config['credentials']['api_key'] - os.environ['LOADTESTING_API_KEY'] = firetext_config['credentials']['loadtesting_api_key'] - - -def extract_redis_config(redis_config): - os.environ['REDIS_ENABLED'] = redis_config['credentials']['redis_enabled'] - os.environ['REDIS_URL'] = redis_config['credentials']['redis_url'] - - -def extract_template_preview_config(template_preview_config): - os.environ['TEMPLATE_PREVIEW_API_HOST'] = template_preview_config['credentials']['api_host'] - os.environ['TEMPLATE_PREVIEW_API_KEY'] = template_preview_config['credentials']['api_key'] diff --git a/app/config.py b/app/config.py index 9c6a11055..3368ca68c 100644 --- a/app/config.py +++ b/app/config.py @@ -103,7 +103,6 @@ class Config(object): # Performance platform PERFORMANCE_PLATFORM_ENABLED = False PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/' - PERFORMANCE_PLATFORM_TOKEN = os.getenv('PERFORMANCE_PLATFORM_TOKEN') # Logging DEBUG = False diff --git a/manifest-api-base.yml b/manifest-api-base.yml index 9afb9f860..eea8d8459 100644 --- a/manifest-api-base.yml +++ b/manifest-api-base.yml @@ -3,19 +3,45 @@ buildpack: python_buildpack command: scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py --error-logfile /home/vcap/logs/gunicorn_error.log -w 5 -b 0.0.0.0:$PORT application services: - - notify-aws - - notify-config - notify-db - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform env: NOTIFY_APP_NAME: public-api CW_APP_NAME: api # required by cf run-task FLASK_APP: application.py + + # Credentials variables + ADMIN_BASE_URL: null + ADMIN_CLIENT_SECRET: null + API_HOST_NAME: null + DANGEROUS_SALT: null + SECRET_KEY: null + ROUTE_SECRET_KEY_1: null + ROUTE_SECRET_KEY_2: null + + PERFORMANCE_PLATFORM_ENDPOINTS: null + + NOTIFICATION_QUEUE_PREFIX: null + AWS_ACCESS_KEY_ID: null + AWS_SECRET_ACCESS_KEY: null + + STATSD_PREFIX: null + + MMG_URL: null + MMG_API_KEY: null + MMG_INBOUND_SMS_AUTH: null + MMG_INBOUND_SMS_USERNAME: null + + FIRETEXT_API_KEY: null + LOADTESTING_API_KEY: null + FIRETEXT_INBOUND_SMS_AUTH: null + + REDIS_ENABLED: null + REDIS_URL: null + + TEMPLATE_PREVIEW_API_HOST: null + TEMPLATE_PREVIEW_API_KEY: null + instances: 1 memory: 1G diff --git a/manifest-api-preview.yml b/manifest-api-preview.yml index 1e4174cb1..406b53cb5 100644 --- a/manifest-api-preview.yml +++ b/manifest-api-preview.yml @@ -3,14 +3,7 @@ inherit: manifest-api-base.yml services: - - notify-aws - - notify-config - notify-db - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform - logit-ssl-syslog-drain routes: diff --git a/manifest-api-staging.yml b/manifest-api-staging.yml index 868d2853b..30304c2fd 100644 --- a/manifest-api-staging.yml +++ b/manifest-api-staging.yml @@ -2,14 +2,7 @@ inherit: manifest-api-base.yml services: - - notify-aws - - notify-config - notify-db - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform - logit-ssl-syslog-drain routes: diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml index 26a12ea0e..78cff9fa5 100644 --- a/manifest-delivery-base.yml +++ b/manifest-delivery-base.yml @@ -4,18 +4,43 @@ buildpack: python_buildpack health-check-type: none no-route: true services: - - notify-aws - - notify-config - notify-db - - notify-template-preview - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform instances: 1 memory: 1G +env: + # Credentials variables + ADMIN_BASE_URL: null + ADMIN_CLIENT_SECRET: null + API_HOST_NAME: null + DANGEROUS_SALT: null + SECRET_KEY: null + ROUTE_SECRET_KEY_1: null + ROUTE_SECRET_KEY_2: null + + PERFORMANCE_PLATFORM_ENDPOINTS: null + + NOTIFICATION_QUEUE_PREFIX: null + AWS_ACCESS_KEY_ID: null + AWS_SECRET_ACCESS_KEY: null + + STATSD_PREFIX: null + + MMG_URL: null + MMG_API_KEY: null + MMG_INBOUND_SMS_AUTH: null + MMG_INBOUND_SMS_USERNAME: null + + FIRETEXT_API_KEY: null + LOADTESTING_API_KEY: null + FIRETEXT_INBOUND_SMS_AUTH: null + + REDIS_ENABLED: null + REDIS_URL: null + + TEMPLATE_PREVIEW_API_HOST: null + TEMPLATE_PREVIEW_API_KEY: null + applications: - name: notify-delivery-celery-beat command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO diff --git a/manifest-delivery-preview.yml b/manifest-delivery-preview.yml index 8de718214..f86d53582 100644 --- a/manifest-delivery-preview.yml +++ b/manifest-delivery-preview.yml @@ -3,15 +3,7 @@ inherit: manifest-delivery-base.yml services: - - notify-aws - - notify-config - notify-db - - notify-template-preview - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform - logit-ssl-syslog-drain memory: 1G diff --git a/manifest-delivery-staging.yml b/manifest-delivery-staging.yml index 7e7f37c0e..973b43a85 100644 --- a/manifest-delivery-staging.yml +++ b/manifest-delivery-staging.yml @@ -3,15 +3,7 @@ inherit: manifest-delivery-base.yml services: - - notify-aws - - notify-config - notify-db - - notify-template-preview - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform - logit-ssl-syslog-drain instances: 2 diff --git a/scripts/generate_manifest.py b/scripts/generate_manifest.py new file mode 100755 index 000000000..8814049c8 --- /dev/null +++ b/scripts/generate_manifest.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys + +import json +import yaml + + +def merge_dicts(a, b): + if not (isinstance(a, dict) and isinstance(b, dict)): + raise ValueError("Error merging variables: '{}' and '{}'".format( + type(a).__name__, type(b).__name__ + )) + + result = a.copy() + for key, val in b.items(): + if isinstance(result.get(key), dict): + result[key] = merge_dicts(a[key], b[key]) + else: + result[key] = val + + return result + + +def load_manifest(manifest_file): + with open(manifest_file) as f: + manifest = yaml.load(f) + + if 'inherit' in manifest: + inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit')) + manifest = merge_dicts(load_manifest(inherit_file), manifest) + + return manifest + + +def load_variables(vars_files): + variables = {} + for vars_file in vars_files: + with open(vars_file) as f: + variables = merge_dicts(variables, yaml.load(f)) + + return { + k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v + for k, v in variables.items() + } + + +def paas_manifest(manifest_file, *vars_files): + """Generate a PaaS manifest file from a Jinja2 template""" + + manifest = load_manifest(manifest_file) + variables = load_variables(vars_files) + + for key in manifest.get('env', {}): + if key in variables: + manifest['env'][key] = variables[key] + + return yaml.dump(manifest, default_flow_style=False, allow_unicode=True) + + +if __name__ == "__main__": + print('---') + print(paas_manifest(*sys.argv[1:])) diff --git a/scripts/run_app_paas.sh b/scripts/run_app_paas.sh index 2a4d2988d..cf0c7ccc7 100755 --- a/scripts/run_app_paas.sh +++ b/scripts/run_app_paas.sh @@ -22,9 +22,6 @@ function configure_aws_logs { aws configure set plugins.cwlogs cwlogs - export AWS_ACCESS_KEY_ID=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_access_key_id') - export AWS_SECRET_ACCESS_KEY=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_secret_access_key') - cat > /home/vcap/app/awslogs.conf << EOF [general] state_file = /home/vcap/logs/awslogs-state diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index aafd894c7..ccf238d49 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -6,70 +6,6 @@ import pytest from app.cloudfoundry_config import extract_cloudfoundry_config, set_config_env_vars -@pytest.fixture -def notify_config(): - return { - 'name': 'notify-config', - 'credentials': { - 'admin_base_url': 'admin base url', - 'api_host_name': 'api host name', - 'admin_client_secret': 'admin client secret', - 'secret_key': 'secret key', - 'dangerous_salt': 'dangerous salt', - 'allow_ip_inbound_sms': ['111.111.111.111', '100.100.100.100'], - 'firetext_inbound_sms_auth': ['testkey'], - 'mmg_inbound_sms_auth': ['testkey'], - 'mmg_inbound_sms_username': ['username'], - 'route_secret_key_1': "key_1", - 'route_secret_key_2': "" - } - } - - -@pytest.fixture -def aws_config(): - return { - 'name': 'notify-aws', - 'credentials': { - 'sqs_queue_prefix': 'sqs queue prefix', - 'aws_access_key_id': 'aws access key id', - 'aws_secret_access_key': 'aws secret access key', - } - } - - -@pytest.fixture -def hosted_graphite_config(): - return { - 'name': 'hosted-graphite', - 'credentials': { - 'statsd_prefix': 'statsd prefix' - } - } - - -@pytest.fixture -def mmg_config(): - return { - 'name': 'mmg', - 'credentials': { - 'api_url': 'mmg api url', - 'api_key': 'mmg api key' - } - } - - -@pytest.fixture -def firetext_config(): - return { - 'name': 'firetext', - 'credentials': { - 'api_key': 'firetext api key', - 'loadtesting_api_key': 'loadtesting api key' - } - } - - @pytest.fixture def postgres_config(): return [ @@ -82,62 +18,10 @@ def postgres_config(): @pytest.fixture -def redis_config(): - return { - 'name': 'redis', - 'credentials': { - 'redis_enabled': '1', - 'redis_url': 'redis url' - } - } - - -@pytest.fixture -def performance_platform_config(): - return { - 'name': 'performance-platform', - 'credentials': { - 'foo': 'my_token', - 'bar': 'other_token' - } - } - - -@pytest.fixture -def template_preview_config(): - return { - 'name': 'notify-template-preview', - 'credentials': { - 'api_host': 'template-preview api host', - 'api_key': 'template-preview api key' - } - } - - -@pytest.fixture -def cloudfoundry_config( - postgres_config, - notify_config, - aws_config, - hosted_graphite_config, - mmg_config, - firetext_config, - redis_config, - performance_platform_config, - template_preview_config -): +def cloudfoundry_config(postgres_config): return { 'postgres': postgres_config, - 'user-provided': [ - notify_config, - aws_config, - hosted_graphite_config, - mmg_config, - firetext_config, - redis_config, - performance_platform_config, - template_preview_config - ] + 'user-provided': [] } @@ -167,100 +51,3 @@ def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config): assert 'foo' not in os.environ assert 'bar' not in os.environ - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_notify_config(): - extract_cloudfoundry_config() - - assert os.environ['ADMIN_BASE_URL'] == 'admin base url' - assert os.environ['API_HOST_NAME'] == 'api host name' - assert os.environ['ADMIN_CLIENT_SECRET'] == 'admin client secret' - assert os.environ['SECRET_KEY'] == 'secret key' - assert os.environ['DANGEROUS_SALT'] == 'dangerous salt' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_aws_config(): - extract_cloudfoundry_config() - - assert os.environ['NOTIFICATION_QUEUE_PREFIX'] == 'sqs queue prefix' - assert os.environ['AWS_ACCESS_KEY_ID'] == 'aws access key id' - assert os.environ['AWS_SECRET_ACCESS_KEY'] == 'aws secret access key' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_hosted_graphite_config(): - extract_cloudfoundry_config() - - assert os.environ['STATSD_PREFIX'] == 'statsd prefix' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_mmg_config(): - extract_cloudfoundry_config() - - assert os.environ['MMG_URL'] == 'mmg api url' - assert os.environ['MMG_API_KEY'] == 'mmg api key' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_firetext_config(): - extract_cloudfoundry_config() - - assert os.environ['FIRETEXT_API_KEY'] == 'firetext api key' - assert os.environ['LOADTESTING_API_KEY'] == 'loadtesting api key' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_redis_config(): - extract_cloudfoundry_config() - - assert os.environ['REDIS_ENABLED'] == '1' - assert os.environ['REDIS_URL'] == 'redis url' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_sms_inbound_config(): - extract_cloudfoundry_config() - - assert os.environ['SMS_INBOUND_WHITELIST'] == json.dumps(['111.111.111.111', '100.100.100.100']) - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_firetext_inbound_sms_auth_config(): - extract_cloudfoundry_config() - - assert os.environ['FIRETEXT_INBOUND_SMS_AUTH'] == json.dumps(['testkey']) - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_mmg_inbound_sms_auth_config(): - extract_cloudfoundry_config() - - assert os.environ['MMG_INBOUND_SMS_AUTH'] == json.dumps(['testkey']) - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_mmg_inbound_sms_username_config(): - extract_cloudfoundry_config() - - assert os.environ['MMG_INBOUND_SMS_USERNAME'] == json.dumps(['username']) - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_performance_platform_config(): - extract_cloudfoundry_config() - - assert json.loads(os.environ['PERFORMANCE_PLATFORM_ENDPOINTS']) == { - 'foo': 'my_token', - 'bar': 'other_token' - } - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_template_preview_config(): - extract_cloudfoundry_config() - - assert os.environ['TEMPLATE_PREVIEW_API_HOST'] == 'template-preview api host' - assert os.environ['TEMPLATE_PREVIEW_API_KEY'] == 'template-preview api key'