From da1a7ceb0dafdeb14b366139e0ec13fe95c55c00 Mon Sep 17 00:00:00 2001 From: Alexey Bezhan Date: Wed, 3 Jan 2018 16:27:44 +0000 Subject: [PATCH 1/7] Add a script to generate PaaS manifest with environment variables `./scripts/generate_manifest.py` takes a path to a PaaS manifest file and a list of variable files and prints a single CloudFoundry manifest. The generated manifest replaces all `inherit` keys by loading the data from parent manifests. This allows us to pipe the script output directly to CF CLI, without saving it to disk, which minimises the risk of it being accidentally included in the deployment artefact. The combined manifest might differ from the results produced by CF CLI itself, so the original manifest shouldn't normally be used on its own. After combining the manifests the script will load and parse all listed variable files and add them to the manifest's `env` by merging the files together in the order they were listed (so in case of any key conflicts the latest file overwrites previous values), upper-casing keys and processing any list or dictionary values with `json.dumps`, so that they can be set as environment variables. This gives us a full list of environment variables that were previously parsed from the CloudFoundry user services data. --- scripts/generate_manifest.py | 62 ++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 scripts/generate_manifest.py diff --git a/scripts/generate_manifest.py b/scripts/generate_manifest.py new file mode 100755 index 000000000..b25c6d7ef --- /dev/null +++ b/scripts/generate_manifest.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys + +import json +import yaml + + +def merge_dicts(a, b): + if not (isinstance(a, dict) and isinstance(b, dict)): + raise ValueError("Error merging variables: '{}' and '{}'".format( + type(a).__name__, type(b).__name__ + )) + + result = a.copy() + for key, val in b.items(): + if isinstance(result.get(key), dict): + result[key] = merge_dicts(a[key], b[key]) + else: + result[key] = val + + return result + + +def load_manifest(manifest_file): + with open(manifest_file) as f: + manifest = yaml.load(f) + + if 'inherit' in manifest: + inherit_file = os.path.join(os.path.dirname(manifest_file), manifest.pop('inherit')) + manifest = merge_dicts(load_manifest(inherit_file), manifest) + + return manifest + + +def load_variables(vars_files): + variables = {} + for vars_file in vars_files: + with open(vars_file) as f: + variables = merge_dicts(variables, json.load(f)) + + return { + k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v + for k, v in variables.items() + } + + +def paas_manifest(manifest_file, *vars_files): + """Generate a PaaS manifest file from a Jinja2 template""" + + manifest = load_manifest(manifest_file) + variables = load_variables(vars_files) + + manifest['env'].update(variables) + + return yaml.dump(manifest, default_flow_style=False, allow_unicode=True) + + +if __name__ == "__main__": + print(paas_manifest(*sys.argv[1:])) From e02c8339e7f760f313fedf9cde18a7cb895b7b0d Mon Sep 17 00:00:00 2001 From: Alexey Bezhan Date: Thu, 4 Jan 2018 16:55:49 +0000 Subject: [PATCH 2/7] Remove cloudfoundry config parsing logic By replacing user-provided services with manifest environment variables we avoid the need to set the application environment variables from the service data. Most of the variable names already match the service JSON keys, but we need to rename the ones that don't (eg MMG and Firetext `api_key`) this is done in a separate credentials PR. --- app/cloudfoundry_config.py | 67 -------- tests/app/test_cloudfoundry_config.py | 217 +------------------------- 2 files changed, 2 insertions(+), 282 deletions(-) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 3a21a7a29..5cb7c437f 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -19,70 +19,3 @@ def set_config_env_vars(vcap_services): vcap_application = json.loads(os.environ['VCAP_APPLICATION']) os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name'] os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log' - - # Notify common config - for s in vcap_services['user-provided']: - if s['name'] == 'notify-config': - extract_notify_config(s) - elif s['name'] == 'notify-aws': - extract_notify_aws_config(s) - elif s['name'] == 'hosted-graphite': - extract_hosted_graphite_config(s) - elif s['name'] == 'mmg': - extract_mmg_config(s) - elif s['name'] == 'firetext': - extract_firetext_config(s) - elif s['name'] == 'redis': - extract_redis_config(s) - elif s['name'] == 'performance-platform': - extract_performance_platform_config(s) - elif s['name'] == 'notify-template-preview': - extract_template_preview_config(s) - - -def extract_notify_config(notify_config): - os.environ['ADMIN_BASE_URL'] = notify_config['credentials']['admin_base_url'] - os.environ['API_HOST_NAME'] = notify_config['credentials']['api_host_name'] - os.environ['ADMIN_CLIENT_SECRET'] = notify_config['credentials']['admin_client_secret'] - os.environ['SECRET_KEY'] = notify_config['credentials']['secret_key'] - os.environ['DANGEROUS_SALT'] = notify_config['credentials']['dangerous_salt'] - os.environ['SMS_INBOUND_WHITELIST'] = json.dumps(notify_config['credentials']['allow_ip_inbound_sms']) - os.environ['FIRETEXT_INBOUND_SMS_AUTH'] = json.dumps(notify_config['credentials']['firetext_inbound_sms_auth']) - os.environ['MMG_INBOUND_SMS_AUTH'] = json.dumps(notify_config['credentials']['mmg_inbound_sms_auth']) - os.environ['MMG_INBOUND_SMS_USERNAME'] = json.dumps(notify_config['credentials']['mmg_inbound_sms_username']) - os.environ['ROUTE_SECRET_KEY_1'] = notify_config['credentials']['route_secret_key_1'] - os.environ['ROUTE_SECRET_KEY_2'] = notify_config['credentials']['route_secret_key_2'] - - -def extract_performance_platform_config(performance_platform_config): - os.environ['PERFORMANCE_PLATFORM_ENDPOINTS'] = json.dumps(performance_platform_config['credentials']) - - -def extract_notify_aws_config(aws_config): - os.environ['NOTIFICATION_QUEUE_PREFIX'] = aws_config['credentials']['sqs_queue_prefix'] - os.environ['AWS_ACCESS_KEY_ID'] = aws_config['credentials']['aws_access_key_id'] - os.environ['AWS_SECRET_ACCESS_KEY'] = aws_config['credentials']['aws_secret_access_key'] - - -def extract_hosted_graphite_config(hosted_graphite_config): - os.environ['STATSD_PREFIX'] = hosted_graphite_config['credentials']['statsd_prefix'] - - -def extract_mmg_config(mmg_config): - os.environ['MMG_URL'] = mmg_config['credentials']['api_url'] - os.environ['MMG_API_KEY'] = mmg_config['credentials']['api_key'] - - -def extract_firetext_config(firetext_config): - os.environ['FIRETEXT_API_KEY'] = firetext_config['credentials']['api_key'] - os.environ['LOADTESTING_API_KEY'] = firetext_config['credentials']['loadtesting_api_key'] - - -def extract_redis_config(redis_config): - os.environ['REDIS_ENABLED'] = redis_config['credentials']['redis_enabled'] - os.environ['REDIS_URL'] = redis_config['credentials']['redis_url'] - - -def extract_template_preview_config(template_preview_config): - os.environ['TEMPLATE_PREVIEW_API_HOST'] = template_preview_config['credentials']['api_host'] - os.environ['TEMPLATE_PREVIEW_API_KEY'] = template_preview_config['credentials']['api_key'] diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py index aafd894c7..ccf238d49 100644 --- a/tests/app/test_cloudfoundry_config.py +++ b/tests/app/test_cloudfoundry_config.py @@ -6,70 +6,6 @@ import pytest from app.cloudfoundry_config import extract_cloudfoundry_config, set_config_env_vars -@pytest.fixture -def notify_config(): - return { - 'name': 'notify-config', - 'credentials': { - 'admin_base_url': 'admin base url', - 'api_host_name': 'api host name', - 'admin_client_secret': 'admin client secret', - 'secret_key': 'secret key', - 'dangerous_salt': 'dangerous salt', - 'allow_ip_inbound_sms': ['111.111.111.111', '100.100.100.100'], - 'firetext_inbound_sms_auth': ['testkey'], - 'mmg_inbound_sms_auth': ['testkey'], - 'mmg_inbound_sms_username': ['username'], - 'route_secret_key_1': "key_1", - 'route_secret_key_2': "" - } - } - - -@pytest.fixture -def aws_config(): - return { - 'name': 'notify-aws', - 'credentials': { - 'sqs_queue_prefix': 'sqs queue prefix', - 'aws_access_key_id': 'aws access key id', - 'aws_secret_access_key': 'aws secret access key', - } - } - - -@pytest.fixture -def hosted_graphite_config(): - return { - 'name': 'hosted-graphite', - 'credentials': { - 'statsd_prefix': 'statsd prefix' - } - } - - -@pytest.fixture -def mmg_config(): - return { - 'name': 'mmg', - 'credentials': { - 'api_url': 'mmg api url', - 'api_key': 'mmg api key' - } - } - - -@pytest.fixture -def firetext_config(): - return { - 'name': 'firetext', - 'credentials': { - 'api_key': 'firetext api key', - 'loadtesting_api_key': 'loadtesting api key' - } - } - - @pytest.fixture def postgres_config(): return [ @@ -82,62 +18,10 @@ def postgres_config(): @pytest.fixture -def redis_config(): - return { - 'name': 'redis', - 'credentials': { - 'redis_enabled': '1', - 'redis_url': 'redis url' - } - } - - -@pytest.fixture -def performance_platform_config(): - return { - 'name': 'performance-platform', - 'credentials': { - 'foo': 'my_token', - 'bar': 'other_token' - } - } - - -@pytest.fixture -def template_preview_config(): - return { - 'name': 'notify-template-preview', - 'credentials': { - 'api_host': 'template-preview api host', - 'api_key': 'template-preview api key' - } - } - - -@pytest.fixture -def cloudfoundry_config( - postgres_config, - notify_config, - aws_config, - hosted_graphite_config, - mmg_config, - firetext_config, - redis_config, - performance_platform_config, - template_preview_config -): +def cloudfoundry_config(postgres_config): return { 'postgres': postgres_config, - 'user-provided': [ - notify_config, - aws_config, - hosted_graphite_config, - mmg_config, - firetext_config, - redis_config, - performance_platform_config, - template_preview_config - ] + 'user-provided': [] } @@ -167,100 +51,3 @@ def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config): assert 'foo' not in os.environ assert 'bar' not in os.environ - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_notify_config(): - extract_cloudfoundry_config() - - assert os.environ['ADMIN_BASE_URL'] == 'admin base url' - assert os.environ['API_HOST_NAME'] == 'api host name' - assert os.environ['ADMIN_CLIENT_SECRET'] == 'admin client secret' - assert os.environ['SECRET_KEY'] == 'secret key' - assert os.environ['DANGEROUS_SALT'] == 'dangerous salt' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_aws_config(): - extract_cloudfoundry_config() - - assert os.environ['NOTIFICATION_QUEUE_PREFIX'] == 'sqs queue prefix' - assert os.environ['AWS_ACCESS_KEY_ID'] == 'aws access key id' - assert os.environ['AWS_SECRET_ACCESS_KEY'] == 'aws secret access key' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_hosted_graphite_config(): - extract_cloudfoundry_config() - - assert os.environ['STATSD_PREFIX'] == 'statsd prefix' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_mmg_config(): - extract_cloudfoundry_config() - - assert os.environ['MMG_URL'] == 'mmg api url' - assert os.environ['MMG_API_KEY'] == 'mmg api key' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_firetext_config(): - extract_cloudfoundry_config() - - assert os.environ['FIRETEXT_API_KEY'] == 'firetext api key' - assert os.environ['LOADTESTING_API_KEY'] == 'loadtesting api key' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_redis_config(): - extract_cloudfoundry_config() - - assert os.environ['REDIS_ENABLED'] == '1' - assert os.environ['REDIS_URL'] == 'redis url' - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_sms_inbound_config(): - extract_cloudfoundry_config() - - assert os.environ['SMS_INBOUND_WHITELIST'] == json.dumps(['111.111.111.111', '100.100.100.100']) - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_firetext_inbound_sms_auth_config(): - extract_cloudfoundry_config() - - assert os.environ['FIRETEXT_INBOUND_SMS_AUTH'] == json.dumps(['testkey']) - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_mmg_inbound_sms_auth_config(): - extract_cloudfoundry_config() - - assert os.environ['MMG_INBOUND_SMS_AUTH'] == json.dumps(['testkey']) - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_mmg_inbound_sms_username_config(): - extract_cloudfoundry_config() - - assert os.environ['MMG_INBOUND_SMS_USERNAME'] == json.dumps(['username']) - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_performance_platform_config(): - extract_cloudfoundry_config() - - assert json.loads(os.environ['PERFORMANCE_PLATFORM_ENDPOINTS']) == { - 'foo': 'my_token', - 'bar': 'other_token' - } - - -@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') -def test_template_preview_config(): - extract_cloudfoundry_config() - - assert os.environ['TEMPLATE_PREVIEW_API_HOST'] == 'template-preview api host' - assert os.environ['TEMPLATE_PREVIEW_API_KEY'] == 'template-preview api key' From 0ad5c184c2150a5028e3414240ddf7d08d6f86da Mon Sep 17 00:00:00 2001 From: Alexey Bezhan Date: Fri, 5 Jan 2018 13:27:24 +0000 Subject: [PATCH 3/7] Only update existing manifest variables when adding env secrets Changes generate manifest script to parse variables file as YAML and only add variables to the manifest if they're already listed in the `env` section. This allows us to use a single variables file for all applications and avoid duplicating secrets across multiple files while adding only the relevant secrets to the application manifest. --- scripts/generate_manifest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/generate_manifest.py b/scripts/generate_manifest.py index b25c6d7ef..8814049c8 100755 --- a/scripts/generate_manifest.py +++ b/scripts/generate_manifest.py @@ -39,7 +39,7 @@ def load_variables(vars_files): variables = {} for vars_file in vars_files: with open(vars_file) as f: - variables = merge_dicts(variables, json.load(f)) + variables = merge_dicts(variables, yaml.load(f)) return { k.upper(): json.dumps(v) if isinstance(v, (dict, list)) else v @@ -53,10 +53,13 @@ def paas_manifest(manifest_file, *vars_files): manifest = load_manifest(manifest_file) variables = load_variables(vars_files) - manifest['env'].update(variables) + for key in manifest.get('env', {}): + if key in variables: + manifest['env'][key] = variables[key] return yaml.dump(manifest, default_flow_style=False, allow_unicode=True) if __name__ == "__main__": + print('---') print(paas_manifest(*sys.argv[1:])) From ff8373600d859fbfc1978f751df8c6421b387f8c Mon Sep 17 00:00:00 2001 From: Alexey Bezhan Date: Fri, 5 Jan 2018 14:27:17 +0000 Subject: [PATCH 4/7] Replace manifest services with lists of env variables Removes bindings to user provided services for new application deployments and sets the expected environment variables to `null`. Listing variables in the manifest allows us to only inject secrets used by the app. `null` is not a valid value for a variable in a CloudFoundry manifest, so if any of the keys don't have a match in the credentials file `cf push` will return an error and stop the deployment. --- manifest-api-base.yml | 40 ++++++++++++++++++++++++++++------ manifest-api-preview.yml | 7 ------ manifest-api-staging.yml | 7 ------ manifest-delivery-base.yml | 41 ++++++++++++++++++++++++++++------- manifest-delivery-preview.yml | 8 ------- manifest-delivery-staging.yml | 8 ------- 6 files changed, 66 insertions(+), 45 deletions(-) diff --git a/manifest-api-base.yml b/manifest-api-base.yml index 9afb9f860..eea8d8459 100644 --- a/manifest-api-base.yml +++ b/manifest-api-base.yml @@ -3,19 +3,45 @@ buildpack: python_buildpack command: scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py --error-logfile /home/vcap/logs/gunicorn_error.log -w 5 -b 0.0.0.0:$PORT application services: - - notify-aws - - notify-config - notify-db - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform env: NOTIFY_APP_NAME: public-api CW_APP_NAME: api # required by cf run-task FLASK_APP: application.py + + # Credentials variables + ADMIN_BASE_URL: null + ADMIN_CLIENT_SECRET: null + API_HOST_NAME: null + DANGEROUS_SALT: null + SECRET_KEY: null + ROUTE_SECRET_KEY_1: null + ROUTE_SECRET_KEY_2: null + + PERFORMANCE_PLATFORM_ENDPOINTS: null + + NOTIFICATION_QUEUE_PREFIX: null + AWS_ACCESS_KEY_ID: null + AWS_SECRET_ACCESS_KEY: null + + STATSD_PREFIX: null + + MMG_URL: null + MMG_API_KEY: null + MMG_INBOUND_SMS_AUTH: null + MMG_INBOUND_SMS_USERNAME: null + + FIRETEXT_API_KEY: null + LOADTESTING_API_KEY: null + FIRETEXT_INBOUND_SMS_AUTH: null + + REDIS_ENABLED: null + REDIS_URL: null + + TEMPLATE_PREVIEW_API_HOST: null + TEMPLATE_PREVIEW_API_KEY: null + instances: 1 memory: 1G diff --git a/manifest-api-preview.yml b/manifest-api-preview.yml index 1e4174cb1..406b53cb5 100644 --- a/manifest-api-preview.yml +++ b/manifest-api-preview.yml @@ -3,14 +3,7 @@ inherit: manifest-api-base.yml services: - - notify-aws - - notify-config - notify-db - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform - logit-ssl-syslog-drain routes: diff --git a/manifest-api-staging.yml b/manifest-api-staging.yml index 868d2853b..30304c2fd 100644 --- a/manifest-api-staging.yml +++ b/manifest-api-staging.yml @@ -2,14 +2,7 @@ inherit: manifest-api-base.yml services: - - notify-aws - - notify-config - notify-db - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform - logit-ssl-syslog-drain routes: diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml index 26a12ea0e..78cff9fa5 100644 --- a/manifest-delivery-base.yml +++ b/manifest-delivery-base.yml @@ -4,18 +4,43 @@ buildpack: python_buildpack health-check-type: none no-route: true services: - - notify-aws - - notify-config - notify-db - - notify-template-preview - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform instances: 1 memory: 1G +env: + # Credentials variables + ADMIN_BASE_URL: null + ADMIN_CLIENT_SECRET: null + API_HOST_NAME: null + DANGEROUS_SALT: null + SECRET_KEY: null + ROUTE_SECRET_KEY_1: null + ROUTE_SECRET_KEY_2: null + + PERFORMANCE_PLATFORM_ENDPOINTS: null + + NOTIFICATION_QUEUE_PREFIX: null + AWS_ACCESS_KEY_ID: null + AWS_SECRET_ACCESS_KEY: null + + STATSD_PREFIX: null + + MMG_URL: null + MMG_API_KEY: null + MMG_INBOUND_SMS_AUTH: null + MMG_INBOUND_SMS_USERNAME: null + + FIRETEXT_API_KEY: null + LOADTESTING_API_KEY: null + FIRETEXT_INBOUND_SMS_AUTH: null + + REDIS_ENABLED: null + REDIS_URL: null + + TEMPLATE_PREVIEW_API_HOST: null + TEMPLATE_PREVIEW_API_KEY: null + applications: - name: notify-delivery-celery-beat command: scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO diff --git a/manifest-delivery-preview.yml b/manifest-delivery-preview.yml index 8de718214..f86d53582 100644 --- a/manifest-delivery-preview.yml +++ b/manifest-delivery-preview.yml @@ -3,15 +3,7 @@ inherit: manifest-delivery-base.yml services: - - notify-aws - - notify-config - notify-db - - notify-template-preview - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform - logit-ssl-syslog-drain memory: 1G diff --git a/manifest-delivery-staging.yml b/manifest-delivery-staging.yml index 7e7f37c0e..973b43a85 100644 --- a/manifest-delivery-staging.yml +++ b/manifest-delivery-staging.yml @@ -3,15 +3,7 @@ inherit: manifest-delivery-base.yml services: - - notify-aws - - notify-config - notify-db - - notify-template-preview - - mmg - - firetext - - hosted-graphite - - redis - - performance-platform - logit-ssl-syslog-drain instances: 2 From f2e601851266c6a241b893aaf5b83eb84c236cd4 Mon Sep 17 00:00:00 2001 From: Alexey Bezhan Date: Fri, 5 Jan 2018 14:32:20 +0000 Subject: [PATCH 5/7] Use generated manifest when deploying apps to PaaS Adds a new `make generate-manifest` target which is called by deployment steps. --- Makefile | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 3b2d46d16..a09f929dd 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,8 @@ $(eval export CF_HOME) CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml +NOTIFY_CREDENTIALS ?= ~/.notify-credentials + .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -188,13 +190,23 @@ cf-login: ## Log in to Cloud Foundry @echo "Logging in to Cloud Foundry on ${CF_API}" @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" +.PHONY: generate-manifest +generate-manifest: + $(if ${CF_APP},,$(error Must specify CF_APP)) + $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) + $(if $(shell which gpg2), $(eval export GPG=gpg2), $(eval export GPG=gpg)) + $(if ${GPG_PASSPHRASE_TXT}, $(eval export DECRYPT_CMD=echo -n $$$${GPG_PASSPHRASE_TXT} | ${GPG} --quiet --batch --passphrase-fd 0 --pinentry-mode loopback -d), $(eval export DECRYPT_CMD=${GPG} --quiet --batch -d)) + + @./scripts/generate_manifest.py ${CF_MANIFEST_FILE} \ + <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) + .PHONY: cf-deploy cf-deploy: ## Deploys the app to Cloud Foundry $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) $(if ${CF_APP},,$(error Must specify CF_APP)) @cf app --guid ${CF_APP} || exit 1 cf rename ${CF_APP} ${CF_APP}-rollback - cf push ${CF_APP} -f ${CF_MANIFEST_FILE} + cf push ${CF_APP} -f <(make -s generate-manifest) cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP} cf stop ${CF_APP}-rollback # sleep for 10 seconds to try and make sure that all worker threads (either web api or celery) have finished before we delete @@ -211,7 +223,7 @@ cf-deploy-api-db-migration: cf unbind-service notify-api-db-migration notify-db cf unbind-service notify-api-db-migration notify-config cf unbind-service notify-api-db-migration notify-aws - cf push notify-api-db-migration -f manifest-api-${CF_SPACE}.yml + cf push notify-api-db-migration -f -f <(make -s CF_APP=api generate-manifest) cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task @@ -230,7 +242,7 @@ cf-rollback: ## Rollbacks the app to the previous release cf-push: $(if ${CF_APP},,$(error Must specify CF_APP)) cf target -o ${CF_ORG} -s ${CF_SPACE} - cf push ${CF_APP} -f ${CF_MANIFEST_FILE} + cf push ${CF_APP} -f <(make -s generate-manifest) .PHONY: check-if-migrations-to-run check-if-migrations-to-run: From d82801fa5df84103239974781717d52e8aa1a411 Mon Sep 17 00:00:00 2001 From: Alexey Bezhan Date: Fri, 5 Jan 2018 17:04:18 +0000 Subject: [PATCH 6/7] Remove unused PERFORMANCE_PLATFORM_TOKEN config variable --- app/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/config.py b/app/config.py index 9c6a11055..3368ca68c 100644 --- a/app/config.py +++ b/app/config.py @@ -103,7 +103,6 @@ class Config(object): # Performance platform PERFORMANCE_PLATFORM_ENABLED = False PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/' - PERFORMANCE_PLATFORM_TOKEN = os.getenv('PERFORMANCE_PLATFORM_TOKEN') # Logging DEBUG = False From 40c27c6e704b42bad3ae46433211f82b2a160a46 Mon Sep 17 00:00:00 2001 From: Alexey Bezhan Date: Mon, 8 Jan 2018 14:14:41 +0000 Subject: [PATCH 7/7] Don't parse AWS credentials from VCAP_SERVICES in run_app_paas AWS credentials are provided in the environment variables directly, so we don't need to parse them from VCAP_SERVICES --- scripts/run_app_paas.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/run_app_paas.sh b/scripts/run_app_paas.sh index 2a4d2988d..cf0c7ccc7 100755 --- a/scripts/run_app_paas.sh +++ b/scripts/run_app_paas.sh @@ -22,9 +22,6 @@ function configure_aws_logs { aws configure set plugins.cwlogs cwlogs - export AWS_ACCESS_KEY_ID=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_access_key_id') - export AWS_SECRET_ACCESS_KEY=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_secret_access_key') - cat > /home/vcap/app/awslogs.conf << EOF [general] state_file = /home/vcap/logs/awslogs-state