diff --git a/.cfignore b/.cfignore new file mode 120000 index 000000000..3e4e48b0b --- /dev/null +++ b/.cfignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2194aa0bc..d9a4d419a 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,4 @@ environment.sh celerybeat-schedule -app/version.py - -wheelhouse/ \ No newline at end of file +wheelhouse/ diff --git a/Makefile b/Makefile index f90dd940a..2fb2068dc 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,8 @@ APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") GIT_COMMIT ?= $(shell git rev-parse HEAD) -DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder +DOCKER_IMAGE_TAG := $(shell cat docker/VERSION) +DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:${DOCKER_IMAGE_TAG} BUILD_TAG ?= notifications-api-manual BUILD_NUMBER ?= 0 @@ -17,6 +18,10 @@ BUILD_URL ?= DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG} +CF_API ?= api.cloud.service.gov.uk +CF_ORG ?= govuk-notify +CF_SPACE ?= ${DEPLOY_ENV} + .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -25,8 +30,8 @@ help: venv: venv/bin/activate ## Create virtualenv if it does not exist venv/bin/activate: - test -d venv || virtualenv venv - ./venv/bin/pip install pip-accel + test -d venv || virtualenv venv -p python3 + . venv/bin/activate && pip install pip-accel .PHONY: check-env-vars check-env-vars: ## Check mandatory environment variables @@ -35,6 +40,12 @@ check-env-vars: ## Check mandatory environment variables $(if ${AWS_ACCESS_KEY_ID},,$(error Must specify AWS_ACCESS_KEY_ID)) $(if ${AWS_SECRET_ACCESS_KEY},,$(error Must specify AWS_SECRET_ACCESS_KEY)) +.PHONY: sandbox +sandbox: ## Set environment to sandbox + $(eval export DEPLOY_ENV=sandbox) + $(eval export DNS_NAME="cloudapps.digital") + @true + .PHONY: preview preview: ## Set environment to preview $(eval export DEPLOY_ENV=preview) @@ -56,7 +67,7 @@ production: ## Set environment to production .PHONY: dependencies dependencies: venv ## Install build dependencies mkdir -p ${PIP_ACCEL_CACHE} - PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} ./venv/bin/pip-accel install -r requirements_for_test.txt + . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements_for_test.txt .PHONY: generate-version-file generate-version-file: ## Generates the app version file @@ -64,7 +75,10 @@ generate-version-file: ## Generates the app version file .PHONY: build build: dependencies generate-version-file ## Build project - ./venv/bin/pip-accel wheel --wheel-dir=wheelhouse -r requirements.txt + . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel wheel --wheel-dir=wheelhouse -r requirements.txt + +.PHONY: cf-build +cf-build: dependencies generate-version-file ## Build project for PAAS .PHONY: build-codedeploy-artifact build-codedeploy-artifact: ## Build the deploy artifact for CodeDeploy @@ -105,12 +119,12 @@ deploy-delivery: check-env-vars ## Trigger CodeDeploy for the delivery app .PHONY: coverage coverage: venv ## Create coverage report - ./venv/bin/coveralls + . venv/bin/activate && coveralls .PHONY: prepare-docker-build-image prepare-docker-build-image: ## Prepare the Docker builder image mkdir -p ${PIP_ACCEL_CACHE} - make -C docker build-build-image + make -C docker build .PHONY: build-with-docker build-with-docker: prepare-docker-build-image ## Build inside a Docker container @@ -129,6 +143,23 @@ build-with-docker: prepare-docker-build-image ## Build inside a Docker container ${DOCKER_BUILDER_IMAGE_NAME} \ make build +.PHONY: cf-build-with-docker +cf-build-with-docker: prepare-docker-build-image ## Build inside a Docker container + @docker run -i --rm \ + --name "${DOCKER_CONTAINER_PREFIX}-build" \ + -v "`pwd`:/var/project" \ + -v "${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel" \ + -e GIT_COMMIT=${GIT_COMMIT} \ + -e BUILD_NUMBER=${BUILD_NUMBER} \ + -e BUILD_URL=${BUILD_URL} \ + -e http_proxy="${HTTP_PROXY}" \ + -e HTTP_PROXY="${HTTP_PROXY}" \ + -e https_proxy="${HTTPS_PROXY}" \ + -e HTTPS_PROXY="${HTTPS_PROXY}" \ + -e NO_PROXY="${NO_PROXY}" \ + ${DOCKER_BUILDER_IMAGE_NAME} \ + make cf-build + .PHONY: test-with-docker test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container @docker run -i --rm \ @@ -182,5 +213,79 @@ coverage-with-docker: prepare-docker-build-image ## Generates coverage report in clean-docker-containers: ## Clean up any remaining docker containers docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true +.PHONY: clean clean: - rm -rf node_modules cache target venv .coverage build tests/.cache + rm -rf node_modules cache target venv .coverage build tests/.cache wheelhouse + +.PHONY: cf-login +cf-login: ## Log in to Cloud Foundry + $(if ${CF_USERNAME},,$(error Must specify CF_USERNAME)) + $(if ${CF_PASSWORD},,$(error Must specify CF_PASSWORD)) + $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) + @echo "Logging in to Cloud Foundry on ${CF_API}" + @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" + +.PHONY: cf-deploy-api +cf-deploy-api: ## Deploys the API to Cloud Foundry + $(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid notify-api) | jq -r ".entity.instances")) + @echo "Original instance count: ${ORIG_INSTANCES}" + cf check-manifest notify-api -f manifest-api-${CF_SPACE}.yml + cf zero-downtime-push notify-api -f manifest-api-${CF_SPACE}.yml + cf scale -i ${ORIG_INSTANCES} notify-api + +.PHONY: cf-push-api +cf-push-api: ## + cf push notify-api -f manifest-api-${CF_SPACE}.yml + +.PHONY: cf-deploy-api-db-migration +cf-deploy-api-db-migration: ## Deploys the API db migration to Cloud Foundry + cf check-manifest notify-api-db-migration -f manifest-api-db-migration.yml + cf push notify-api-db-migration -f manifest-api-db-migration.yml + +cf-push-api-db-migration: cf-deploy-api-db-migration ## Deploys the API db migration to Cloud Foundry + +.PHONY: cf-deploy-delivery +cf-deploy-delivery: ## Deploys a delivery app to Cloud Foundry + $(if ${CF_APP},,$(error Must specify CF_APP)) + $(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid ${CF_APP}) | jq -r ".entity.instances")) + @echo "Original instance count: ${ORIG_INSTANCES}" + cf check-manifest ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml + cf zero-downtime-push ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml + cf scale -i ${ORIG_INSTANCES} ${CF_APP} + +.PHONY: cf-push-delivery +cf-push-delivery: ## Deploys a delivery app to Cloud Foundry + $(if ${CF_APP},,$(error Must specify CF_APP)) + cf push ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml + +define cf_deploy_with_docker + @docker run -i --rm \ + --name "${DOCKER_CONTAINER_PREFIX}-${1}" \ + -v `pwd`:/var/project \ + -e http_proxy="${HTTP_PROXY}" \ + -e HTTP_PROXY="${HTTP_PROXY}" \ + -e https_proxy="${HTTPS_PROXY}" \ + -e HTTPS_PROXY="${HTTPS_PROXY}" \ + -e NO_PROXY="${NO_PROXY}" \ + -e CF_API="${CF_API}" \ + -e CF_USERNAME="${CF_USERNAME}" \ + -e CF_PASSWORD="${CF_PASSWORD}" \ + -e CF_ORG="${CF_ORG}" \ + -e CF_SPACE="${CF_SPACE}" \ + -e CF_APP="${CF_APP}" \ + ${DOCKER_BUILDER_IMAGE_NAME} \ + ${2} +endef + +.PHONY: cf-deploy-api-with-docker +cf-deploy-api-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container + $(call cf_deploy_with_docker,cf-deploy-api,make cf-login cf-deploy-api) + +.PHONY: cf-deploy-api-db-migration-with-docker +cf-deploy-api-db-migration-with-docker: prepare-docker-build-image ## Deploys the API db migration to Cloud Foundry from a Docker container + $(call cf_deploy_with_docker,cf-deploy-api-db-migration,make cf-login cf-deploy-api-db-migration) + +.PHONY: cf-deploy-delivery-with-docker +cf-deploy-delivery-with-docker: prepare-docker-build-image ## Deploys a delivery app to Cloud Foundry from a Docker container + $(if ${CF_APP},,$(error Must specify CF_APP)) + $(call cf_deploy_with_docker,cf-deploy-delivery-${CF_APP},make cf-login cf-deploy-delivery) diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 000000000..985278646 --- /dev/null +++ b/app/.gitignore @@ -0,0 +1 @@ +version.py diff --git a/app/__init__.py b/app/__init__.py index 4184e80d2..a13dacd8b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,5 +1,6 @@ import os import uuid +import json from flask import Flask, _request_ctx_stack from flask import request, url_for, g, jsonify @@ -42,8 +43,12 @@ api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user) def create_app(app_name=None): application = Flask(__name__) - from config import configs - application.config.from_object(configs[os.environ['NOTIFY_ENVIRONMENT']]) + from app.config import configs + + notify_environment = os.environ['NOTIFY_ENVIRONMENT'] + + application.config.from_object(configs[notify_environment]) + if app_name: application.config['NOTIFY_APP_NAME'] = app_name diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py new file mode 100644 index 000000000..ed8f7f350 --- /dev/null +++ b/app/cloudfoundry_config.py @@ -0,0 +1,62 @@ +""" +Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated +on local/aws boxes +""" + +import os +import json + + +def extract_cloudfoundry_config(): + vcap_services = json.loads(os.environ['VCAP_SERVICES']) + set_config_env_vars(vcap_services) + + +def set_config_env_vars(vcap_services): + # Postgres config + os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri'] + + vcap_application = json.loads(os.environ['VCAP_APPLICATION']) + os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name'] + os.environ['LOGGING_STDOUT_JSON'] = '1' + + # Notify common config + for s in vcap_services['user-provided']: + if s['name'] == 'notify-config': + extract_notify_config(s) + elif s['name'] == 'notify-aws': + extract_notify_aws_config(s) + elif s['name'] == 'hosted-graphite': + extract_hosted_graphite_config(s) + elif s['name'] == 'mmg': + extract_mmg_config(s) + elif s['name'] == 'firetext': + extract_firetext_config(s) + + +def extract_notify_config(notify_config): + os.environ['ADMIN_BASE_URL'] = notify_config['credentials']['admin_base_url'] + os.environ['API_HOST_NAME'] = notify_config['credentials']['api_host_name'] + os.environ['ADMIN_CLIENT_SECRET'] = notify_config['credentials']['admin_client_secret'] + os.environ['SECRET_KEY'] = notify_config['credentials']['secret_key'] + os.environ['DANGEROUS_SALT'] = notify_config['credentials']['dangerous_salt'] + + +def extract_notify_aws_config(aws_config): + os.environ['NOTIFICATION_QUEUE_PREFIX'] = aws_config['credentials']['sqs_queue_prefix'] + os.environ['AWS_ACCESS_KEY_ID'] = aws_config['credentials']['aws_access_key_id'] + os.environ['AWS_SECRET_ACCESS_KEY'] = aws_config['credentials']['aws_secret_access_key'] + + +def extract_hosted_graphite_config(hosted_graphite_config): + os.environ['STATSD_PREFIX'] = hosted_graphite_config['credentials']['statsd_prefix'] + + +def extract_mmg_config(mmg_config): + os.environ['MMG_URL'] = mmg_config['credentials']['api_url'] + os.environ['MMG_API_KEY'] = mmg_config['credentials']['api_key'] + + +def extract_firetext_config(firetext_config): + os.environ['FIRETEXT_API_KEY'] = firetext_config['credentials']['api_key'] + os.environ['LOADTESTING_API_KEY'] = firetext_config['credentials']['loadtesting_api_key'] diff --git a/config.py b/app/config.py similarity index 89% rename from config.py rename to app/config.py index 884fed7eb..21b1c1a53 100644 --- a/config.py +++ b/app/config.py @@ -4,14 +4,20 @@ from kombu import Exchange, Queue import os -class Config(object): - ######################################## - # Secrets that are held in credstash ### - ######################################## +if os.environ.get('VCAP_SERVICES'): + # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate + # standard environment variables from it + from app.cloudfoundry_config import extract_cloudfoundry_config + extract_cloudfoundry_config() + +class Config(object): # URL of admin app ADMIN_BASE_URL = os.environ['ADMIN_BASE_URL'] + # URL of api app (on AWS this is the internal api endpoint) + API_HOST_NAME = os.getenv('API_HOST_NAME') + # admin app api key ADMIN_CLIENT_SECRET = os.environ['ADMIN_CLIENT_SECRET'] @@ -42,12 +48,16 @@ class Config(object): # URL of redis instance REDIS_URL = os.getenv('REDIS_URL') + REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1' + + # Logging + DEBUG = False + LOGGING_STDOUT_JSON = os.getenv('LOGGING_STDOUT_JSON') == '1' ########################### # Default config values ### ########################### - DEBUG = False NOTIFY_ENVIRONMENT = 'development' ADMIN_CLIENT_USER_NAME = 'notify-admin' AWS_REGION = 'eu-west-1' @@ -126,8 +136,6 @@ class Config(object): Queue('notify', Exchange('default'), routing_key='notify') ] - API_HOST_NAME = "http://localhost:6011" - NOTIFICATIONS_ALERT = 5 # five mins FROM_NUMBER = 'development' @@ -135,8 +143,6 @@ class Config(object): STATSD_HOST = "statsd.hostedgraphite.com" STATSD_PORT = 8125 - REDIS_ENABLED = False - SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200 # 3 days SIMULATED_EMAIL_ADDRESSES = ('simulate-delivered@notifications.service.gov.uk', @@ -165,6 +171,7 @@ class Development(Config): Queue('send-email', Exchange('default'), routing_key='send-email'), Queue('research-mode', Exchange('default'), routing_key='research-mode') ] + API_HOST_NAME = "http://localhost:6011" class Test(Config): @@ -172,7 +179,6 @@ class Test(Config): FROM_NUMBER = 'testing' NOTIFY_ENVIRONMENT = 'test' DEBUG = True - REDIS_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' STATSD_ENABLED = True STATSD_HOST = "localhost" @@ -184,6 +190,8 @@ class Test(Config): Queue('send-email', Exchange('default'), routing_key='send-email'), Queue('research-mode', Exchange('default'), routing_key='research-mode') ] + REDIS_ENABLED = True + API_HOST_NAME = "http://localhost:6011" class Preview(Config): @@ -192,7 +200,6 @@ class Preview(Config): CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' API_HOST_NAME = 'http://admin-api.internal' FROM_NUMBER = 'preview' - REDIS_ENABLED = True class Staging(Config): @@ -202,7 +209,6 @@ class Staging(Config): STATSD_ENABLED = True API_HOST_NAME = 'http://admin-api.internal' FROM_NUMBER = 'stage' - REDIS_ENABLED = True class Live(Config): @@ -212,7 +218,18 @@ class Live(Config): STATSD_ENABLED = True API_HOST_NAME = 'http://admin-api.internal' FROM_NUMBER = '40604' - REDIS_ENABLED = True + + +class CloudFoundryConfig(Config): + pass + + +# CloudFoundry sandbox +class Sandbox(CloudFoundryConfig): + NOTIFY_EMAIL_DOMAIN = 'notify.works' + NOTIFY_ENVIRONMENT = 'sandbox' + CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload' + FROM_NUMBER = 'sandbox' configs = { @@ -220,5 +237,6 @@ configs = { 'test': Test, 'live': Live, 'staging': Staging, - 'preview': Preview + 'preview': Preview, + 'sandbox': Sandbox } diff --git a/aws_run_celery.py b/aws_run_celery.py index e1db882a2..36a1f480a 100644 --- a/aws_run_celery.py +++ b/aws_run_celery.py @@ -3,8 +3,9 @@ from app import notify_celery, create_app from credstash import getAllSecrets import os -# on aws get secrets and export to env -os.environ.update(getAllSecrets(region="eu-west-1")) +# On AWS get secrets and export to env, skip this on Cloud Foundry +if os.getenv('VCAP_SERVICES') is None: + os.environ.update(getAllSecrets(region="eu-west-1")) application = create_app("delivery") application.app_context().push() diff --git a/db.py b/db.py index 7ded5ba1e..3a8da01d4 100644 --- a/db.py +++ b/db.py @@ -4,8 +4,9 @@ from app import create_app, db from credstash import getAllSecrets import os -# on aws get secrets and export to env -os.environ.update(getAllSecrets(region="eu-west-1")) +# On AWS get secrets and export to env, skip this on Cloud Foundry +if os.getenv('VCAP_SERVICES') is None: + os.environ.update(getAllSecrets(region="eu-west-1")) application = create_app() diff --git a/docker/Dockerfile-build b/docker/Dockerfile similarity index 60% rename from docker/Dockerfile-build rename to docker/Dockerfile index b8d145cb5..019e74168 100644 --- a/docker/Dockerfile-build +++ b/docker/Dockerfile @@ -13,10 +13,19 @@ RUN \ && apt-get update \ && apt-get install -y --no-install-recommends \ make \ + curl \ git \ build-essential \ zip \ libpq-dev \ + jq \ + + && echo "Install Cloud Foundry CLI" \ + && curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \ + && dpkg -i /tmp/cloudfoundry-cli.deb \ + && cf install-plugin -r CF-Community -f "autopilot" \ + && cf install-plugin -r CF-Community -f "blue-green-deploy" \ + && cf install-plugin -r CF-Community -f "antifreeze" \ && echo "Clean up" \ && rm -rf /var/lib/apt/lists/* /tmp/* diff --git a/docker/Makefile b/docker/Makefile index ab9f9d444..9a9bb76c9 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,17 +1,23 @@ .DEFAULT_GOAL := help SHELL := /bin/bash +DOCKER_IMAGE_TAG := $(shell cat VERSION) .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -.PHONY: build-build-image -build-build-image: +.PHONY: build +build: docker build \ --pull \ --build-arg HTTP_PROXY="${HTTP_PROXY}" \ --build-arg HTTPS_PROXY="${HTTP_PROXY}" \ --build-arg NO_PROXY="${NO_PROXY}" \ - -f Dockerfile-build \ - -t govuk/notify-api-builder \ + -t govuk/notify-api-builder:${DOCKER_IMAGE_TAG} \ . + +.PHONY: bash +bash: + docker run -it --rm \ + govuk/notify-api-builder \ + bash diff --git a/docker/VERSION b/docker/VERSION new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/docker/VERSION @@ -0,0 +1 @@ +2 diff --git a/manifest-api-db-migration.yml b/manifest-api-db-migration.yml new file mode 100644 index 000000000..5760e111d --- /dev/null +++ b/manifest-api-db-migration.yml @@ -0,0 +1,19 @@ +--- + +applications: + - name: notify-api-db-migration + buildpack: python_buildpack + command: python db.py db upgrade && sleep infinity + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + env: + NOTIFY_APP_NAME: public-api + no-route: true + health-check-type: none + instances: 1 + memory: 128M diff --git a/manifest-api-preview.yml b/manifest-api-preview.yml new file mode 100644 index 000000000..e6645a9e2 --- /dev/null +++ b/manifest-api-preview.yml @@ -0,0 +1,20 @@ +--- + +applications: + - name: notify-api + buildpack: python_buildpack + command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + env: + NOTIFY_APP_NAME: public-api + routes: + - route: notify-api-preview.cloudapps.digital + - route: api-paas.notify.works + instances: 1 + memory: 512M diff --git a/manifest-api-production.yml b/manifest-api-production.yml new file mode 100644 index 000000000..9ea72569e --- /dev/null +++ b/manifest-api-production.yml @@ -0,0 +1,20 @@ +--- + +applications: + - name: notify-api + buildpack: python_buildpack + command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + env: + NOTIFY_APP_NAME: public-api + routes: + - route: notify-api-production.cloudapps.digital + - route: api-paas.notifications.service.gov.uk + instances: 2 + memory: 2048M diff --git a/manifest-api-sandbox.yml b/manifest-api-sandbox.yml new file mode 100644 index 000000000..fc6e117dc --- /dev/null +++ b/manifest-api-sandbox.yml @@ -0,0 +1,19 @@ +--- + +applications: + - name: notify-api + buildpack: python_buildpack + command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + env: + NOTIFY_APP_NAME: public-api + routes: + - route: notify-api-sandbox.cloudapps.digital + instances: 1 + memory: 512M diff --git a/manifest-api-staging.yml b/manifest-api-staging.yml new file mode 100644 index 000000000..e593f1e26 --- /dev/null +++ b/manifest-api-staging.yml @@ -0,0 +1,20 @@ +--- + +applications: + - name: notify-api + buildpack: python_buildpack + command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + env: + NOTIFY_APP_NAME: public-api + routes: + - route: notify-api-staging.cloudapps.digital + - route: api-paas.staging-notify.works + instances: 2 + memory: 2048M diff --git a/manifest-delivery-celery-beat.yml b/manifest-delivery-celery-beat.yml new file mode 100644 index 000000000..0f65fd635 --- /dev/null +++ b/manifest-delivery-celery-beat.yml @@ -0,0 +1,19 @@ +--- + +applications: + - name: notify-delivery-celery-beat + buildpack: python_buildpack + health-check-type: none + no-route: true + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + instances: 2 + memory: 128M + command: celery -A aws_run_celery.notify_celery beat --loglevel=INFO + env: + NOTIFY_APP_NAME: delivery-celery-beat diff --git a/manifest-delivery-worker-database.yml b/manifest-delivery-worker-database.yml new file mode 100644 index 000000000..eaff9e337 --- /dev/null +++ b/manifest-delivery-worker-database.yml @@ -0,0 +1,19 @@ +--- + +applications: + - name: notify-delivery-worker-database + buildpack: python_buildpack + health-check-type: none + no-route: true + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + instances: 2 + memory: 256M + command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email + env: + NOTIFY_APP_NAME: delivery-worker-database diff --git a/manifest-delivery-worker-research.yml b/manifest-delivery-worker-research.yml new file mode 100644 index 000000000..cf935aac9 --- /dev/null +++ b/manifest-delivery-worker-research.yml @@ -0,0 +1,19 @@ +--- + +applications: + - name: notify-delivery-worker-research + buildpack: python_buildpack + health-check-type: none + no-route: true + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + instances: 2 + memory: 256M + command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode + env: + NOTIFY_APP_NAME: delivery-worker-research diff --git a/manifest-delivery-worker-sender.yml b/manifest-delivery-worker-sender.yml new file mode 100644 index 000000000..3774c86ff --- /dev/null +++ b/manifest-delivery-worker-sender.yml @@ -0,0 +1,19 @@ +--- + +applications: + - name: notify-delivery-worker-sender + buildpack: python_buildpack + health-check-type: none + no-route: true + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + instances: 2 + memory: 256M + command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email + env: + NOTIFY_APP_NAME: delivery-worker-sender diff --git a/manifest-delivery-worker.yml b/manifest-delivery-worker.yml new file mode 100644 index 000000000..ee53bb82a --- /dev/null +++ b/manifest-delivery-worker.yml @@ -0,0 +1,19 @@ +--- + +applications: + - name: notify-delivery-worker + buildpack: python_buildpack + health-check-type: none + no-route: true + services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite + instances: 2 + memory: 256M + command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 + env: + NOTIFY_APP_NAME: delivery-worker diff --git a/requirements.txt b/requirements.txt index ab817c0e1..067ddb3dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,10 +19,11 @@ monotonic==1.2 statsd==3.2.1 jsonschema==2.5.1 Flask-Redis==0.1.0 +gunicorn==19.6.0 # pin to minor version 3.1.x notifications-python-client>=3.1,<3.2 -git+https://github.com/alphagov/notifications-utils.git@13.0.1#egg=notifications-utils==13.0.1 +git+https://github.com/alphagov/notifications-utils.git@13.1.0#egg=notifications-utils==13.1.0 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 000000000..c0354eefe --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.5.2 diff --git a/server_commands.py b/server_commands.py index 0b72ee425..6289da765 100644 --- a/server_commands.py +++ b/server_commands.py @@ -11,10 +11,11 @@ if os.path.isfile(default_env_file): with open(default_env_file, 'r') as environment_file: environment = environment_file.readline().strip() -# on aws get secrets and export to env -os.environ.update(getAllSecrets(region="eu-west-1")) +# On AWS get secrets and export to env, skip this on Cloud Foundry +if os.getenv('VCAP_SERVICES') is None: + os.environ.update(getAllSecrets(region="eu-west-1")) -from config import configs +from app.config import configs os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment] diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py new file mode 100644 index 000000000..9b29d90ef --- /dev/null +++ b/tests/app/test_cloudfoundry_config.py @@ -0,0 +1,167 @@ +import os +import json + +import pytest + +from app.cloudfoundry_config import extract_cloudfoundry_config, set_config_env_vars + + +@pytest.fixture +def notify_config(): + return { + 'name': 'notify-config', + 'credentials': { + 'admin_base_url': 'admin base url', + 'api_host_name': 'api host name', + 'admin_client_secret': 'admin client secret', + 'secret_key': 'secret key', + 'dangerous_salt': 'dangerous salt', + } + } + + +@pytest.fixture +def aws_config(): + return { + 'name': 'notify-aws', + 'credentials': { + 'sqs_queue_prefix': 'sqs queue prefix', + 'aws_access_key_id': 'aws access key id', + 'aws_secret_access_key': 'aws secret access key', + } + } + + +@pytest.fixture +def hosted_graphite_config(): + return { + 'name': 'hosted-graphite', + 'credentials': { + 'statsd_prefix': 'statsd prefix' + } + } + + +@pytest.fixture +def mmg_config(): + return { + 'name': 'mmg', + 'credentials': { + 'api_url': 'mmg api url', + 'api_key': 'mmg api key' + } + } + + +@pytest.fixture +def firetext_config(): + return { + 'name': 'firetext', + 'credentials': { + 'api_key': 'firetext api key', + 'loadtesting_api_key': 'loadtesting api key' + } + } + + +@pytest.fixture +def postgres_config(): + return [ + { + 'credentials': { + 'uri': 'postgres uri' + } + } + ] + + +@pytest.fixture +def cloudfoundry_config( + postgres_config, + notify_config, + aws_config, + hosted_graphite_config, + mmg_config, + firetext_config +): + return { + 'postgres': postgres_config, + 'user-provided': [ + notify_config, + aws_config, + hosted_graphite_config, + mmg_config, + firetext_config + ] + } + + +@pytest.fixture +def cloudfoundry_environ(monkeypatch, cloudfoundry_config): + monkeypatch.setenv('VCAP_SERVICES', json.dumps(cloudfoundry_config)) + monkeypatch.setenv('VCAP_APPLICATION', '{"space_name": "🚀🌌"}') + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_extract_cloudfoundry_config_populates_other_vars(): + extract_cloudfoundry_config() + + assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgres uri' + assert os.environ['LOGGING_STDOUT_JSON'] == '1' + assert os.environ['NOTIFY_ENVIRONMENT'] == '🚀🌌' + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config): + cloudfoundry_config['foo'] = {'credentials': {'foo': 'foo'}} + cloudfoundry_config['user-provided'].append({ + 'name': 'bar', 'credentials': {'bar': 'bar'} + }) + + set_config_env_vars(cloudfoundry_config) + + assert 'foo' not in os.environ + assert 'bar' not in os.environ + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_notify_config(): + extract_cloudfoundry_config() + + assert os.environ['ADMIN_BASE_URL'] == 'admin base url' + assert os.environ['API_HOST_NAME'] == 'api host name' + assert os.environ['ADMIN_CLIENT_SECRET'] == 'admin client secret' + assert os.environ['SECRET_KEY'] == 'secret key' + assert os.environ['DANGEROUS_SALT'] == 'dangerous salt' + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_aws_config(): + extract_cloudfoundry_config() + + assert os.environ['NOTIFICATION_QUEUE_PREFIX'] == 'sqs queue prefix' + assert os.environ['AWS_ACCESS_KEY_ID'] == 'aws access key id' + assert os.environ['AWS_SECRET_ACCESS_KEY'] == 'aws secret access key' + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_hosted_graphite_config(): + extract_cloudfoundry_config() + + assert os.environ['STATSD_PREFIX'] == 'statsd prefix' + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_mmg_config(): + extract_cloudfoundry_config() + + assert os.environ['MMG_URL'] == 'mmg api url' + assert os.environ['MMG_API_KEY'] == 'mmg api key' + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_firetext_config(): + extract_cloudfoundry_config() + + assert os.environ['FIRETEXT_API_KEY'] == 'firetext api key' + assert os.environ['LOADTESTING_API_KEY'] == 'loadtesting api key' diff --git a/tests/app/test_config.py b/tests/app/test_config.py new file mode 100644 index 000000000..63a9bff9a --- /dev/null +++ b/tests/app/test_config.py @@ -0,0 +1,80 @@ +import os +import importlib +from unittest import mock + +import pytest + +from app import config + + +def cf_conf(): + os.environ['ADMIN_BASE_URL'] = 'cf' + + +@pytest.fixture +def reload_config(): + """ + Reset config, by simply re-running config.py from a fresh environment + """ + old_env = os.environ.copy() + + yield + + os.environ = old_env + importlib.reload(config) + + +def test_load_cloudfoundry_config_if_available(monkeypatch, reload_config): + os.environ['ADMIN_BASE_URL'] = 'env' + monkeypatch.setenv('VCAP_SERVICES', 'some json blob') + monkeypatch.setenv('VCAP_APPLICATION', 'some json blob') + + with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config: + # reload config so that its module level code (ie: all of it) is re-instantiated + importlib.reload(config) + + assert cf_config.called + + assert os.environ['ADMIN_BASE_URL'] == 'cf' + assert config.Config.ADMIN_BASE_URL == 'cf' + + +def test_load_config_if_cloudfoundry_not_available(monkeypatch, reload_config): + os.environ['ADMIN_BASE_URL'] = 'env' + + monkeypatch.delenv('VCAP_SERVICES', raising=False) + + with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config: + # reload config so that its module level code (ie: all of it) is re-instantiated + importlib.reload(config) + + assert not cf_config.called + + assert os.environ['ADMIN_BASE_URL'] == 'env' + assert config.Config.ADMIN_BASE_URL == 'env' + + +def test_cloudfoundry_config_has_different_defaults(): + # these should always be set on Sandbox + assert config.Sandbox.REDIS_ENABLED is False + + +def test_logging_stdout_json_defaults_to_off(reload_config): + os.environ.pop('LOGGING_STDOUT_JSON', None) + assert config.Config.LOGGING_STDOUT_JSON is False + + +def test_logging_stdout_json_sets_to_off_if_not_recognised(reload_config): + os.environ['LOGGING_STDOUT_JSON'] = 'foo' + + importlib.reload(config) + + assert config.Config.LOGGING_STDOUT_JSON is False + + +def test_logging_stdout_json_sets_to_on_if_set_to_1(reload_config): + os.environ['LOGGING_STDOUT_JSON'] = '1' + + importlib.reload(config) + + assert config.Config.LOGGING_STDOUT_JSON is True diff --git a/tests/conftest.py b/tests/conftest.py index 13e705880..ae9583ec3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -75,9 +75,16 @@ def notify_db_session(notify_db): notify_db.session.commit() -@pytest.fixture(scope='function') -def os_environ(mocker): - mocker.patch('os.environ', {}) +@pytest.fixture +def os_environ(): + """ + clear os.environ, and restore it after the test runs + """ + # for use whenever you expect code to edit environment variables + old_env = os.environ.copy() + os.environ = {} + yield + os.environ = old_env @pytest.fixture(scope='function') diff --git a/wsgi.py b/wsgi.py index e78f4c136..2df2c3976 100644 --- a/wsgi.py +++ b/wsgi.py @@ -4,8 +4,9 @@ from app import create_app from credstash import getAllSecrets -# on aws get secrets and export to env -os.environ.update(getAllSecrets(region="eu-west-1")) +# On AWS get secrets and export to env, skip this on Cloud Foundry +if os.getenv('VCAP_SERVICES') is None: + os.environ.update(getAllSecrets(region="eu-west-1")) application = create_app()