Merge pull request #1049 from alphagov/cloudfoundry

Run on Paas

.cfignore

@@ -0,0 +1 @@
+.gitignore

.gitignore

@@ -64,13 +64,9 @@ target/
 app/assets/stylesheets/govuk_template/.sass-cache/
 .sass-cache/
 cache/
-app/static
 node_modules
 bower_components
-app/templates/govuk_template.html
 npm-debug.log
 
 environment.sh
 .envrc
-
-app/version.py

Jenkinsfile

@@ -0,0 +1,165 @@
+#!groovy
+
+def deploy(cfEnv) {
+  waitUntil {
+    try {
+      lock(cfEnv) {
+        withCredentials([
+          string(credentialsId: 'paas_username', variable: 'CF_USERNAME'),
+          string(credentialsId: 'paas_password', variable: 'CF_PASSWORD')
+        ]) {
+          withEnv(["CF_SPACE=${cfEnv}"]) {
+            sh 'make cf-deploy-with-docker'
+          }
+        }
+        gitCommit = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
+        sh("git tag -f deployed-to-cf-${cfEnv} ${gitCommit}")
+        sh("git push -f origin deployed-to-cf-${cfEnv}")
+      }
+      true
+    } catch(err) {
+      echo "Deployment to ${cfEnv} failed: ${err}"
+      try {
+        slackSend channel: '#govuk-notify', message: "Deployment to ${cfEnv} failed. Please retry or abort: <${env.BUILD_URL}|${env.JOB_NAME} - #${env.BUILD_NUMBER}>", color: 'danger'
+      } catch(err2) {
+        echo "Sending Slack message failed: ${err2}"
+      }
+      input "Stage failed. Retry?"
+      false
+    }
+  }
+}
+
+def buildJobWithRetry(jobName) {
+  waitUntil {
+    try {
+      build job: jobName
+      true
+    } catch(err) {
+      echo "${jobName} failed: ${err}"
+      try {
+        slackSend channel: '#govuk-notify', message: "${jobName} failed. Please retry or abort: <${env.BUILD_URL}|${env.JOB_NAME} - #${env.BUILD_NUMBER}>", color: 'danger'
+      } catch(err2) {
+        echo "Sending Slack message failed: ${err2}"
+      }
+      input "${jobName} failed. Retry?"
+      false
+    }
+  }
+}
+
+try {
+  node {
+    stage('Build') {
+      git url: 'git@github.com:alphagov/notifications-admin.git', branch: 'cloudfoundry', credentialsId: 'github_com_and_gds'
+      checkout scm
+
+      milestone 10
+      withEnv(["PIP_ACCEL_CACHE=${env.JENKINS_HOME}/cache/pip-accel"]) {
+        sh 'make cf-build-with-docker'
+      }
+
+      stash name: 'source', excludes: 'node_modules/**,venv/**,wheelhouse/**', useDefaultExcludes: false
+    }
+
+    stage('Test') {
+      milestone 20
+      sh 'make test-with-docker'
+
+      try {
+        junit 'test_results.xml'
+      } catch(err) {
+        echo "Collecting jUnit results failed: ${err}"
+      }
+
+      try {
+        withCredentials([string(credentialsId: 'coveralls_repo_token_api', variable: 'COVERALLS_REPO_TOKEN')]) {
+          sh 'make coverage-with-docker'
+        }
+      } catch(err) {
+        echo "Coverage failed: ${err}"
+      }
+    }
+
+    stage('Preview') {
+      if (deployToPreview == "true") {
+        milestone 30
+        deploy 'preview'
+      } else {
+        echo 'Preview skipped.'
+      }
+    }
+
+    stage('Preview tests') {
+      if (deployToPreview == "true") {
+        buildJobWithRetry 'notify-functional-tests-preview'
+      } else {
+        echo 'Preview tests skipped.'
+      }
+    }
+  }
+
+  stage('Staging') {
+    if (deployToStaging == "true") {
+      input 'Approve?'
+      milestone 40
+      node {
+        unstash 'source'
+        deploy 'staging'
+      }
+    } else {
+      echo 'Staging skipped.'
+    }
+  }
+
+  stage('Staging tests') {
+    if (deployToStaging == "true") {
+      buildJobWithRetry 'notify-functional-tests-staging'
+    } else {
+      echo 'Staging tests skipped'
+    }
+  }
+
+  stage('Prod') {
+    if (deployToProduction == "true") {
+      input 'Approve?'
+      milestone 50
+      node {
+        unstash 'source'
+        deploy 'production'
+      }
+    } else {
+      echo 'Production skipped.'
+    }
+  }
+
+  stage('Prod tests') {
+    if (deployToProduction == "true") {
+      buildJobWithRetry 'notify-functional-admin-tests-production'
+      buildJobWithRetry 'notify-functional-api-email-test-production'
+      buildJobWithRetry 'notify-functional-api-sms-test-production'
+    } else {
+      echo 'Production tests skipped.'
+    }
+  }
+} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException fie) {
+  currentBuild.result = 'ABORTED'
+} catch (err) {
+  currentBuild.result = 'FAILURE'
+  echo "Pipeline failed: ${err}"
+  slackSend channel: '#govuk-notify', message: "${env.JOB_NAME} - #${env.BUILD_NUMBER} failed (<${env.BUILD_URL}|Open>)", color: 'danger'
+} finally {
+  node {
+    try {
+      step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: 'notify-support+jenkins@digital.cabinet-office.gov.uk', sendToIndividuals: false])
+    } catch(err) {
+      echo "Sending email failed: ${err}"
+    }
+
+    try {
+      sh 'make clean-docker-containers'
+    } catch(err) {
+      echo "Cleaning up Docker containers failed: ${err}"
+    }
+  }
+}

Makefile

@@ -8,7 +8,9 @@ APP_VERSION_FILE = app/version.py
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
 GIT_COMMIT ?= $(shell git rev-parse HEAD)
 
-DOCKER_BUILDER_IMAGE_NAME = govuk/notify-admin-builder
+DOCKER_IMAGE_TAG := $(shell cat docker/VERSION)
+DOCKER_BUILDER_IMAGE_NAME = govuk/notify-admin-builder:${DOCKER_IMAGE_TAG}
+DOCKER_TTY ?= $(if ${JENKINS_HOME},,t)
 
 BUILD_TAG ?= notifications-admin-manual
 BUILD_NUMBER ?= 0
@@ -20,6 +22,10 @@ DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG}
 CODEDEPLOY_PREFIX ?= notifications-admin
 CODEDEPLOY_APP_NAME ?= notify-admin
 
+CF_API ?= api.cloud.service.gov.uk
+CF_ORG ?= govuk-notify
+CF_SPACE ?= ${DEPLOY_ENV}
+
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -28,8 +34,8 @@ help:
 venv: venv/bin/activate ## Create virtualenv if it does not exist
 
 venv/bin/activate:
-	test -d venv || virtualenv venv
+	test -d venv || virtualenv venv -p python3
-	./venv/bin/pip install pip-accel
+	. venv/bin/activate && pip install pip-accel
 
 .PHONY: check-env-vars
 check-env-vars: ## Check mandatory environment variables
@@ -38,6 +44,12 @@ check-env-vars: ## Check mandatory environment variables
 	$(if ${AWS_ACCESS_KEY_ID},,$(error Must specify AWS_ACCESS_KEY_ID))
 	$(if ${AWS_SECRET_ACCESS_KEY},,$(error Must specify AWS_SECRET_ACCESS_KEY))
 
+.PHONY: sandbox
+sandbox: ## Set environment to sandbox
+	$(eval export DEPLOY_ENV=sandbox)
+	$(eval export DNS_NAME="cloudapps.digital")
+	@true
+
 .PHONY: preview
 preview: ## Set environment to preview
 	$(eval export DEPLOY_ENV=preview)
@@ -62,7 +74,7 @@ dependencies: venv ## Install build dependencies
 	npm install
 	npm rebuild node-sass
 	mkdir -p ${PIP_ACCEL_CACHE}
-	PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} ./venv/bin/pip-accel install -r requirements_for_test.txt
+	. venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements_for_test.txt
 
 .PHONY: generate-version-file
 generate-version-file: ## Generates the app version file
@@ -70,7 +82,11 @@ generate-version-file: ## Generates the app version file
 
 .PHONY: build
 build: dependencies generate-version-file ## Build project
-	./venv/bin/pip-accel wheel --wheel-dir=wheelhouse -r requirements.txt
+	npm run build
+	. venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel wheel --wheel-dir=wheelhouse -r requirements.txt
+
+.PHONY: cf-build
+cf-build: dependencies generate-version-file ## Build project
 	npm run build
 
 .PHONY: build-codedeploy-artifact
@@ -109,19 +125,20 @@ deploy-check-autoscaling-processes: check-aws-vars ## Returns with the number of
 
 .PHONY: coverage
 coverage: venv ## Create coverage report
-	./venv/bin/coveralls
+	. venv/bin/activate && coveralls
 
 .PHONY: prepare-docker-build-image
 prepare-docker-build-image: ## Prepare the Docker builder image
 	mkdir -p ${PIP_ACCEL_CACHE}
-	make -C docker build-build-image
+	make -C docker build
 
-.PHONY: build-with-docker
+define run_docker_container
-build-with-docker: prepare-docker-build-image ## Build inside a Docker container
+	@docker run -i${DOCKER_TTY} --rm \
-	@docker run -i --rm \
+		--name "${DOCKER_CONTAINER_PREFIX}-${1}" \
-		--name "${DOCKER_CONTAINER_PREFIX}-build" \
+		-v "`pwd`:/var/project" \
-		-v `pwd`:/var/project \
+		-v "${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel" \
-		-v ${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel \
+		-e UID=$(shell id -u) \
+		-e GID=$(shell id -g) \
 		-e GIT_COMMIT=${GIT_COMMIT} \
 		-e BUILD_NUMBER=${BUILD_NUMBER} \
 		-e BUILD_URL=${BUILD_URL} \
@@ -130,31 +147,6 @@ build-with-docker: prepare-docker-build-image ## Build inside a Docker container
 		-e https_proxy="${HTTPS_PROXY}" \
 		-e HTTPS_PROXY="${HTTPS_PROXY}" \
 		-e NO_PROXY="${NO_PROXY}" \
-		${DOCKER_BUILDER_IMAGE_NAME} \
-		make build
-
-.PHONY: test-with-docker
-test-with-docker: prepare-docker-build-image ## Run tests inside a Docker container
-	@docker run -i --rm \
-		--name "${DOCKER_CONTAINER_PREFIX}-test" \
-		-v `pwd`:/var/project \
-		-e GIT_COMMIT=${GIT_COMMIT} \
-		-e BUILD_NUMBER=${BUILD_NUMBER} \
-		-e BUILD_URL=${BUILD_URL} \
-		-e http_proxy="${HTTP_PROXY}" \
-		-e HTTP_PROXY="${HTTP_PROXY}" \
-		-e https_proxy="${HTTPS_PROXY}" \
-		-e HTTPS_PROXY="${HTTPS_PROXY}" \
-		-e NO_PROXY="${NO_PROXY}" \
-		${DOCKER_BUILDER_IMAGE_NAME} \
-		make test
-
-# FIXME: CIRCLECI=1 is an ugly hack because the coveralls-python library sends the PR link only this way
-.PHONY: coverage-with-docker
-coverage-with-docker: prepare-docker-build-image ## Generates coverage report inside a Docker container
-	@docker run -i --rm \
-		--name "${DOCKER_CONTAINER_PREFIX}-coverage" \
-		-v `pwd`:/var/project \
 		-e COVERALLS_REPO_TOKEN=${COVERALLS_REPO_TOKEN} \
 		-e CIRCLECI=1 \
 		-e CI_NAME=${CI_NAME} \
@@ -162,17 +154,60 @@ coverage-with-docker: prepare-docker-build-image ## Generates coverage report in
 		-e CI_BUILD_URL=${BUILD_URL} \
 		-e CI_BRANCH=${GIT_BRANCH} \
 		-e CI_PULL_REQUEST=${CI_PULL_REQUEST} \
-		-e http_proxy="${HTTP_PROXY}" \
+		-e CF_API="${CF_API}" \
-		-e HTTP_PROXY="${HTTP_PROXY}" \
+		-e CF_USERNAME="${CF_USERNAME}" \
-		-e https_proxy="${HTTPS_PROXY}" \
+		-e CF_PASSWORD="${CF_PASSWORD}" \
-		-e HTTPS_PROXY="${HTTPS_PROXY}" \
+		-e CF_ORG="${CF_ORG}" \
-		-e NO_PROXY="${NO_PROXY}" \
+		-e CF_SPACE="${CF_SPACE}" \
 		${DOCKER_BUILDER_IMAGE_NAME} \
-		make coverage
+		${2}
+endef
+
+.PHONY: build-with-docker
+build-with-docker: prepare-docker-build-image ## Build inside a Docker container
+	$(call run_docker_container,build,gosu hostuser make build)
+
+.PHONY: cf-build-with-docker
+cf-build-with-docker: prepare-docker-build-image ## Build inside a Docker container
+	$(call run_docker_container,build,gosu hostuser make cf-build)
+
+.PHONY: test-with-docker
+test-with-docker: prepare-docker-build-image ## Run tests inside a Docker container
+	$(call run_docker_container,test,gosu hostuser make test)
+
+# FIXME: CIRCLECI=1 is an ugly hack because the coveralls-python library sends the PR link only this way
+.PHONY: coverage-with-docker
+coverage-with-docker: prepare-docker-build-image ## Generates coverage report inside a Docker container
+	$(call run_docker_container,coverage,gosu hostuser make coverage)
 
 .PHONY: clean-docker-containers
 clean-docker-containers: ## Clean up any remaining docker containers
 	docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true
 
+.PHONY: clean
 clean:
-	rm -rf node_modules cache target venv .coverage
+	rm -rf node_modules cache target venv .coverage wheelhouse
+
+.PHONY: cf-login
+cf-login: ## Log in to Cloud Foundry
+	$(if ${CF_USERNAME},,$(error Must specify CF_USERNAME))
+	$(if ${CF_PASSWORD},,$(error Must specify CF_PASSWORD))
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
+	@echo "Logging in to Cloud Foundry on ${CF_API}"
+	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
+
+.PHONY: cf-deploy
+cf-deploy: ## Deploys the app to Cloud Foundry
+	$(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid notify-admin) | jq -r ".entity.instances"))
+	@echo "Original instance count: ${ORIG_INSTANCES}"
+	cf check-manifest notify-admin -f manifest-${CF_SPACE}.yml
+	cf zero-downtime-push notify-admin -f manifest-${CF_SPACE}.yml
+	cf scale -i ${ORIG_INSTANCES} notify-admin
+
+.PHONY: cf-push
+cf-push:
+	cf push notify-admin -f manifest-${CF_SPACE}.yml
+
+.PHONY: cf-deploy-with-docker
+cf-deploy-with-docker: prepare-docker-build-image ## Deploys the app to Cloud Foundry from a new Docker container
+	$(call run_docker_container,cf-deploy,make cf-login cf-deploy)

app/.gitignore

@@ -0,0 +1,3 @@
+/version.py
+/templates/govuk_template.html
+/static

app/__init__.py

@@ -1,6 +1,7 @@
 import os
 import re
 import urllib
+import json
 from datetime import datetime, timedelta, timezone
 from time import monotonic
 
@@ -72,11 +73,13 @@ current_service = LocalProxy(partial(_lookup_req_object, 'service'))
 
 
 def create_app():
-    from config import configs
+    from app.config import configs
 
     application = Flask(__name__)
 
-    application.config.from_object(configs[os.environ['NOTIFY_ENVIRONMENT']])
+    notify_environment = os.environ['NOTIFY_ENVIRONMENT']
+
+    application.config.from_object(configs[notify_environment])
 
     init_app(application)
     statsd_client.init_app(application)

app/cloudfoundry_config.py

@@ -0,0 +1,50 @@
+"""
+Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated
+on local/aws boxes
+"""
+
+import os
+import json
+
+
+def extract_cloudfoundry_config():
+    vcap_services = json.loads(os.environ['VCAP_SERVICES'])
+
+    set_config_env_vars(vcap_services)
+
+
+def set_config_env_vars(vcap_services):
+    vcap_application = json.loads(os.environ.get('VCAP_APPLICATION'))
+    os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
+    os.environ['LOGGING_STDOUT_JSON'] = '1'
+
+    for s in vcap_services['user-provided']:
+        if s['name'] == 'notify-config':
+            extract_notify_config(s)
+        elif s['name'] == 'notify-aws':
+            extract_notify_aws_config(s)
+        elif s['name'] == 'hosted-graphite':
+            extract_hosted_graphite_config(s)
+        elif s['name'] == 'deskpro':
+            extract_deskpro_config(s)
+
+
+def extract_notify_config(notify_config):
+    os.environ['ADMIN_CLIENT_SECRET'] = notify_config['credentials']['admin_client_secret']
+    os.environ['API_HOST_NAME'] = notify_config['credentials']['api_host_name']
+    os.environ['SECRET_KEY'] = notify_config['credentials']['secret_key']
+    os.environ['DANGEROUS_SALT'] = notify_config['credentials']['dangerous_salt']
+
+
+def extract_notify_aws_config(aws_config):
+    os.environ['AWS_ACCESS_KEY_ID'] = aws_config['credentials']['aws_access_key_id']
+    os.environ['AWS_SECRET_ACCESS_KEY'] = aws_config['credentials']['aws_secret_access_key']
+
+
+def extract_hosted_graphite_config(hosted_graphite_config):
+    os.environ['STATSD_PREFIX'] = hosted_graphite_config['credentials']['statsd_prefix']
+
+
+def extract_deskpro_config(deskpro_config):
+    os.environ['DESKPRO_API_HOST'] = deskpro_config['credentials']['api_host']
+    os.environ['DESKPRO_API_KEY'] = deskpro_config['credentials']['api_key']

app/config.py

@@ -2,6 +2,13 @@ import os
 from datetime import timedelta
 
 
+if os.environ.get('VCAP_SERVICES'):
+    # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
+    # standard environment variables from it
+    from app.cloudfoundry_config import extract_cloudfoundry_config
+    extract_cloudfoundry_config()
+
+
 class Config(object):
     ADMIN_CLIENT_SECRET = os.environ['ADMIN_CLIENT_SECRET']
     API_HOST_NAME = os.environ['API_HOST_NAME']
@@ -9,13 +16,17 @@ class Config(object):
     DANGEROUS_SALT = os.environ['DANGEROUS_SALT']
     DESKPRO_API_HOST = os.environ['DESKPRO_API_HOST']
     DESKPRO_API_KEY = os.environ['DESKPRO_API_KEY']
+
     # Hosted graphite statsd prefix
     STATSD_PREFIX = os.getenv('STATSD_PREFIX')
 
+    # Logging
+    DEBUG = False
+    LOGGING_STDOUT_JSON = os.getenv('LOGGING_STDOUT_JSON') == '1'
+
     DESKPRO_DEPT_ID = 5
     DESKPRO_ASSIGNED_AGENT_TEAM_ID = 5
 
-    DEBUG = False
     ADMIN_CLIENT_USER_NAME = 'notify-admin'
     ASSETS_DEBUG = False
     AWS_REGION = 'eu-west-1'
@@ -112,10 +123,24 @@ class Live(Config):
     NOTIFY_ENVIRONMENT = 'live'
 
 
+class CloudFoundryConfig(Config):
+    pass
+
+
+# CloudFoundry sandbox
+class Sandbox(CloudFoundryConfig):
+    HTTP_PROTOCOL = 'https'
+    HEADER_COLOUR = '#F499BE'  # $baby-pink
+    STATSD_ENABLED = True
+    CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
+    NOTIFY_ENVIRONMENT = 'sandbox'
+
+
 configs = {
     'development': Development,
     'test': Test,
     'preview': Preview,
     'staging': Staging,
-    'live': Live
+    'live': Live,
+    'sandbox': Sandbox
 }

docker/Dockerfile

@@ -0,0 +1,82 @@
+FROM python:3.4-slim
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+ENV PYTHONUNBUFFERED=1 \
+	DEBIAN_FRONTEND=noninteractive \
+	NODEJS_VERSION=6.3.1-1nodesource1~jessie1 \
+	GOSU_VERSION=1.10
+
+RUN \
+	echo "Install base packages" \
+	&& ([ -z "$HTTP_PROXY" ] || echo "Acquire::http::Proxy \"${HTTP_PROXY}\";" > /etc/apt/apt.conf.d/99HttpProxy) \
+	&& apt-get update \
+	&& apt-get install -y --no-install-recommends \
+		apt-transport-https \
+		make \
+		curl \
+		git \
+		build-essential \
+		libxml2-dev \
+		libxslt-dev \
+		zlib1g-dev \
+		zip \
+		rlwrap \
+		python-dev \
+		libffi-dev \
+		libssl-dev \
+		libexif-dev \
+		libfreetype6-dev \
+		libjpeg-dev \
+		liblcms2-2 \
+		libtiff5-dev \
+		zlib1g-dev \
+		libpango1.0-dev \
+		libcairo2-dev \
+		libmagickwand-dev \
+		ghostscript \
+		jq \
+
+	&& echo "Install nodejs" \
+	&& cd /tmp \
+	&& curl -x "$HTTP_PROXY" -sSLO https://deb.nodesource.com/node_6.x/pool/main/n/nodejs/nodejs_${NODEJS_VERSION}_amd64.deb \
+	&& dpkg -i /tmp/nodejs_${NODEJS_VERSION}_amd64.deb \
+
+	&& echo "Clean up" \
+	&& rm -rf /var/lib/apt/lists/* /tmp/*
+
+RUN \
+	echo "Install global pip packages" \
+	&& pip install \
+		virtualenv \
+		awscli \
+		wheel
+
+RUN \
+	echo "Install Cloud Foundry CLI" \
+	&& curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \
+	&& dpkg -i /tmp/cloudfoundry-cli.deb \
+	&& cf install-plugin -r CF-Community -f "autopilot" \
+	&& cf install-plugin -r CF-Community -f "blue-green-deploy" \
+	&& cf install-plugin -r CF-Community -f "antifreeze"
+
+COPY tianon.gpg /tmp/tianon.gpg
+
+RUN \
+	echo "Install gosu" \
+	&& curl -sSL -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
+	&& curl -sSL -o /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
+	&& export GNUPGHOME="$(mktemp -d)" \
+	&& gpg --import /tmp/tianon.gpg \
+	&& gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
+	&& rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \
+	&& chmod +x /usr/local/bin/gosu \
+	&& gosu nobody true
+
+WORKDIR /var/project
+
+COPY entrypoint.sh /usr/local/bin/docker-entrypoint
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]

docker/Dockerfile-build

@@ -1,55 +0,0 @@
-FROM python:3.4-slim
-
-ARG HTTP_PROXY
-ARG HTTPS_PROXY
-ARG NO_PROXY
-
-ENV PYTHONUNBUFFERED=1 \
-	DEBIAN_FRONTEND=noninteractive \
-	NODEJS_VERSION=6.3.1-1nodesource1~jessie1
-
-RUN \
-	echo "Install base packages" \
-	&& ([ -z "$HTTP_PROXY" ] || echo "Acquire::http::Proxy \"${HTTP_PROXY}\";" > /etc/apt/apt.conf.d/99HttpProxy) \
-	&& apt-get update \
-	&& apt-get install -y --no-install-recommends \
-		apt-transport-https \
-		make \
-		curl \
-		git \
-		build-essential \
-		libxml2-dev \
-		libxslt-dev \
-		zlib1g-dev \
-		zip \
-		rlwrap \
-		python-dev \
-		libffi-dev \
-		libssl-dev \
-		libexif-dev \
-		libfreetype6-dev \
-		libjpeg-dev \
-		liblcms2-2 \
-		libtiff5-dev \
-		zlib1g-dev \
-		libpango1.0-dev \
-		libcairo2-dev \
-		libmagickwand-dev \
-		ghostscript \
-
-	&& echo "Install nodejs" \
-	&& cd /tmp \
-	&& curl -x "$HTTP_PROXY" -sSLO https://deb.nodesource.com/node_6.x/pool/main/n/nodejs/nodejs_${NODEJS_VERSION}_amd64.deb \
-	&& dpkg -i /tmp/nodejs_${NODEJS_VERSION}_amd64.deb \
-
-	&& echo "Clean up" \
-	&& rm -rf /var/lib/apt/lists/* /tmp/*
-
-RUN \
-	echo "Install global pip packages" \
-	&& pip install \
-		virtualenv \
-		awscli \
-		wheel
-
-WORKDIR /var/project

docker/Makefile

@@ -1,17 +1,33 @@
 .DEFAULT_GOAL := help
 SHELL := /bin/bash
+DOCKER_IMAGE_TAG := $(shell cat VERSION)
 
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-.PHONY: build-build-image
+.PHONY: build
-build-build-image:
+build:
 	docker build \
 		--pull \
 		--build-arg HTTP_PROXY="${HTTP_PROXY}" \
 		--build-arg HTTPS_PROXY="${HTTP_PROXY}" \
 		--build-arg NO_PROXY="${NO_PROXY}" \
-		-f Dockerfile-build \
+		-t govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} \
-		-t govuk/notify-admin-builder \
 		.
+
+.PHONY: bash
+bash:
+	docker run -it --rm \
+		-e UID=$(shell id -u) \
+		-e GID=$(shell id -g) \
+		govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} \
+		bash
+
+.PHONY: bash
+bash-hostuser:
+	docker run -it --rm \
+		-e UID=$(shell id -u) \
+		-e GID=$(shell id -g) \
+		govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} \
+		gosu hostuser bash

docker/VERSION

@@ -0,0 +1 @@
+2

docker/entrypoint.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -eo pipefail; [[ "$TRACE" ]] && set -x
+
+if [[ "$(id -u)" -ne 0 ]]; then
+    echo 'docker-entrypoint requires root' >&2
+    exit 1
+fi
+
+if [ -z "$UID" ] || [ "$UID" = "0" ]; then
+  echo "UID must be specified as a positive integer"
+  exit 1
+fi
+
+if [ -z "$GID" ] || [ "$GID" = "0" ]; then
+  echo "GID must be specified as positive integer"
+  exit 1
+fi
+
+USER=$(id -un $UID 2>/dev/null || echo "hostuser")
+GROUP=$(getent group $GID | cut -d: -f1 || echo "hostgroup")
+
+if [ "$USER" = "hostuser" ]; then
+  useradd -u $UID -s /bin/bash -m $USER
+fi
+
+if [ "$GROUP" = "hostgroup" ]; then
+  groupadd -g $GID $GROUP
+fi
+
+usermod -g $GROUP $USER
+
+exec "$@"

manifest-preview.yml

@@ -0,0 +1,16 @@
+---
+
+applications:
+  - name: notify-admin
+    buildpack: python_buildpack
+    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+    services:
+      - notify-aws
+      - notify-config
+      - hosted-graphite
+      - deskpro
+    routes:
+      - route: notify-admin-preview.cloudapps.digital
+      - route: admin-paas.notify.works
+    instances: 1
+    memory: 512M

manifest-production.yml

@@ -0,0 +1,16 @@
+---
+
+applications:
+  - name: notify-admin
+    buildpack: python_buildpack
+    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+    services:
+      - notify-aws
+      - notify-config
+      - hosted-graphite
+      - deskpro
+    routes:
+      - route: notify-admin-production.cloudapps.digital
+      - route: admin-paas.notifications.service.gov.uk
+    instances: 2
+    memory: 2048M

manifest-sandbox.yml

@@ -0,0 +1,15 @@
+---
+
+applications:
+  - name: notify-admin
+    buildpack: python_buildpack
+    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+    services:
+      - notify-aws
+      - notify-config
+      - hosted-graphite
+      - deskpro
+    routes:
+      - route: notify-admin-sandbox.cloudapps.digital
+    instances: 1
+    memory: 512M

manifest-staging.yml

@@ -0,0 +1,16 @@
+---
+
+applications:
+  - name: notify-admin
+    buildpack: python_buildpack
+    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+    services:
+      - notify-aws
+      - notify-config
+      - hosted-graphite
+      - deskpro
+    routes:
+      - route: notify-admin-staging.cloudapps.digital
+      - route: admin-paas.staging-notify.works
+    instances: 2
+    memory: 2048M

requirements.txt

@@ -1,3 +1,4 @@
+cffi==1.9.1
 ago==0.0.8
 Flask==0.10.1
 Flask-Script==2.0.5
@@ -22,7 +23,10 @@ pyexcel-xlsx==0.1.0
 pyexcel-ods3==0.1.1
 pytz==2016.4
 wand==0.4.4
+gunicorn==19.6.0
+whitenoise==1.0.6  #manages static assets
 
-git+https://github.com/alphagov/notifications-python-client.git@3.0.1#egg=notifications-python-client==3.0.1
+# pin to minor version 3.1.x
+notifications-python-client>=3.1,<3.2
 
-git+https://github.com/alphagov/notifications-utils.git@13.0.1#egg=notifications-utils==13.0.1
+git+https://github.com/alphagov/notifications-utils.git@13.1.0#egg=notifications-utils==13.1.0

runtime.txt

@@ -0,0 +1 @@
+python-3.5.2

tests/app/test_cloudfoundry_config.py

@@ -0,0 +1,128 @@
+import os
+import json
+
+import pytest
+
+from app.cloudfoundry_config import extract_cloudfoundry_config, set_config_env_vars
+
+
+@pytest.fixture
+def notify_config():
+    return {
+        'name': 'notify-config',
+        'credentials': {
+            'api_host_name': 'api host name',
+            'admin_client_secret': 'admin client secret',
+            'secret_key': 'secret key',
+            'dangerous_salt': 'dangerous salt',
+        }
+    }
+
+
+@pytest.fixture
+def aws_config():
+    return {
+        'name': 'notify-aws',
+        'credentials': {
+            'aws_access_key_id': 'aws access key id',
+            'aws_secret_access_key': 'aws secret access key',
+        }
+    }
+
+
+@pytest.fixture
+def hosted_graphite_config():
+    return {
+        'name': 'hosted-graphite',
+        'credentials': {
+            'statsd_prefix': 'statsd prefix'
+        }
+    }
+
+
+@pytest.fixture
+def deskpro_config():
+    return {
+        'name': 'deskpro',
+        'credentials': {
+            'api_host': 'deskpro api host',
+            'api_key': 'deskpro api key'
+        }
+    }
+
+
+@pytest.fixture
+def cloudfoundry_config(
+        notify_config,
+        aws_config,
+        hosted_graphite_config,
+        deskpro_config,
+):
+    return {
+        'user-provided': [
+            notify_config,
+            aws_config,
+            hosted_graphite_config,
+            deskpro_config,
+        ]
+    }
+
+
+@pytest.fixture
+def cloudfoundry_environ(monkeypatch, cloudfoundry_config):
+    monkeypatch.setenv('VCAP_SERVICES', json.dumps(cloudfoundry_config))
+    monkeypatch.setenv('VCAP_APPLICATION', '{"space_name":"🚀🌌"}')
+
+
+@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
+def test_extract_cloudfoundry_config_populates_other_vars():
+    extract_cloudfoundry_config()
+
+    assert os.environ['LOGGING_STDOUT_JSON'] == '1'
+    assert os.environ['NOTIFY_ENVIRONMENT'] == '🚀🌌'
+
+
+@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
+def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config):
+    cloudfoundry_config['foo'] = {'credentials': {'foo': 'foo'}}
+    cloudfoundry_config['user-provided'].append({
+        'name': 'bar', 'credentials': {'bar': 'bar'}
+    })
+
+    set_config_env_vars(cloudfoundry_config)
+
+    assert 'foo' not in os.environ
+    assert 'bar' not in os.environ
+
+
+@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
+def test_notify_config():
+    extract_cloudfoundry_config()
+
+    assert os.environ['API_HOST_NAME'] == 'api host name'
+    assert os.environ['ADMIN_CLIENT_SECRET'] == 'admin client secret'
+    assert os.environ['SECRET_KEY'] == 'secret key'
+    assert os.environ['DANGEROUS_SALT'] == 'dangerous salt'
+
+
+@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
+def test_aws_config():
+    extract_cloudfoundry_config()
+
+    assert os.environ['AWS_ACCESS_KEY_ID'] == 'aws access key id'
+    assert os.environ['AWS_SECRET_ACCESS_KEY'] == 'aws secret access key'
+
+
+@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
+def test_hosted_graphite_config():
+    extract_cloudfoundry_config()
+
+    assert os.environ['STATSD_PREFIX'] == 'statsd prefix'
+
+
+@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
+def test_deskpro_config():
+    extract_cloudfoundry_config()
+
+    assert os.environ['DESKPRO_API_HOST'] == 'deskpro api host'
+    assert os.environ['DESKPRO_API_KEY'] == 'deskpro api key'

tests/app/test_config.py

@@ -0,0 +1,74 @@
+import os
+import importlib
+from unittest import mock
+
+import pytest
+
+from app import config
+
+
+def cf_conf():
+    os.environ['API_HOST_NAME'] = 'cf'
+
+
+@pytest.fixture
+def reload_config():
+    """
+    Reset config, by simply re-running config.py from a fresh environment
+    """
+    old_env = os.environ.copy()
+
+    yield
+
+    os.environ = old_env
+    importlib.reload(config)
+
+
+def test_load_cloudfoundry_config_if_available(monkeypatch, reload_config):
+    os.environ['API_HOST_NAME'] = 'env'
+    monkeypatch.setenv('VCAP_SERVICES', 'some json blob')
+
+    with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config:
+        # reload config so that its module level code (ie: all of it) is re-instantiated
+        importlib.reload(config)
+
+    assert cf_config.called
+
+    assert os.environ['API_HOST_NAME'] == 'cf'
+    assert config.Config.API_HOST_NAME == 'cf'
+
+
+def test_load_config_if_cloudfoundry_not_available(monkeypatch, reload_config):
+    os.environ['API_HOST_NAME'] = 'env'
+
+    monkeypatch.delenv('VCAP_SERVICES', raising=False)
+
+    with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config:
+        # reload config so that its module level code (ie: all of it) is re-instantiated
+        importlib.reload(config)
+
+    assert not cf_config.called
+
+    assert os.environ['API_HOST_NAME'] == 'env'
+    assert config.Config.API_HOST_NAME == 'env'
+
+
+def test_logging_stdout_json_defaults_to_off(reload_config):
+    os.environ.pop('LOGGING_STDOUT_JSON', None)
+    assert config.Config.LOGGING_STDOUT_JSON is False
+
+
+def test_logging_stdout_json_sets_to_off_if_not_recognised(reload_config):
+    os.environ['LOGGING_STDOUT_JSON'] = 'foo'
+
+    importlib.reload(config)
+
+    assert config.Config.LOGGING_STDOUT_JSON is False
+
+
+def test_logging_stdout_json_sets_to_on_if_set_to_1(reload_config):
+    os.environ['LOGGING_STDOUT_JSON'] = '1'
+
+    importlib.reload(config)
+
+    assert config.Config.LOGGING_STDOUT_JSON is True

tests/conftest.py

@@ -1,8 +1,16 @@
+import os
 from datetime import date, datetime, timedelta
 from unittest.mock import Mock
+
 import pytest
+from notifications_python_client.errors import HTTPError
 
 from app import create_app
+from app.notify_client.models import (
+    User,
+    InvitedUser
+)
+
 
 from . import (
     service_json,
@@ -14,14 +22,10 @@ from . import (
     notification_json,
     invite_json,
     sample_uuid,
-    generate_uuid, single_notification_json)
+    generate_uuid,
-from app.notify_client.models import (
+    single_notification_json
-    User,
-    InvitedUser
 )
 
-from notifications_python_client.errors import HTTPError
-
 
 @pytest.fixture(scope='session')
 def app_(request):
@@ -1340,3 +1344,15 @@ def logged_in_client(
 ):
     client.login(active_user_with_permissions)
     yield client
+
+
+@pytest.fixture
+def os_environ():
+    """
+    clear os.environ, and restore it after the test runs
+    """
+    # for use whenever you expect code to edit environment variables
+    old_env = os.environ.copy()
+    os.environ = {}
+    yield
+    os.environ = old_env

wsgi.py

@@ -1,12 +1,18 @@
 from credstash import getAllSecrets
+from whitenoise import WhiteNoise
 import os
 
-# on aws get secrets and export to env
+# On AWS get secrets and export to env, skip this on Cloud Foundry
-os.environ.update(getAllSecrets(region="eu-west-1"))
+if os.getenv('VCAP_SERVICES') is None:
+    os.environ.update(getAllSecrets(region="eu-west-1"))
 
 from app import create_app  # noqa
 
-application = create_app()
+PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
+STATIC_ROOT = os.path.join(PROJECT_ROOT, 'app', 'static')
+STATIC_URL = 'static/'
+
+application = WhiteNoise(create_app(), STATIC_ROOT, STATIC_URL)
 
 if __name__ == "__main__":
         application.run()