diff --git a/.cfignore b/.cfignore new file mode 120000 index 000000000..3e4e48b0b --- /dev/null +++ b/.cfignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 264e9b956..3f7cdab2e 100644 --- a/.gitignore +++ b/.gitignore @@ -64,13 +64,9 @@ target/ app/assets/stylesheets/govuk_template/.sass-cache/ .sass-cache/ cache/ -app/static node_modules bower_components -app/templates/govuk_template.html npm-debug.log environment.sh .envrc - -app/version.py diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..12921c371 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,165 @@ +#!groovy + +def deploy(cfEnv) { + waitUntil { + try { + lock(cfEnv) { + withCredentials([ + string(credentialsId: 'paas_username', variable: 'CF_USERNAME'), + string(credentialsId: 'paas_password', variable: 'CF_PASSWORD') + ]) { + withEnv(["CF_SPACE=${cfEnv}"]) { + sh 'make cf-deploy-with-docker' + } + } + gitCommit = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() + sh("git tag -f deployed-to-cf-${cfEnv} ${gitCommit}") + sh("git push -f origin deployed-to-cf-${cfEnv}") + } + true + } catch(err) { + echo "Deployment to ${cfEnv} failed: ${err}" + try { + slackSend channel: '#govuk-notify', message: "Deployment to ${cfEnv} failed. Please retry or abort: <${env.BUILD_URL}|${env.JOB_NAME} - #${env.BUILD_NUMBER}>", color: 'danger' + } catch(err2) { + echo "Sending Slack message failed: ${err2}" + } + input "Stage failed. Retry?" + false + } + } +} + +def buildJobWithRetry(jobName) { + waitUntil { + try { + build job: jobName + true + } catch(err) { + echo "${jobName} failed: ${err}" + try { + slackSend channel: '#govuk-notify', message: "${jobName} failed. Please retry or abort: <${env.BUILD_URL}|${env.JOB_NAME} - #${env.BUILD_NUMBER}>", color: 'danger' + } catch(err2) { + echo "Sending Slack message failed: ${err2}" + } + input "${jobName} failed. Retry?" + false + } + } +} + +try { + node { + stage('Build') { + git url: 'git@github.com:alphagov/notifications-admin.git', branch: 'cloudfoundry', credentialsId: 'github_com_and_gds' + checkout scm + + milestone 10 + withEnv(["PIP_ACCEL_CACHE=${env.JENKINS_HOME}/cache/pip-accel"]) { + sh 'make cf-build-with-docker' + } + + stash name: 'source', excludes: 'node_modules/**,venv/**,wheelhouse/**', useDefaultExcludes: false + } + + stage('Test') { + milestone 20 + sh 'make test-with-docker' + + try { + junit 'test_results.xml' + } catch(err) { + echo "Collecting jUnit results failed: ${err}" + } + + try { + withCredentials([string(credentialsId: 'coveralls_repo_token_api', variable: 'COVERALLS_REPO_TOKEN')]) { + sh 'make coverage-with-docker' + } + } catch(err) { + echo "Coverage failed: ${err}" + } + } + + stage('Preview') { + if (deployToPreview == "true") { + milestone 30 + deploy 'preview' + } else { + echo 'Preview skipped.' + } + } + + stage('Preview tests') { + if (deployToPreview == "true") { + buildJobWithRetry 'notify-functional-tests-preview' + } else { + echo 'Preview tests skipped.' + } + } + } + + stage('Staging') { + if (deployToStaging == "true") { + input 'Approve?' + milestone 40 + node { + unstash 'source' + deploy 'staging' + } + } else { + echo 'Staging skipped.' + } + } + + stage('Staging tests') { + if (deployToStaging == "true") { + buildJobWithRetry 'notify-functional-tests-staging' + } else { + echo 'Staging tests skipped' + } + } + + stage('Prod') { + if (deployToProduction == "true") { + input 'Approve?' + milestone 50 + node { + unstash 'source' + deploy 'production' + } + } else { + echo 'Production skipped.' + } + } + + stage('Prod tests') { + if (deployToProduction == "true") { + buildJobWithRetry 'notify-functional-admin-tests-production' + buildJobWithRetry 'notify-functional-api-email-test-production' + buildJobWithRetry 'notify-functional-api-sms-test-production' + } else { + echo 'Production tests skipped.' + } + } +} catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException fie) { + currentBuild.result = 'ABORTED' +} catch (err) { + currentBuild.result = 'FAILURE' + echo "Pipeline failed: ${err}" + slackSend channel: '#govuk-notify', message: "${env.JOB_NAME} - #${env.BUILD_NUMBER} failed (<${env.BUILD_URL}|Open>)", color: 'danger' +} finally { + node { + try { + step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: 'notify-support+jenkins@digital.cabinet-office.gov.uk', sendToIndividuals: false]) + } catch(err) { + echo "Sending email failed: ${err}" + } + + try { + sh 'make clean-docker-containers' + } catch(err) { + echo "Cleaning up Docker containers failed: ${err}" + } + } +} diff --git a/Makefile b/Makefile index d677d5ff8..5ad767fb1 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,9 @@ APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") GIT_COMMIT ?= $(shell git rev-parse HEAD) -DOCKER_BUILDER_IMAGE_NAME = govuk/notify-admin-builder +DOCKER_IMAGE_TAG := $(shell cat docker/VERSION) +DOCKER_BUILDER_IMAGE_NAME = govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} +DOCKER_TTY ?= $(if ${JENKINS_HOME},,t) BUILD_TAG ?= notifications-admin-manual BUILD_NUMBER ?= 0 @@ -20,6 +22,10 @@ DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG} CODEDEPLOY_PREFIX ?= notifications-admin CODEDEPLOY_APP_NAME ?= notify-admin +CF_API ?= api.cloud.service.gov.uk +CF_ORG ?= govuk-notify +CF_SPACE ?= ${DEPLOY_ENV} + .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -28,8 +34,8 @@ help: venv: venv/bin/activate ## Create virtualenv if it does not exist venv/bin/activate: - test -d venv || virtualenv venv - ./venv/bin/pip install pip-accel + test -d venv || virtualenv venv -p python3 + . venv/bin/activate && pip install pip-accel .PHONY: check-env-vars check-env-vars: ## Check mandatory environment variables @@ -38,6 +44,12 @@ check-env-vars: ## Check mandatory environment variables $(if ${AWS_ACCESS_KEY_ID},,$(error Must specify AWS_ACCESS_KEY_ID)) $(if ${AWS_SECRET_ACCESS_KEY},,$(error Must specify AWS_SECRET_ACCESS_KEY)) +.PHONY: sandbox +sandbox: ## Set environment to sandbox + $(eval export DEPLOY_ENV=sandbox) + $(eval export DNS_NAME="cloudapps.digital") + @true + .PHONY: preview preview: ## Set environment to preview $(eval export DEPLOY_ENV=preview) @@ -62,7 +74,7 @@ dependencies: venv ## Install build dependencies npm install npm rebuild node-sass mkdir -p ${PIP_ACCEL_CACHE} - PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} ./venv/bin/pip-accel install -r requirements_for_test.txt + . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements_for_test.txt .PHONY: generate-version-file generate-version-file: ## Generates the app version file @@ -70,7 +82,11 @@ generate-version-file: ## Generates the app version file .PHONY: build build: dependencies generate-version-file ## Build project - ./venv/bin/pip-accel wheel --wheel-dir=wheelhouse -r requirements.txt + npm run build + . venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel wheel --wheel-dir=wheelhouse -r requirements.txt + +.PHONY: cf-build +cf-build: dependencies generate-version-file ## Build project npm run build .PHONY: build-codedeploy-artifact @@ -109,19 +125,20 @@ deploy-check-autoscaling-processes: check-aws-vars ## Returns with the number of .PHONY: coverage coverage: venv ## Create coverage report - ./venv/bin/coveralls + . venv/bin/activate && coveralls .PHONY: prepare-docker-build-image prepare-docker-build-image: ## Prepare the Docker builder image mkdir -p ${PIP_ACCEL_CACHE} - make -C docker build-build-image + make -C docker build -.PHONY: build-with-docker -build-with-docker: prepare-docker-build-image ## Build inside a Docker container - @docker run -i --rm \ - --name "${DOCKER_CONTAINER_PREFIX}-build" \ - -v `pwd`:/var/project \ - -v ${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel \ +define run_docker_container + @docker run -i${DOCKER_TTY} --rm \ + --name "${DOCKER_CONTAINER_PREFIX}-${1}" \ + -v "`pwd`:/var/project" \ + -v "${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel" \ + -e UID=$(shell id -u) \ + -e GID=$(shell id -g) \ -e GIT_COMMIT=${GIT_COMMIT} \ -e BUILD_NUMBER=${BUILD_NUMBER} \ -e BUILD_URL=${BUILD_URL} \ @@ -130,31 +147,6 @@ build-with-docker: prepare-docker-build-image ## Build inside a Docker container -e https_proxy="${HTTPS_PROXY}" \ -e HTTPS_PROXY="${HTTPS_PROXY}" \ -e NO_PROXY="${NO_PROXY}" \ - ${DOCKER_BUILDER_IMAGE_NAME} \ - make build - -.PHONY: test-with-docker -test-with-docker: prepare-docker-build-image ## Run tests inside a Docker container - @docker run -i --rm \ - --name "${DOCKER_CONTAINER_PREFIX}-test" \ - -v `pwd`:/var/project \ - -e GIT_COMMIT=${GIT_COMMIT} \ - -e BUILD_NUMBER=${BUILD_NUMBER} \ - -e BUILD_URL=${BUILD_URL} \ - -e http_proxy="${HTTP_PROXY}" \ - -e HTTP_PROXY="${HTTP_PROXY}" \ - -e https_proxy="${HTTPS_PROXY}" \ - -e HTTPS_PROXY="${HTTPS_PROXY}" \ - -e NO_PROXY="${NO_PROXY}" \ - ${DOCKER_BUILDER_IMAGE_NAME} \ - make test - -# FIXME: CIRCLECI=1 is an ugly hack because the coveralls-python library sends the PR link only this way -.PHONY: coverage-with-docker -coverage-with-docker: prepare-docker-build-image ## Generates coverage report inside a Docker container - @docker run -i --rm \ - --name "${DOCKER_CONTAINER_PREFIX}-coverage" \ - -v `pwd`:/var/project \ -e COVERALLS_REPO_TOKEN=${COVERALLS_REPO_TOKEN} \ -e CIRCLECI=1 \ -e CI_NAME=${CI_NAME} \ @@ -162,17 +154,60 @@ coverage-with-docker: prepare-docker-build-image ## Generates coverage report in -e CI_BUILD_URL=${BUILD_URL} \ -e CI_BRANCH=${GIT_BRANCH} \ -e CI_PULL_REQUEST=${CI_PULL_REQUEST} \ - -e http_proxy="${HTTP_PROXY}" \ - -e HTTP_PROXY="${HTTP_PROXY}" \ - -e https_proxy="${HTTPS_PROXY}" \ - -e HTTPS_PROXY="${HTTPS_PROXY}" \ - -e NO_PROXY="${NO_PROXY}" \ + -e CF_API="${CF_API}" \ + -e CF_USERNAME="${CF_USERNAME}" \ + -e CF_PASSWORD="${CF_PASSWORD}" \ + -e CF_ORG="${CF_ORG}" \ + -e CF_SPACE="${CF_SPACE}" \ ${DOCKER_BUILDER_IMAGE_NAME} \ - make coverage + ${2} +endef + +.PHONY: build-with-docker +build-with-docker: prepare-docker-build-image ## Build inside a Docker container + $(call run_docker_container,build,gosu hostuser make build) + +.PHONY: cf-build-with-docker +cf-build-with-docker: prepare-docker-build-image ## Build inside a Docker container + $(call run_docker_container,build,gosu hostuser make cf-build) + +.PHONY: test-with-docker +test-with-docker: prepare-docker-build-image ## Run tests inside a Docker container + $(call run_docker_container,test,gosu hostuser make test) + +# FIXME: CIRCLECI=1 is an ugly hack because the coveralls-python library sends the PR link only this way +.PHONY: coverage-with-docker +coverage-with-docker: prepare-docker-build-image ## Generates coverage report inside a Docker container + $(call run_docker_container,coverage,gosu hostuser make coverage) .PHONY: clean-docker-containers clean-docker-containers: ## Clean up any remaining docker containers docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true +.PHONY: clean clean: - rm -rf node_modules cache target venv .coverage + rm -rf node_modules cache target venv .coverage wheelhouse + +.PHONY: cf-login +cf-login: ## Log in to Cloud Foundry + $(if ${CF_USERNAME},,$(error Must specify CF_USERNAME)) + $(if ${CF_PASSWORD},,$(error Must specify CF_PASSWORD)) + $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) + @echo "Logging in to Cloud Foundry on ${CF_API}" + @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" + +.PHONY: cf-deploy +cf-deploy: ## Deploys the app to Cloud Foundry + $(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid notify-admin) | jq -r ".entity.instances")) + @echo "Original instance count: ${ORIG_INSTANCES}" + cf check-manifest notify-admin -f manifest-${CF_SPACE}.yml + cf zero-downtime-push notify-admin -f manifest-${CF_SPACE}.yml + cf scale -i ${ORIG_INSTANCES} notify-admin + +.PHONY: cf-push +cf-push: + cf push notify-admin -f manifest-${CF_SPACE}.yml + +.PHONY: cf-deploy-with-docker +cf-deploy-with-docker: prepare-docker-build-image ## Deploys the app to Cloud Foundry from a new Docker container + $(call run_docker_container,cf-deploy,make cf-login cf-deploy) diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 000000000..c60deb95a --- /dev/null +++ b/app/.gitignore @@ -0,0 +1,3 @@ +/version.py +/templates/govuk_template.html +/static diff --git a/app/__init__.py b/app/__init__.py index 87fd5665c..8b24a6623 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,6 +1,7 @@ import os import re import urllib +import json from datetime import datetime, timedelta, timezone from time import monotonic @@ -72,11 +73,13 @@ current_service = LocalProxy(partial(_lookup_req_object, 'service')) def create_app(): - from config import configs + from app.config import configs application = Flask(__name__) - application.config.from_object(configs[os.environ['NOTIFY_ENVIRONMENT']]) + notify_environment = os.environ['NOTIFY_ENVIRONMENT'] + + application.config.from_object(configs[notify_environment]) init_app(application) statsd_client.init_app(application) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py new file mode 100644 index 000000000..efd554019 --- /dev/null +++ b/app/cloudfoundry_config.py @@ -0,0 +1,50 @@ +""" +Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated +on local/aws boxes +""" + +import os +import json + + +def extract_cloudfoundry_config(): + vcap_services = json.loads(os.environ['VCAP_SERVICES']) + + set_config_env_vars(vcap_services) + + +def set_config_env_vars(vcap_services): + vcap_application = json.loads(os.environ.get('VCAP_APPLICATION')) + os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name'] + os.environ['LOGGING_STDOUT_JSON'] = '1' + + for s in vcap_services['user-provided']: + if s['name'] == 'notify-config': + extract_notify_config(s) + elif s['name'] == 'notify-aws': + extract_notify_aws_config(s) + elif s['name'] == 'hosted-graphite': + extract_hosted_graphite_config(s) + elif s['name'] == 'deskpro': + extract_deskpro_config(s) + + +def extract_notify_config(notify_config): + os.environ['ADMIN_CLIENT_SECRET'] = notify_config['credentials']['admin_client_secret'] + os.environ['API_HOST_NAME'] = notify_config['credentials']['api_host_name'] + os.environ['SECRET_KEY'] = notify_config['credentials']['secret_key'] + os.environ['DANGEROUS_SALT'] = notify_config['credentials']['dangerous_salt'] + + +def extract_notify_aws_config(aws_config): + os.environ['AWS_ACCESS_KEY_ID'] = aws_config['credentials']['aws_access_key_id'] + os.environ['AWS_SECRET_ACCESS_KEY'] = aws_config['credentials']['aws_secret_access_key'] + + +def extract_hosted_graphite_config(hosted_graphite_config): + os.environ['STATSD_PREFIX'] = hosted_graphite_config['credentials']['statsd_prefix'] + + +def extract_deskpro_config(deskpro_config): + os.environ['DESKPRO_API_HOST'] = deskpro_config['credentials']['api_host'] + os.environ['DESKPRO_API_KEY'] = deskpro_config['credentials']['api_key'] diff --git a/config.py b/app/config.py similarity index 83% rename from config.py rename to app/config.py index 8a40c51a5..e6035e5da 100644 --- a/config.py +++ b/app/config.py @@ -2,6 +2,13 @@ import os from datetime import timedelta +if os.environ.get('VCAP_SERVICES'): + # on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate + # standard environment variables from it + from app.cloudfoundry_config import extract_cloudfoundry_config + extract_cloudfoundry_config() + + class Config(object): ADMIN_CLIENT_SECRET = os.environ['ADMIN_CLIENT_SECRET'] API_HOST_NAME = os.environ['API_HOST_NAME'] @@ -9,13 +16,17 @@ class Config(object): DANGEROUS_SALT = os.environ['DANGEROUS_SALT'] DESKPRO_API_HOST = os.environ['DESKPRO_API_HOST'] DESKPRO_API_KEY = os.environ['DESKPRO_API_KEY'] + # Hosted graphite statsd prefix STATSD_PREFIX = os.getenv('STATSD_PREFIX') + # Logging + DEBUG = False + LOGGING_STDOUT_JSON = os.getenv('LOGGING_STDOUT_JSON') == '1' + DESKPRO_DEPT_ID = 5 DESKPRO_ASSIGNED_AGENT_TEAM_ID = 5 - DEBUG = False ADMIN_CLIENT_USER_NAME = 'notify-admin' ASSETS_DEBUG = False AWS_REGION = 'eu-west-1' @@ -112,10 +123,24 @@ class Live(Config): NOTIFY_ENVIRONMENT = 'live' +class CloudFoundryConfig(Config): + pass + + +# CloudFoundry sandbox +class Sandbox(CloudFoundryConfig): + HTTP_PROTOCOL = 'https' + HEADER_COLOUR = '#F499BE' # $baby-pink + STATSD_ENABLED = True + CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload' + NOTIFY_ENVIRONMENT = 'sandbox' + + configs = { 'development': Development, 'test': Test, 'preview': Preview, 'staging': Staging, - 'live': Live + 'live': Live, + 'sandbox': Sandbox } diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..ff7727315 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,82 @@ +FROM python:3.4-slim + +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +ENV PYTHONUNBUFFERED=1 \ + DEBIAN_FRONTEND=noninteractive \ + NODEJS_VERSION=6.3.1-1nodesource1~jessie1 \ + GOSU_VERSION=1.10 + +RUN \ + echo "Install base packages" \ + && ([ -z "$HTTP_PROXY" ] || echo "Acquire::http::Proxy \"${HTTP_PROXY}\";" > /etc/apt/apt.conf.d/99HttpProxy) \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + make \ + curl \ + git \ + build-essential \ + libxml2-dev \ + libxslt-dev \ + zlib1g-dev \ + zip \ + rlwrap \ + python-dev \ + libffi-dev \ + libssl-dev \ + libexif-dev \ + libfreetype6-dev \ + libjpeg-dev \ + liblcms2-2 \ + libtiff5-dev \ + zlib1g-dev \ + libpango1.0-dev \ + libcairo2-dev \ + libmagickwand-dev \ + ghostscript \ + jq \ + + && echo "Install nodejs" \ + && cd /tmp \ + && curl -x "$HTTP_PROXY" -sSLO https://deb.nodesource.com/node_6.x/pool/main/n/nodejs/nodejs_${NODEJS_VERSION}_amd64.deb \ + && dpkg -i /tmp/nodejs_${NODEJS_VERSION}_amd64.deb \ + + && echo "Clean up" \ + && rm -rf /var/lib/apt/lists/* /tmp/* + +RUN \ + echo "Install global pip packages" \ + && pip install \ + virtualenv \ + awscli \ + wheel + +RUN \ + echo "Install Cloud Foundry CLI" \ + && curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \ + && dpkg -i /tmp/cloudfoundry-cli.deb \ + && cf install-plugin -r CF-Community -f "autopilot" \ + && cf install-plugin -r CF-Community -f "blue-green-deploy" \ + && cf install-plugin -r CF-Community -f "antifreeze" + +COPY tianon.gpg /tmp/tianon.gpg + +RUN \ + echo "Install gosu" \ + && curl -sSL -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ + && curl -sSL -o /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --import /tmp/tianon.gpg \ + && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \ + && rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \ + && chmod +x /usr/local/bin/gosu \ + && gosu nobody true + +WORKDIR /var/project + +COPY entrypoint.sh /usr/local/bin/docker-entrypoint + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] diff --git a/docker/Dockerfile-build b/docker/Dockerfile-build deleted file mode 100644 index 114b230c9..000000000 --- a/docker/Dockerfile-build +++ /dev/null @@ -1,55 +0,0 @@ -FROM python:3.4-slim - -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV PYTHONUNBUFFERED=1 \ - DEBIAN_FRONTEND=noninteractive \ - NODEJS_VERSION=6.3.1-1nodesource1~jessie1 - -RUN \ - echo "Install base packages" \ - && ([ -z "$HTTP_PROXY" ] || echo "Acquire::http::Proxy \"${HTTP_PROXY}\";" > /etc/apt/apt.conf.d/99HttpProxy) \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - apt-transport-https \ - make \ - curl \ - git \ - build-essential \ - libxml2-dev \ - libxslt-dev \ - zlib1g-dev \ - zip \ - rlwrap \ - python-dev \ - libffi-dev \ - libssl-dev \ - libexif-dev \ - libfreetype6-dev \ - libjpeg-dev \ - liblcms2-2 \ - libtiff5-dev \ - zlib1g-dev \ - libpango1.0-dev \ - libcairo2-dev \ - libmagickwand-dev \ - ghostscript \ - - && echo "Install nodejs" \ - && cd /tmp \ - && curl -x "$HTTP_PROXY" -sSLO https://deb.nodesource.com/node_6.x/pool/main/n/nodejs/nodejs_${NODEJS_VERSION}_amd64.deb \ - && dpkg -i /tmp/nodejs_${NODEJS_VERSION}_amd64.deb \ - - && echo "Clean up" \ - && rm -rf /var/lib/apt/lists/* /tmp/* - -RUN \ - echo "Install global pip packages" \ - && pip install \ - virtualenv \ - awscli \ - wheel - -WORKDIR /var/project diff --git a/docker/Makefile b/docker/Makefile index 49a3111ac..7f7d726ef 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,17 +1,33 @@ .DEFAULT_GOAL := help SHELL := /bin/bash +DOCKER_IMAGE_TAG := $(shell cat VERSION) .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -.PHONY: build-build-image -build-build-image: +.PHONY: build +build: docker build \ --pull \ --build-arg HTTP_PROXY="${HTTP_PROXY}" \ --build-arg HTTPS_PROXY="${HTTP_PROXY}" \ --build-arg NO_PROXY="${NO_PROXY}" \ - -f Dockerfile-build \ - -t govuk/notify-admin-builder \ + -t govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} \ . + +.PHONY: bash +bash: + docker run -it --rm \ + -e UID=$(shell id -u) \ + -e GID=$(shell id -g) \ + govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} \ + bash + +.PHONY: bash +bash-hostuser: + docker run -it --rm \ + -e UID=$(shell id -u) \ + -e GID=$(shell id -g) \ + govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} \ + gosu hostuser bash diff --git a/docker/VERSION b/docker/VERSION new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/docker/VERSION @@ -0,0 +1 @@ +2 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 000000000..a57515244 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -eo pipefail; [[ "$TRACE" ]] && set -x + +if [[ "$(id -u)" -ne 0 ]]; then + echo 'docker-entrypoint requires root' >&2 + exit 1 +fi + +if [ -z "$UID" ] || [ "$UID" = "0" ]; then + echo "UID must be specified as a positive integer" + exit 1 +fi + +if [ -z "$GID" ] || [ "$GID" = "0" ]; then + echo "GID must be specified as positive integer" + exit 1 +fi + +USER=$(id -un $UID 2>/dev/null || echo "hostuser") +GROUP=$(getent group $GID | cut -d: -f1 || echo "hostgroup") + +if [ "$USER" = "hostuser" ]; then + useradd -u $UID -s /bin/bash -m $USER +fi + +if [ "$GROUP" = "hostgroup" ]; then + groupadd -g $GID $GROUP +fi + +usermod -g $GROUP $USER + +exec "$@" diff --git a/docker/tianon.gpg b/docker/tianon.gpg new file mode 100644 index 000000000..415d54907 Binary files /dev/null and b/docker/tianon.gpg differ diff --git a/manifest-preview.yml b/manifest-preview.yml new file mode 100644 index 000000000..c17b8b98f --- /dev/null +++ b/manifest-preview.yml @@ -0,0 +1,16 @@ +--- + +applications: + - name: notify-admin + buildpack: python_buildpack + command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi + services: + - notify-aws + - notify-config + - hosted-graphite + - deskpro + routes: + - route: notify-admin-preview.cloudapps.digital + - route: admin-paas.notify.works + instances: 1 + memory: 512M diff --git a/manifest-production.yml b/manifest-production.yml new file mode 100644 index 000000000..96bdf3ebc --- /dev/null +++ b/manifest-production.yml @@ -0,0 +1,16 @@ +--- + +applications: + - name: notify-admin + buildpack: python_buildpack + command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi + services: + - notify-aws + - notify-config + - hosted-graphite + - deskpro + routes: + - route: notify-admin-production.cloudapps.digital + - route: admin-paas.notifications.service.gov.uk + instances: 2 + memory: 2048M diff --git a/manifest-sandbox.yml b/manifest-sandbox.yml new file mode 100644 index 000000000..4dbb453b2 --- /dev/null +++ b/manifest-sandbox.yml @@ -0,0 +1,15 @@ +--- + +applications: + - name: notify-admin + buildpack: python_buildpack + command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi + services: + - notify-aws + - notify-config + - hosted-graphite + - deskpro + routes: + - route: notify-admin-sandbox.cloudapps.digital + instances: 1 + memory: 512M diff --git a/manifest-staging.yml b/manifest-staging.yml new file mode 100644 index 000000000..aa38c4c0b --- /dev/null +++ b/manifest-staging.yml @@ -0,0 +1,16 @@ +--- + +applications: + - name: notify-admin + buildpack: python_buildpack + command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi + services: + - notify-aws + - notify-config + - hosted-graphite + - deskpro + routes: + - route: notify-admin-staging.cloudapps.digital + - route: admin-paas.staging-notify.works + instances: 2 + memory: 2048M diff --git a/requirements.txt b/requirements.txt index e55de9ad3..9093e56d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +cffi==1.9.1 ago==0.0.8 Flask==0.10.1 Flask-Script==2.0.5 @@ -22,7 +23,10 @@ pyexcel-xlsx==0.1.0 pyexcel-ods3==0.1.1 pytz==2016.4 wand==0.4.4 +gunicorn==19.6.0 +whitenoise==1.0.6 #manages static assets -git+https://github.com/alphagov/notifications-python-client.git@3.0.1#egg=notifications-python-client==3.0.1 +# pin to minor version 3.1.x +notifications-python-client>=3.1,<3.2 -git+https://github.com/alphagov/notifications-utils.git@13.0.1#egg=notifications-utils==13.0.1 +git+https://github.com/alphagov/notifications-utils.git@13.1.0#egg=notifications-utils==13.1.0 diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 000000000..c0354eefe --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.5.2 diff --git a/tests/app/test_cloudfoundry_config.py b/tests/app/test_cloudfoundry_config.py new file mode 100644 index 000000000..ba17fdede --- /dev/null +++ b/tests/app/test_cloudfoundry_config.py @@ -0,0 +1,128 @@ +import os +import json + +import pytest + +from app.cloudfoundry_config import extract_cloudfoundry_config, set_config_env_vars + + +@pytest.fixture +def notify_config(): + return { + 'name': 'notify-config', + 'credentials': { + 'api_host_name': 'api host name', + 'admin_client_secret': 'admin client secret', + 'secret_key': 'secret key', + 'dangerous_salt': 'dangerous salt', + } + } + + +@pytest.fixture +def aws_config(): + return { + 'name': 'notify-aws', + 'credentials': { + 'aws_access_key_id': 'aws access key id', + 'aws_secret_access_key': 'aws secret access key', + } + } + + +@pytest.fixture +def hosted_graphite_config(): + return { + 'name': 'hosted-graphite', + 'credentials': { + 'statsd_prefix': 'statsd prefix' + } + } + + +@pytest.fixture +def deskpro_config(): + return { + 'name': 'deskpro', + 'credentials': { + 'api_host': 'deskpro api host', + 'api_key': 'deskpro api key' + } + } + + +@pytest.fixture +def cloudfoundry_config( + notify_config, + aws_config, + hosted_graphite_config, + deskpro_config, +): + return { + 'user-provided': [ + notify_config, + aws_config, + hosted_graphite_config, + deskpro_config, + ] + } + + +@pytest.fixture +def cloudfoundry_environ(monkeypatch, cloudfoundry_config): + monkeypatch.setenv('VCAP_SERVICES', json.dumps(cloudfoundry_config)) + monkeypatch.setenv('VCAP_APPLICATION', '{"space_name":"🚀🌌"}') + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_extract_cloudfoundry_config_populates_other_vars(): + extract_cloudfoundry_config() + + assert os.environ['LOGGING_STDOUT_JSON'] == '1' + assert os.environ['NOTIFY_ENVIRONMENT'] == '🚀🌌' + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config): + cloudfoundry_config['foo'] = {'credentials': {'foo': 'foo'}} + cloudfoundry_config['user-provided'].append({ + 'name': 'bar', 'credentials': {'bar': 'bar'} + }) + + set_config_env_vars(cloudfoundry_config) + + assert 'foo' not in os.environ + assert 'bar' not in os.environ + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_notify_config(): + extract_cloudfoundry_config() + + assert os.environ['API_HOST_NAME'] == 'api host name' + assert os.environ['ADMIN_CLIENT_SECRET'] == 'admin client secret' + assert os.environ['SECRET_KEY'] == 'secret key' + assert os.environ['DANGEROUS_SALT'] == 'dangerous salt' + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_aws_config(): + extract_cloudfoundry_config() + + assert os.environ['AWS_ACCESS_KEY_ID'] == 'aws access key id' + assert os.environ['AWS_SECRET_ACCESS_KEY'] == 'aws secret access key' + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_hosted_graphite_config(): + extract_cloudfoundry_config() + + assert os.environ['STATSD_PREFIX'] == 'statsd prefix' + + +@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ') +def test_deskpro_config(): + extract_cloudfoundry_config() + + assert os.environ['DESKPRO_API_HOST'] == 'deskpro api host' + assert os.environ['DESKPRO_API_KEY'] == 'deskpro api key' diff --git a/tests/app/test_config.py b/tests/app/test_config.py new file mode 100644 index 000000000..eb4dd1e5f --- /dev/null +++ b/tests/app/test_config.py @@ -0,0 +1,74 @@ +import os +import importlib +from unittest import mock + +import pytest + +from app import config + + +def cf_conf(): + os.environ['API_HOST_NAME'] = 'cf' + + +@pytest.fixture +def reload_config(): + """ + Reset config, by simply re-running config.py from a fresh environment + """ + old_env = os.environ.copy() + + yield + + os.environ = old_env + importlib.reload(config) + + +def test_load_cloudfoundry_config_if_available(monkeypatch, reload_config): + os.environ['API_HOST_NAME'] = 'env' + monkeypatch.setenv('VCAP_SERVICES', 'some json blob') + + with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config: + # reload config so that its module level code (ie: all of it) is re-instantiated + importlib.reload(config) + + assert cf_config.called + + assert os.environ['API_HOST_NAME'] == 'cf' + assert config.Config.API_HOST_NAME == 'cf' + + +def test_load_config_if_cloudfoundry_not_available(monkeypatch, reload_config): + os.environ['API_HOST_NAME'] = 'env' + + monkeypatch.delenv('VCAP_SERVICES', raising=False) + + with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config: + # reload config so that its module level code (ie: all of it) is re-instantiated + importlib.reload(config) + + assert not cf_config.called + + assert os.environ['API_HOST_NAME'] == 'env' + assert config.Config.API_HOST_NAME == 'env' + + +def test_logging_stdout_json_defaults_to_off(reload_config): + os.environ.pop('LOGGING_STDOUT_JSON', None) + assert config.Config.LOGGING_STDOUT_JSON is False + + +def test_logging_stdout_json_sets_to_off_if_not_recognised(reload_config): + os.environ['LOGGING_STDOUT_JSON'] = 'foo' + + importlib.reload(config) + + assert config.Config.LOGGING_STDOUT_JSON is False + + +def test_logging_stdout_json_sets_to_on_if_set_to_1(reload_config): + os.environ['LOGGING_STDOUT_JSON'] = '1' + + importlib.reload(config) + + assert config.Config.LOGGING_STDOUT_JSON is True diff --git a/tests/conftest.py b/tests/conftest.py index 9c2ff3df7..40da34ac5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,16 @@ +import os from datetime import date, datetime, timedelta from unittest.mock import Mock + import pytest +from notifications_python_client.errors import HTTPError from app import create_app +from app.notify_client.models import ( + User, + InvitedUser +) + from . import ( service_json, @@ -14,14 +22,10 @@ from . import ( notification_json, invite_json, sample_uuid, - generate_uuid, single_notification_json) -from app.notify_client.models import ( - User, - InvitedUser + generate_uuid, + single_notification_json ) -from notifications_python_client.errors import HTTPError - @pytest.fixture(scope='session') def app_(request): @@ -1340,3 +1344,15 @@ def logged_in_client( ): client.login(active_user_with_permissions) yield client + + +@pytest.fixture +def os_environ(): + """ + clear os.environ, and restore it after the test runs + """ + # for use whenever you expect code to edit environment variables + old_env = os.environ.copy() + os.environ = {} + yield + os.environ = old_env diff --git a/wsgi.py b/wsgi.py index db71cf075..96c25319e 100644 --- a/wsgi.py +++ b/wsgi.py @@ -1,12 +1,18 @@ from credstash import getAllSecrets +from whitenoise import WhiteNoise import os -# on aws get secrets and export to env -os.environ.update(getAllSecrets(region="eu-west-1")) +# On AWS get secrets and export to env, skip this on Cloud Foundry +if os.getenv('VCAP_SERVICES') is None: + os.environ.update(getAllSecrets(region="eu-west-1")) from app import create_app # noqa -application = create_app() +PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) +STATIC_ROOT = os.path.join(PROJECT_ROOT, 'app', 'static') +STATIC_URL = 'static/' + +application = WhiteNoise(create_app(), STATIC_ROOT, STATIC_URL) if __name__ == "__main__": application.run()