Merge pull request #605 from alphagov/jenkins_deploy

Create Docker build image, build project with Docker

.gitignore

@@ -24,6 +24,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+/cache
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -62,5 +63,8 @@ target/
 # Mac
 *.DS_Store
 environment.sh
+.envrc
 
-celerybeat-schedule
+celerybeat-schedule
+
+app/version.py

.travis.yml

@@ -14,6 +14,7 @@ install:
 before_script:
 - psql -c 'create database test_notification_api;' -U postgres
 script:
+- make generate-version-file
 - ./scripts/run_tests.sh
 env:
   secure: tgeumbN2oayp1twu3iVuuo5eMz/XVO303A2wdnR6bFMCfLCA7SVeKvDZ21ZpSh+J7tu8/9MQD2ATo95qyO9oraSg09BQ7UoEtpyrrcP21UBcNMbIsAdmOUAostlvg4hy1ZuSjytpzLDMZfS0QCjWPtZiXKW3XzmHHJyIdatcHsO3Jpi1vPRP11cZHd1SKwd1POYXDuX3Y9e68yt0P7Wr1/3mZ8u0XHtSg++SnZ0qDDwnWIsHqkcxr7R/n1MYvyUD8XPB+guqEq/7G6ipR+QrHN0fCVGXFksXGLSMSBg9sGQ1Mr+2yiOXL+4EmCfpx9VofmEOFDTdK70lFFn1sOG/GmceW4JT2Y2vLG+6vSJTmaHxeZmpYoKRa1EJJqyEpvjRM3A8lV993qIdAEBIE8s0w+DhkmXXCI3chSDT+2B/SlFbGw7G7E4hto/3FUrk7N7w+c5WaOQSqz4ZxTX4iIg9T7Bxo3s8l+UYYw4NfzEreRieEiFo58FgYrghEOvMp9PZ3tN3u2H+2yISE0C/+MSFUB2CWgFiTTD2XtWuQJgGNxyTYD1sbHaYT1EeDoz8JbhsACvIxpQdycVibHjP4hvP32nFFaznNpCm1ArS+vDtzR6Psx2vYb/u0rf1QoipVE/GPzqB9bwGHZ/0Cpsqy4KxYM74MOu3Gi3KCYzKGq7hRGI=
@@ -160,7 +161,6 @@ deploy:
   wait-until-deployed: true
   on: *2
 before_deploy:
-  - ./scripts/update_version_file.sh
   - zip -r  --exclude=*__pycache__* notifications-api *
   - mkdir -p dpl_cd_upload
   - mv notifications-api.zip dpl_cd_upload/notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip

Makefile

@@ -0,0 +1,144 @@
+.DEFAULT_GOAL := help
+SHELL := /bin/bash
+DATE = $(shell date +%Y-%m-%d:%H:%M:%S)
+
+PIP_ACCEL_CACHE ?= ${CURDIR}/cache/pip-accel
+APP_VERSION_FILE = app/version.py
+
+GIT_BRANCH = $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
+GIT_COMMIT ?= $(shell git rev-parse HEAD)
+
+DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder
+
+BUILD_TAG ?= notifications-api-manual
+BUILD_NUMBER ?= 0
+DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER}
+BUILD_URL ?=
+
+DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG}
+
+.PHONY: help
+help:
+	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: venv
+venv: venv/bin/activate ## Create virtualenv if it does not exist
+
+venv/bin/activate:
+	test -d venv || virtualenv venv
+	./venv/bin/pip install pip-accel
+
+.PHONY: check-env-vars
+check-env-vars: ## Check mandatory environment variables
+	$(if ${DEPLOY_ENV},,$(error Must specify DEPLOY_ENV))
+	$(if ${DNS_NAME},,$(error Must specify DNS_NAME))
+	$(if ${AWS_ACCESS_KEY_ID},,$(error Must specify AWS_ACCESS_KEY_ID))
+	$(if ${AWS_SECRET_ACCESS_KEY},,$(error Must specify AWS_SECRET_ACCESS_KEY))
+
+.PHONY: development
+development: ## Set environment to development
+	$(eval export DEPLOY_ENV=development)
+	$(eval export DNS_NAME="notify.works")
+	@true
+
+.PHONY: staging
+staging: ## Set environment to staging
+	$(eval export DEPLOY_ENV=staging)
+	$(eval export DNS_NAME="staging-notify.works")
+	@true
+
+.PHONY: production
+production: ## Set environment to production
+	$(eval export DEPLOY_ENV=production)
+	$(eval export DNS_NAME="notifications.service.gov.uk")
+	@true
+
+.PHONY: dependencies
+dependencies: venv ## Install build dependencies
+	mkdir -p ${PIP_ACCEL_CACHE}
+	PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} ./venv/bin/pip-accel install -r requirements_for_test.txt
+
+.PHONY: generate-version-file
+generate-version-file: ## Generates the app version file
+	@echo -e "__travis_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"\n__travis_job_number__ = \"${BUILD_NUMBER}\"\n__travis_job_url__ = \"${BUILD_URL}\"" > ${APP_VERSION_FILE}
+
+.PHONY: build
+build: dependencies generate-version-file ## Build project
+
+.PHONY: build-codedeploy-artifact
+build-codedeploy-artifact: ## Build the deploy artifact for CodeDeploy
+	mkdir -p target
+	zip -r -x@deploy-exclude.lst target/notifications-api.zip *
+
+.PHONY: upload-codedeploy-artifact ## Upload the deploy artifact for CodeDeploy
+upload-codedeploy-artifact: check-env-vars
+	aws s3 cp --region eu-west-1 target/notifications-api.zip s3://${DNS_NAME}-codedeploy/notifications-api-${DEPLOY_BUILD_NUMBER}.zip
+
+.PHONY: test
+test: venv ## Run tests
+	./scripts/run_tests.sh
+
+.PHONY: deploy-api
+deploy-api: check-env-vars ## Trigger CodeDeploy for the api
+	aws deploy create-deployment --application-name notify-api --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-api --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1
+
+.PHONY: deploy-admin-api
+deploy-admin-api: check-env-vars ## Trigger CodeDeploy for the admin api
+	aws deploy create-deployment --application-name notify-admin-api --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-admin-api --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1
+
+.PHONY: deploy-delivery
+deploy-delivery: check-env-vars ## Trigger CodeDeploy for the delivery app
+	aws deploy create-deployment --application-name notify-delivery --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-delivery --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1
+
+.PHONY: coverage
+coverage: venv ## Create coverage report
+	./venv/bin/coveralls
+
+.PHONY: prepare-docker-build-image
+prepare-docker-build-image: ## Prepare the Docker builder image
+	mkdir -p ${PIP_ACCEL_CACHE}
+	make -C docker build-build-image
+
+.PHONY: build-with-docker
+build-with-docker: prepare-docker-build-image ## Build inside a Docker container
+	docker run -i --rm \
+		--name "${DOCKER_CONTAINER_PREFIX}-build" \
+		-v `pwd`:/var/project \
+		-v ${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel \
+		${DOCKER_BUILDER_IMAGE_NAME} \
+		make build
+
+.PHONY: test-with-docker
+test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container
+	docker run -i --rm \
+		--name "${DOCKER_CONTAINER_PREFIX}-test" \
+		--link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \
+		-e TEST_DATABASE=postgresql://postgres:postgres@postgres/test_notification_api \
+		-v `pwd`:/var/project \
+		${DOCKER_BUILDER_IMAGE_NAME} \
+		make test
+
+.PHONY: test-with-docker
+create-docker-test-db: ## Start the test database in a Docker container
+	docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true
+	docker run -d \
+		--name "${DOCKER_CONTAINER_PREFIX}-db" \
+		-e POSTGRES_PASSWORD="postgres" \
+		-e POSTGRES_DB=test_notification_api \
+		postgres:9.5
+	sleep 3
+
+.PHONY: clean-docker-containers
+clean-docker-containers: ## Clean up any remaining docker containers
+	docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true
+
+.PHONY: coverage-with-docker
+coverage-with-docker: prepare-docker-build-image ## Generates coverage report inside a Docker container
+	docker run -i --rm \
+		--name "${DOCKER_CONTAINER_PREFIX}-coverage" \
+		-v `pwd`:/var/project \
+		${DOCKER_BUILDER_IMAGE_NAME} \
+		make coverage
+
+clean:
+	rm -rf node_modules cache target venv .coverage

app/version.py.dist

@@ -1,3 +1,4 @@
 __travis_commit__ = ""
-__time__ = "2016-07-05:15:38:37"
+__time__ = ""
 __travis_job_number__ = ""
+__travis_job_url__ = ""

deploy-exclude.lst

@@ -0,0 +1,9 @@
+*__pycache__*
+*.git*
+*app/assets*
+*bower_components*
+*cache*
+*docker*
+*node_modules*
+*target*
+*venv*

docker/Dockerfile-build

@@ -0,0 +1,25 @@
+FROM python:3.4-slim
+
+ENV PYTHONUNBUFFERED=1 \
+	DEBIAN_FRONTEND=noninteractive
+
+RUN \
+	echo "Install base packages" \
+	&& apt-get update \
+	&& apt-get install -y --no-install-recommends \
+		make \
+		git \
+		build-essential \
+		zip \
+		libpq-dev \
+
+	&& echo "Clean up" \
+	&& rm -rf /var/lib/apt/lists/* /tmp/*
+
+RUN \
+	echo "Install global pip packages" \
+	&& pip install \
+		virtualenv \
+		awscli
+
+WORKDIR /var/project

docker/Makefile

@@ -0,0 +1,10 @@
+.DEFAULT_GOAL := help
+SHELL := /bin/bash
+
+.PHONY: help
+help:
+	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: build-build-image
+build-build-image:
+	docker build -f Dockerfile-build -t govuk/notify-api-builder .

scripts/common_functions.sh

@@ -361,7 +361,6 @@ get_instance_name_from_tags() {
         error_exit "Couldn't get instance name for '$instance_id'"
     fi
     echo $instance_name
-    echo $instance_id
     return $?
 }
 
@@ -371,15 +370,12 @@ get_elb_name_for_instance_name() {
     local instance_name=$1
 
     declare -A elb_to_instance_mapping
-    
+
+    elb_to_instance_mapping['notify-api']='notify-api'
+    elb_to_instance_mapping['notify-admin-api']='notify-admin-api'
+
     elb_to_instance_mapping['notify_api']='notify-api-elb'
     elb_to_instance_mapping['notify_admin_api']='notify-admin-api-elb'
-    elb_to_instance_mapping['live_notify_api']='live-notify-api-elb'
-    elb_to_instance_mapping['staging_notify_api']='staging-notify-api-elb'
-    elb_to_instance_mapping['NotifyApi']='notify-api-elb'
-    elb_to_instance_mapping['live_notify_admin_api']='live-notify-admin-api-elb'
-    elb_to_instance_mapping['staging_notify_admin_api']='staging-notify-admin-api-elb'
-    elb_to_instance_mapping['NotifyAdminApi']='notify-admin-api-elb'
 
     local elb_name=${elb_to_instance_mapping[${instance_name}]}
     if [ -z $elb_name ]; then
@@ -387,4 +383,4 @@ get_elb_name_for_instance_name() {
     else
         ELB_NAME=$elb_name
     fi
-}
+}

scripts/register_with_elb.sh

@@ -28,7 +28,6 @@ msg "Started $(basename $0) at $(/bin/date "+%F %T")"
 start_sec=$(/bin/date +%s.%N)
 
 msg "Getting relevant load balancer"
-
 INSTANCE_NAME=$(get_instance_name_from_tags $INSTANCE_ID)
 
 if [[ "$(tr [:upper:] [:lower:] <<< "${INSTANCE_NAME}")" =~ 'delivery' ]]; then
@@ -37,6 +36,7 @@ if [[ "$(tr [:upper:] [:lower:] <<< "${INSTANCE_NAME}")" =~ 'delivery' ]]; then
 fi
 
 get_elb_name_for_instance_name $INSTANCE_NAME
+ELB_LIST=$ELB_NAME
 get_elb_list $INSTANCE_ID $ELB_NAME
 
 msg "Checking that user set at least one load balancer"

scripts/run_tests.sh

@@ -24,6 +24,9 @@ function display_result {
   fi
 }
 
+if [ -d venv ]; then
+  source ./venv/bin/activate
+fi
 pep8 .
 display_result $? 1 "Code style check"
 

scripts/update_version_file.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-#
-# Update the version file of the project from the Travis build details
-#
-sed -i -e "s/__travis_commit__ =.*/__travis_commit__ = \"$TRAVIS_COMMIT\"/g" ./app/version.py
-sed -i -e "s/__travis_job_number__ =.*/__travis_job_number__ = \"$TRAVIS_BUILD_NUMBER\"/g" ./app/version.py
-sed -i -e "s/__time__ =.*/__time__ = \"$(date +%Y-%m-%d:%H:%M:%S)\"/g" ./app/version.py