diff --git a/.gitignore b/.gitignore index 1c6c16612..8be1355da 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ var/ *.egg-info/ .installed.cfg *.egg +/cache # PyInstaller # Usually these files are written by a python script from a template @@ -62,5 +63,8 @@ target/ # Mac *.DS_Store environment.sh +.envrc -celerybeat-schedule \ No newline at end of file +celerybeat-schedule + +app/version.py diff --git a/.travis.yml b/.travis.yml index 0d29a03cf..f7d9a4b66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ install: before_script: - psql -c 'create database test_notification_api;' -U postgres script: +- make generate-version-file - ./scripts/run_tests.sh env: secure: tgeumbN2oayp1twu3iVuuo5eMz/XVO303A2wdnR6bFMCfLCA7SVeKvDZ21ZpSh+J7tu8/9MQD2ATo95qyO9oraSg09BQ7UoEtpyrrcP21UBcNMbIsAdmOUAostlvg4hy1ZuSjytpzLDMZfS0QCjWPtZiXKW3XzmHHJyIdatcHsO3Jpi1vPRP11cZHd1SKwd1POYXDuX3Y9e68yt0P7Wr1/3mZ8u0XHtSg++SnZ0qDDwnWIsHqkcxr7R/n1MYvyUD8XPB+guqEq/7G6ipR+QrHN0fCVGXFksXGLSMSBg9sGQ1Mr+2yiOXL+4EmCfpx9VofmEOFDTdK70lFFn1sOG/GmceW4JT2Y2vLG+6vSJTmaHxeZmpYoKRa1EJJqyEpvjRM3A8lV993qIdAEBIE8s0w+DhkmXXCI3chSDT+2B/SlFbGw7G7E4hto/3FUrk7N7w+c5WaOQSqz4ZxTX4iIg9T7Bxo3s8l+UYYw4NfzEreRieEiFo58FgYrghEOvMp9PZ3tN3u2H+2yISE0C/+MSFUB2CWgFiTTD2XtWuQJgGNxyTYD1sbHaYT1EeDoz8JbhsACvIxpQdycVibHjP4hvP32nFFaznNpCm1ArS+vDtzR6Psx2vYb/u0rf1QoipVE/GPzqB9bwGHZ/0Cpsqy4KxYM74MOu3Gi3KCYzKGq7hRGI= @@ -160,7 +161,6 @@ deploy: wait-until-deployed: true on: *2 before_deploy: - - ./scripts/update_version_file.sh - zip -r --exclude=*__pycache__* notifications-api * - mkdir -p dpl_cd_upload - mv notifications-api.zip dpl_cd_upload/notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ff8aea5e4 --- /dev/null +++ b/Makefile @@ -0,0 +1,144 @@ +.DEFAULT_GOAL := help +SHELL := /bin/bash +DATE = $(shell date +%Y-%m-%d:%H:%M:%S) + +PIP_ACCEL_CACHE ?= ${CURDIR}/cache/pip-accel +APP_VERSION_FILE = app/version.py + +GIT_BRANCH = $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") +GIT_COMMIT ?= $(shell git rev-parse HEAD) + +DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder + +BUILD_TAG ?= notifications-api-manual +BUILD_NUMBER ?= 0 +DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER} +BUILD_URL ?= + +DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG} + +.PHONY: help +help: + @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: venv +venv: venv/bin/activate ## Create virtualenv if it does not exist + +venv/bin/activate: + test -d venv || virtualenv venv + ./venv/bin/pip install pip-accel + +.PHONY: check-env-vars +check-env-vars: ## Check mandatory environment variables + $(if ${DEPLOY_ENV},,$(error Must specify DEPLOY_ENV)) + $(if ${DNS_NAME},,$(error Must specify DNS_NAME)) + $(if ${AWS_ACCESS_KEY_ID},,$(error Must specify AWS_ACCESS_KEY_ID)) + $(if ${AWS_SECRET_ACCESS_KEY},,$(error Must specify AWS_SECRET_ACCESS_KEY)) + +.PHONY: development +development: ## Set environment to development + $(eval export DEPLOY_ENV=development) + $(eval export DNS_NAME="notify.works") + @true + +.PHONY: staging +staging: ## Set environment to staging + $(eval export DEPLOY_ENV=staging) + $(eval export DNS_NAME="staging-notify.works") + @true + +.PHONY: production +production: ## Set environment to production + $(eval export DEPLOY_ENV=production) + $(eval export DNS_NAME="notifications.service.gov.uk") + @true + +.PHONY: dependencies +dependencies: venv ## Install build dependencies + mkdir -p ${PIP_ACCEL_CACHE} + PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} ./venv/bin/pip-accel install -r requirements_for_test.txt + +.PHONY: generate-version-file +generate-version-file: ## Generates the app version file + @echo -e "__travis_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"\n__travis_job_number__ = \"${BUILD_NUMBER}\"\n__travis_job_url__ = \"${BUILD_URL}\"" > ${APP_VERSION_FILE} + +.PHONY: build +build: dependencies generate-version-file ## Build project + +.PHONY: build-codedeploy-artifact +build-codedeploy-artifact: ## Build the deploy artifact for CodeDeploy + mkdir -p target + zip -r -x@deploy-exclude.lst target/notifications-api.zip * + +.PHONY: upload-codedeploy-artifact ## Upload the deploy artifact for CodeDeploy +upload-codedeploy-artifact: check-env-vars + aws s3 cp --region eu-west-1 target/notifications-api.zip s3://${DNS_NAME}-codedeploy/notifications-api-${DEPLOY_BUILD_NUMBER}.zip + +.PHONY: test +test: venv ## Run tests + ./scripts/run_tests.sh + +.PHONY: deploy-api +deploy-api: check-env-vars ## Trigger CodeDeploy for the api + aws deploy create-deployment --application-name notify-api --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-api --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1 + +.PHONY: deploy-admin-api +deploy-admin-api: check-env-vars ## Trigger CodeDeploy for the admin api + aws deploy create-deployment --application-name notify-admin-api --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-admin-api --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1 + +.PHONY: deploy-delivery +deploy-delivery: check-env-vars ## Trigger CodeDeploy for the delivery app + aws deploy create-deployment --application-name notify-delivery --deployment-config-name CodeDeployDefault.OneAtATime --deployment-group-name notify-delivery --s3-location bucket=${DNS_NAME}-codedeploy,key=notifications-api-${DEPLOY_BUILD_NUMBER}.zip,bundleType=zip --region eu-west-1 + +.PHONY: coverage +coverage: venv ## Create coverage report + ./venv/bin/coveralls + +.PHONY: prepare-docker-build-image +prepare-docker-build-image: ## Prepare the Docker builder image + mkdir -p ${PIP_ACCEL_CACHE} + make -C docker build-build-image + +.PHONY: build-with-docker +build-with-docker: prepare-docker-build-image ## Build inside a Docker container + docker run -i --rm \ + --name "${DOCKER_CONTAINER_PREFIX}-build" \ + -v `pwd`:/var/project \ + -v ${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel \ + ${DOCKER_BUILDER_IMAGE_NAME} \ + make build + +.PHONY: test-with-docker +test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container + docker run -i --rm \ + --name "${DOCKER_CONTAINER_PREFIX}-test" \ + --link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \ + -e TEST_DATABASE=postgresql://postgres:postgres@postgres/test_notification_api \ + -v `pwd`:/var/project \ + ${DOCKER_BUILDER_IMAGE_NAME} \ + make test + +.PHONY: test-with-docker +create-docker-test-db: ## Start the test database in a Docker container + docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true + docker run -d \ + --name "${DOCKER_CONTAINER_PREFIX}-db" \ + -e POSTGRES_PASSWORD="postgres" \ + -e POSTGRES_DB=test_notification_api \ + postgres:9.5 + sleep 3 + +.PHONY: clean-docker-containers +clean-docker-containers: ## Clean up any remaining docker containers + docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true + +.PHONY: coverage-with-docker +coverage-with-docker: prepare-docker-build-image ## Generates coverage report inside a Docker container + docker run -i --rm \ + --name "${DOCKER_CONTAINER_PREFIX}-coverage" \ + -v `pwd`:/var/project \ + ${DOCKER_BUILDER_IMAGE_NAME} \ + make coverage + +clean: + rm -rf node_modules cache target venv .coverage diff --git a/app/version.py b/app/version.py.dist similarity index 56% rename from app/version.py rename to app/version.py.dist index d6d5d6781..295df522a 100644 --- a/app/version.py +++ b/app/version.py.dist @@ -1,3 +1,4 @@ __travis_commit__ = "" -__time__ = "2016-07-05:15:38:37" +__time__ = "" __travis_job_number__ = "" +__travis_job_url__ = "" diff --git a/deploy-exclude.lst b/deploy-exclude.lst new file mode 100644 index 000000000..5eaf61748 --- /dev/null +++ b/deploy-exclude.lst @@ -0,0 +1,9 @@ +*__pycache__* +*.git* +*app/assets* +*bower_components* +*cache* +*docker* +*node_modules* +*target* +*venv* diff --git a/docker/Dockerfile-build b/docker/Dockerfile-build new file mode 100644 index 000000000..4fa6afb2d --- /dev/null +++ b/docker/Dockerfile-build @@ -0,0 +1,25 @@ +FROM python:3.4-slim + +ENV PYTHONUNBUFFERED=1 \ + DEBIAN_FRONTEND=noninteractive + +RUN \ + echo "Install base packages" \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + make \ + git \ + build-essential \ + zip \ + libpq-dev \ + + && echo "Clean up" \ + && rm -rf /var/lib/apt/lists/* /tmp/* + +RUN \ + echo "Install global pip packages" \ + && pip install \ + virtualenv \ + awscli + +WORKDIR /var/project diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 000000000..2db0aade2 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,10 @@ +.DEFAULT_GOAL := help +SHELL := /bin/bash + +.PHONY: help +help: + @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: build-build-image +build-build-image: + docker build -f Dockerfile-build -t govuk/notify-api-builder . diff --git a/scripts/common_functions.sh b/scripts/common_functions.sh index a2e136b25..a489377ef 100644 --- a/scripts/common_functions.sh +++ b/scripts/common_functions.sh @@ -361,7 +361,6 @@ get_instance_name_from_tags() { error_exit "Couldn't get instance name for '$instance_id'" fi echo $instance_name - echo $instance_id return $? } @@ -371,15 +370,12 @@ get_elb_name_for_instance_name() { local instance_name=$1 declare -A elb_to_instance_mapping - + + elb_to_instance_mapping['notify-api']='notify-api' + elb_to_instance_mapping['notify-admin-api']='notify-admin-api' + elb_to_instance_mapping['notify_api']='notify-api-elb' elb_to_instance_mapping['notify_admin_api']='notify-admin-api-elb' - elb_to_instance_mapping['live_notify_api']='live-notify-api-elb' - elb_to_instance_mapping['staging_notify_api']='staging-notify-api-elb' - elb_to_instance_mapping['NotifyApi']='notify-api-elb' - elb_to_instance_mapping['live_notify_admin_api']='live-notify-admin-api-elb' - elb_to_instance_mapping['staging_notify_admin_api']='staging-notify-admin-api-elb' - elb_to_instance_mapping['NotifyAdminApi']='notify-admin-api-elb' local elb_name=${elb_to_instance_mapping[${instance_name}]} if [ -z $elb_name ]; then @@ -387,4 +383,4 @@ get_elb_name_for_instance_name() { else ELB_NAME=$elb_name fi -} \ No newline at end of file +} diff --git a/scripts/register_with_elb.sh b/scripts/register_with_elb.sh index 8e37f7757..7e6fb80ac 100755 --- a/scripts/register_with_elb.sh +++ b/scripts/register_with_elb.sh @@ -28,7 +28,6 @@ msg "Started $(basename $0) at $(/bin/date "+%F %T")" start_sec=$(/bin/date +%s.%N) msg "Getting relevant load balancer" - INSTANCE_NAME=$(get_instance_name_from_tags $INSTANCE_ID) if [[ "$(tr [:upper:] [:lower:] <<< "${INSTANCE_NAME}")" =~ 'delivery' ]]; then @@ -37,6 +36,7 @@ if [[ "$(tr [:upper:] [:lower:] <<< "${INSTANCE_NAME}")" =~ 'delivery' ]]; then fi get_elb_name_for_instance_name $INSTANCE_NAME +ELB_LIST=$ELB_NAME get_elb_list $INSTANCE_ID $ELB_NAME msg "Checking that user set at least one load balancer" diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index dd229b408..652094dd2 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -24,6 +24,9 @@ function display_result { fi } +if [ -d venv ]; then + source ./venv/bin/activate +fi pep8 . display_result $? 1 "Code style check" diff --git a/scripts/update_version_file.sh b/scripts/update_version_file.sh deleted file mode 100755 index 231c210b7..000000000 --- a/scripts/update_version_file.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# -# Update the version file of the project from the Travis build details -# -sed -i -e "s/__travis_commit__ =.*/__travis_commit__ = \"$TRAVIS_COMMIT\"/g" ./app/version.py -sed -i -e "s/__travis_job_number__ =.*/__travis_job_number__ = \"$TRAVIS_BUILD_NUMBER\"/g" ./app/version.py -sed -i -e "s/__time__ =.*/__time__ = \"$(date +%Y-%m-%d:%H:%M:%S)\"/g" ./app/version.py \ No newline at end of file