From 72dc93c6db0e8eb6dc6725970a32ea8e03dcf5ac Mon Sep 17 00:00:00 2001 From: bandesz Date: Mon, 13 Feb 2017 18:04:08 +0000 Subject: [PATCH] Add awslogs logging for PaaS --- Jenkinsfile | 16 ++--- Makefile | 64 +++++++++----------- docker/Dockerfile | 5 +- manifest-api-base.yml | 26 ++++++++ manifest-api-db-migration.yml | 19 ------ manifest-api-preview.yml | 23 ++----- manifest-api-production.yml | 25 +++----- manifest-api-sandbox.yml | 21 ++----- manifest-api-staging.yml | 25 +++----- manifest-delivery-base.yml | 40 ++++++++++++ manifest-delivery-celery-beat.yml | 19 ------ manifest-delivery-preview.yml | 3 + manifest-delivery-production.yml | 6 ++ manifest-delivery-sandbox.yml | 3 + manifest-delivery-staging.yml | 6 ++ manifest-delivery-worker-database.yml | 19 ------ manifest-delivery-worker-research.yml | 19 ------ manifest-delivery-worker-sender.yml | 19 ------ manifest-delivery-worker.yml | 19 ------ requirements.txt | 4 ++ scripts/run_app_paas.sh | 87 +++++++++++++++++++++++++++ 21 files changed, 238 insertions(+), 230 deletions(-) create mode 100644 manifest-api-base.yml delete mode 100644 manifest-api-db-migration.yml create mode 100644 manifest-delivery-base.yml delete mode 100644 manifest-delivery-celery-beat.yml create mode 100644 manifest-delivery-preview.yml create mode 100644 manifest-delivery-production.yml create mode 100644 manifest-delivery-sandbox.yml create mode 100644 manifest-delivery-staging.yml delete mode 100644 manifest-delivery-worker-database.yml delete mode 100644 manifest-delivery-worker-research.yml delete mode 100644 manifest-delivery-worker-sender.yml delete mode 100644 manifest-delivery-worker.yml create mode 100755 scripts/run_app_paas.sh diff --git a/Jenkinsfile b/Jenkinsfile index 4db188573..c269deea6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,42 +37,44 @@ def deploy(cfEnv) { ]) { withEnv(["CF_SPACE=${cfEnv}"]) { parallel deployApi: { - retry(3) { - sh 'make cf-deploy-api-with-docker' + withEnv(["CF_APP=notify-api"]) { + retry(3) { + sh 'make cf-deploy' + } } }, deployDeliveryCeleryBeat: { sleep(10) withEnv(["CF_APP=notify-delivery-celery-beat"]) { retry(3) { - sh 'make cf-deploy-delivery-with-docker' + sh 'make cf-deploy' } } }, deployDeliveryWorker: { sleep(20) withEnv(["CF_APP=notify-delivery-worker"]) { retry(3) { - sh 'make cf-deploy-delivery-with-docker' + sh 'make cf-deploy' } } }, deployDeliveryWorkerSender: { sleep(30) withEnv(["CF_APP=notify-delivery-worker-sender"]) { retry(3) { - sh 'make cf-deploy-delivery-with-docker' + sh 'make cf-deploy' } } }, deployDeliveryWorkerDatabase: { sleep(40) withEnv(["CF_APP=notify-delivery-worker-database"]) { retry(3) { - sh 'make cf-deploy-delivery-with-docker' + sh 'make cf-deploy' } } }, deployDeliveryWorkerResearch: { sleep(50) withEnv(["CF_APP=notify-delivery-worker-research"]) { retry(3) { - sh 'make cf-deploy-delivery-with-docker' + sh 'make cf-deploy' } } } diff --git a/Makefile b/Makefile index fedc71fb4..e109c3d82 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,8 @@ CF_API ?= api.cloud.service.gov.uk CF_ORG ?= govuk-notify CF_SPACE ?= ${DEPLOY_ENV} +CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml + .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -254,38 +256,33 @@ cf-login: ## Log in to Cloud Foundry @echo "Logging in to Cloud Foundry on ${CF_API}" @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" -.PHONY: cf-deploy-api -cf-deploy-api: ## Deploys the API to Cloud Foundry - $(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid notify-api) | jq -r ".entity.instances")) - @echo "Original instance count: ${ORIG_INSTANCES}" - cf check-manifest notify-api -f manifest-api-${CF_SPACE}.yml - cf zero-downtime-push notify-api -f manifest-api-${CF_SPACE}.yml - cf scale -i ${ORIG_INSTANCES} notify-api - -.PHONY: cf-push-api -cf-push-api: ## - cf push notify-api -f manifest-api-${CF_SPACE}.yml +.PHONY: cf-deploy +cf-deploy: ## Deploys the app to Cloud Foundry + $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) + $(if ${CF_APP},,$(error Must specify CF_APP)) + @cf app --guid ${CF_APP} || exit 1 + cf rename ${CF_APP} ${CF_APP}-rollback + cf push ${CF_APP} -f ${CF_MANIFEST_FILE} + cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP} + cf stop ${CF_APP}-rollback + cf delete -f ${CF_APP}-rollback .PHONY: cf-deploy-api-db-migration -cf-deploy-api-db-migration: ## Deploys the API db migration to Cloud Foundry - cf check-manifest notify-api-db-migration -f manifest-api-db-migration.yml - cf push notify-api-db-migration -f manifest-api-db-migration.yml +cf-deploy-api-db-migration: + $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) + cf push notify-api-db-migration -f manifest-api-${CF_SPACE}.yml + cf run-task notify-api-db-migration "scripts/run_app_paas.sh python db.py db upgrade" --name api_db_migration -cf-push-api-db-migration: cf-deploy-api-db-migration ## Deploys the API db migration to Cloud Foundry - -.PHONY: cf-deploy-delivery -cf-deploy-delivery: ## Deploys a delivery app to Cloud Foundry +.PHONY: cf-rollback +cf-rollback: ## Rollbacks the app to the previous release $(if ${CF_APP},,$(error Must specify CF_APP)) - $(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid ${CF_APP}) | jq -r ".entity.instances")) - @echo "Original instance count: ${ORIG_INSTANCES}" - cf check-manifest ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml - cf zero-downtime-push ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml - cf scale -i ${ORIG_INSTANCES} ${CF_APP} + @cf app --guid ${CF_APP}-rollback || exit 1 + cf delete -f ${CF_APP} || true + cf rename ${CF_APP}-rollback ${CF_APP} -.PHONY: cf-push-delivery -cf-push-delivery: ## Deploys a delivery app to Cloud Foundry - $(if ${CF_APP},,$(error Must specify CF_APP)) - cf push ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml +.PHONY: cf-push +cf-push: + cf push ${CF_APP} -f ${CF_MANIFEST_FILE} define cf_deploy_with_docker @docker run -i${DOCKER_TTY} --rm \ @@ -308,15 +305,14 @@ define cf_deploy_with_docker ${2} endef -.PHONY: cf-deploy-api-with-docker -cf-deploy-api-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container - $(call cf_deploy_with_docker,cf-deploy-api,make cf-login cf-deploy-api) +.PHONY: cf-deploy-with-docker +cf-deploy-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container + $(call cf_deploy_with_docker,cf-deploy,make cf-login cf-deploy) .PHONY: cf-deploy-api-db-migration-with-docker cf-deploy-api-db-migration-with-docker: prepare-docker-build-image ## Deploys the API db migration to Cloud Foundry from a Docker container $(call cf_deploy_with_docker,cf-deploy-api-db-migration,make cf-login cf-deploy-api-db-migration) -.PHONY: cf-deploy-delivery-with-docker -cf-deploy-delivery-with-docker: prepare-docker-build-image ## Deploys a delivery app to Cloud Foundry from a Docker container - $(if ${CF_APP},,$(error Must specify CF_APP)) - $(call cf_deploy_with_docker,cf-deploy-delivery-${CF_APP},make cf-login cf-deploy-delivery) +.PHONY: cf-rollback-with-docker +cf-rollback-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container + $(call cf_deploy_with_docker,cf-rollback,make cf-login cf-rollback) diff --git a/docker/Dockerfile b/docker/Dockerfile index c7daf00bc..2ccfe0e1e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,10 +33,7 @@ RUN \ RUN \ echo "Install Cloud Foundry CLI" \ && curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \ - && dpkg -i /tmp/cloudfoundry-cli.deb \ - && cf install-plugin -r CF-Community -f "autopilot" \ - && cf install-plugin -r CF-Community -f "blue-green-deploy" \ - && cf install-plugin -r CF-Community -f "antifreeze" + && dpkg -i /tmp/cloudfoundry-cli.deb COPY tianon.gpg /tmp/tianon.gpg diff --git a/manifest-api-base.yml b/manifest-api-base.yml new file mode 100644 index 000000000..433a567f2 --- /dev/null +++ b/manifest-api-base.yml @@ -0,0 +1,26 @@ +--- + +buildpack: python_buildpack +command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT wsgi +services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite +env: + NOTIFY_APP_NAME: public-api + CW_APP_NAME: api +instances: 1 +memory: 512M + +applications: + - name: notify-api + + - name: notify-api-db-migration + command: sleep infinity + no-route: true + health-check-type: none + instances: 1 + memory: 128M diff --git a/manifest-api-db-migration.yml b/manifest-api-db-migration.yml deleted file mode 100644 index 5760e111d..000000000 --- a/manifest-api-db-migration.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -applications: - - name: notify-api-db-migration - buildpack: python_buildpack - command: python db.py db upgrade && sleep infinity - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - env: - NOTIFY_APP_NAME: public-api - no-route: true - health-check-type: none - instances: 1 - memory: 128M diff --git a/manifest-api-preview.yml b/manifest-api-preview.yml index e6645a9e2..b2a2571d0 100644 --- a/manifest-api-preview.yml +++ b/manifest-api-preview.yml @@ -1,20 +1,7 @@ --- -applications: - - name: notify-api - buildpack: python_buildpack - command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - env: - NOTIFY_APP_NAME: public-api - routes: - - route: notify-api-preview.cloudapps.digital - - route: api-paas.notify.works - instances: 1 - memory: 512M +inherit: manifest-api-base.yml + +routes: + - route: notify-api-preview.cloudapps.digital + - route: api-paas.notify.works diff --git a/manifest-api-production.yml b/manifest-api-production.yml index 9ea72569e..4b71bf0af 100644 --- a/manifest-api-production.yml +++ b/manifest-api-production.yml @@ -1,20 +1,9 @@ --- -applications: - - name: notify-api - buildpack: python_buildpack - command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - env: - NOTIFY_APP_NAME: public-api - routes: - - route: notify-api-production.cloudapps.digital - - route: api-paas.notifications.service.gov.uk - instances: 2 - memory: 2048M +inherit: manifest-api-base.yml + +routes: + - route: notify-api-production.cloudapps.digital + - route: api-paas.notifications.service.gov.uk +instances: 2 +memory: 2048M diff --git a/manifest-api-sandbox.yml b/manifest-api-sandbox.yml index fc6e117dc..feecde9ef 100644 --- a/manifest-api-sandbox.yml +++ b/manifest-api-sandbox.yml @@ -1,19 +1,6 @@ --- -applications: - - name: notify-api - buildpack: python_buildpack - command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - env: - NOTIFY_APP_NAME: public-api - routes: - - route: notify-api-sandbox.cloudapps.digital - instances: 1 - memory: 512M +inherit: manifest-api-base.yml + +routes: + - route: notify-api-sandbox.cloudapps.digital diff --git a/manifest-api-staging.yml b/manifest-api-staging.yml index e593f1e26..669fdfa36 100644 --- a/manifest-api-staging.yml +++ b/manifest-api-staging.yml @@ -1,20 +1,9 @@ --- -applications: - - name: notify-api - buildpack: python_buildpack - command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - env: - NOTIFY_APP_NAME: public-api - routes: - - route: notify-api-staging.cloudapps.digital - - route: api-paas.staging-notify.works - instances: 2 - memory: 2048M +inherit: manifest-api-base.yml + +routes: + - route: notify-api-staging.cloudapps.digital + - route: api-paas.staging-notify.works +instances: 2 +memory: 2048M diff --git a/manifest-delivery-base.yml b/manifest-delivery-base.yml new file mode 100644 index 000000000..266e39b13 --- /dev/null +++ b/manifest-delivery-base.yml @@ -0,0 +1,40 @@ +--- + +buildpack: python_buildpack +health-check-type: none +no-route: true +services: + - notify-aws + - notify-config + - notify-db + - mmg + - firetext + - hosted-graphite +instances: 1 +memory: 256M + +applications: + - name: notify-delivery-celery-beat + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery beat --loglevel=INFO + env: + NOTIFY_APP_NAME: delivery-celery-beat + + - name: notify-delivery-worker-database + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email,db-letter + env: + NOTIFY_APP_NAME: delivery-worker-database + + - name: notify-delivery-worker-research + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode + env: + NOTIFY_APP_NAME: delivery-worker-research + + - name: notify-delivery-worker-sender + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email + env: + NOTIFY_APP_NAME: delivery-worker-sender + + - name: notify-delivery-worker + command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 + env: + NOTIFY_APP_NAME: delivery-worker diff --git a/manifest-delivery-celery-beat.yml b/manifest-delivery-celery-beat.yml deleted file mode 100644 index 0f65fd635..000000000 --- a/manifest-delivery-celery-beat.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -applications: - - name: notify-delivery-celery-beat - buildpack: python_buildpack - health-check-type: none - no-route: true - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - instances: 2 - memory: 128M - command: celery -A aws_run_celery.notify_celery beat --loglevel=INFO - env: - NOTIFY_APP_NAME: delivery-celery-beat diff --git a/manifest-delivery-preview.yml b/manifest-delivery-preview.yml new file mode 100644 index 000000000..d628e5fc9 --- /dev/null +++ b/manifest-delivery-preview.yml @@ -0,0 +1,3 @@ +--- + +inherit: manifest-delivery-base.yml diff --git a/manifest-delivery-production.yml b/manifest-delivery-production.yml new file mode 100644 index 000000000..588966907 --- /dev/null +++ b/manifest-delivery-production.yml @@ -0,0 +1,6 @@ +--- + +inherit: manifest-delivery-base.yml + +instances: 2 +memory: 1024M diff --git a/manifest-delivery-sandbox.yml b/manifest-delivery-sandbox.yml new file mode 100644 index 000000000..d628e5fc9 --- /dev/null +++ b/manifest-delivery-sandbox.yml @@ -0,0 +1,3 @@ +--- + +inherit: manifest-delivery-base.yml diff --git a/manifest-delivery-staging.yml b/manifest-delivery-staging.yml new file mode 100644 index 000000000..588966907 --- /dev/null +++ b/manifest-delivery-staging.yml @@ -0,0 +1,6 @@ +--- + +inherit: manifest-delivery-base.yml + +instances: 2 +memory: 1024M diff --git a/manifest-delivery-worker-database.yml b/manifest-delivery-worker-database.yml deleted file mode 100644 index ab7704c3b..000000000 --- a/manifest-delivery-worker-database.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -applications: - - name: notify-delivery-worker-database - buildpack: python_buildpack - health-check-type: none - no-route: true - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - instances: 2 - memory: 256M - command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email,db-letter - env: - NOTIFY_APP_NAME: delivery-worker-database diff --git a/manifest-delivery-worker-research.yml b/manifest-delivery-worker-research.yml deleted file mode 100644 index cf935aac9..000000000 --- a/manifest-delivery-worker-research.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -applications: - - name: notify-delivery-worker-research - buildpack: python_buildpack - health-check-type: none - no-route: true - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - instances: 2 - memory: 256M - command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode - env: - NOTIFY_APP_NAME: delivery-worker-research diff --git a/manifest-delivery-worker-sender.yml b/manifest-delivery-worker-sender.yml deleted file mode 100644 index 3774c86ff..000000000 --- a/manifest-delivery-worker-sender.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -applications: - - name: notify-delivery-worker-sender - buildpack: python_buildpack - health-check-type: none - no-route: true - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - instances: 2 - memory: 256M - command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email - env: - NOTIFY_APP_NAME: delivery-worker-sender diff --git a/manifest-delivery-worker.yml b/manifest-delivery-worker.yml deleted file mode 100644 index ee53bb82a..000000000 --- a/manifest-delivery-worker.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -applications: - - name: notify-delivery-worker - buildpack: python_buildpack - health-check-type: none - no-route: true - services: - - notify-aws - - notify-config - - notify-db - - mmg - - firetext - - hosted-graphite - instances: 2 - memory: 256M - command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 - env: - NOTIFY_APP_NAME: delivery-worker diff --git a/requirements.txt b/requirements.txt index 00416dcc8..a82c38e04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,10 @@ docopt==0.6.2 # pin to minor version 3.1.x notifications-python-client>=3.1,<3.2 +# PaaS +awscli>=1.11,<1.12 +awscli-cwlogs>=1.4,<1.5 + git+https://github.com/alphagov/notifications-utils.git@13.5.0#egg=notifications-utils==13.5.0 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3 diff --git a/scripts/run_app_paas.sh b/scripts/run_app_paas.sh new file mode 100755 index 000000000..e892ef010 --- /dev/null +++ b/scripts/run_app_paas.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -e -o pipefail + +function check_params { + if [ -z "${NOTIFY_APP_NAME}" ]; then + echo "You must set NOTIFY_APP_NAME" + exit 1 + fi + + if [ -z "${CW_APP_NAME}" ]; then + CW_APP_NAME=${NOTIFY_APP_NAME} + fi +} + +function configure_aws_logs { + aws configure set plugins.cwlogs cwlogs + + export AWS_ACCESS_KEY_ID=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_access_key_id') + export AWS_SECRET_ACCESS_KEY=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_secret_access_key') + + cat > /home/vcap/app/awslogs.conf << EOF +[general] +state_file = /home/vcap/logs/awslogs-state + +[/home/vcap/logs/app-stdout.log] +file = /home/vcap/logs/app-stdout.log* +log_group_name = paas-${CW_APP_NAME}-application +log_stream_name = {hostname}-stdout + +[/home/vcap/logs/app-stderr.log] +file = /home/vcap/logs/app-stderr.log* +log_group_name = paas-${CW_APP_NAME}-application +log_stream_name = {hostname}-stderr +EOF +} + +function on_exit { + echo "Terminating application process with pid ${APP_PID}" + kill ${APP_PID} || true + while (kill -0 ${APP_PID} 2&>/dev/null); do + echo "Application is still running.." + sleep 1 + done + echo "Application process exited, waiting 10 seconds" + sleep 10 + echo "Terminating remaining subprocesses.." + kill 0 +} + +function start_appplication { + exec "$@" 2>/home/vcap/logs/app-stderr.log | while read line; do echo $line; echo $line >> /home/vcap/logs/app-stdout.log.`date +%Y-%m-%d`; done & + LOGGER_PID=$! + APP_PID=`jobs -p` + echo "Logger process pid: ${LOGGER_PID}" + echo "Application process pid: ${APP_PID}" +} + +function start_aws_logs_agent { + exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf & + AWSLOGS_AGENT_PID=$! + echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" +} + +function run { + while true; do + kill -0 ${APP_PID} 2&>/dev/null || break + kill -0 ${LOGGER_PID} 2&>/dev/null || break + kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent + sleep 1 + done +} + +echo "Run script pid: $$" + +check_params + +trap "on_exit" EXIT + +configure_aws_logs + +# The application has to start first! +start_appplication "$@" + +start_aws_logs_agent + +run