Add awslogs logging for PaaS

Jenkinsfile

@@ -37,42 +37,44 @@ def deploy(cfEnv) {
         ]) {
           withEnv(["CF_SPACE=${cfEnv}"]) {
             parallel deployApi: {
-              retry(3) {
+              withEnv(["CF_APP=notify-api"]) {
-                sh 'make cf-deploy-api-with-docker'
+                retry(3) {
+                  sh 'make cf-deploy'
+                }
               }
             }, deployDeliveryCeleryBeat: {
               sleep(10)
               withEnv(["CF_APP=notify-delivery-celery-beat"]) {
                 retry(3) {
-                  sh 'make cf-deploy-delivery-with-docker'
+                  sh 'make cf-deploy'
                 }
               }
             }, deployDeliveryWorker: {
               sleep(20)
               withEnv(["CF_APP=notify-delivery-worker"]) {
                 retry(3) {
-                  sh 'make cf-deploy-delivery-with-docker'
+                  sh 'make cf-deploy'
                 }
               }
             }, deployDeliveryWorkerSender: {
               sleep(30)
               withEnv(["CF_APP=notify-delivery-worker-sender"]) {
                 retry(3) {
-                  sh 'make cf-deploy-delivery-with-docker'
+                  sh 'make cf-deploy'
                 }
               }
             }, deployDeliveryWorkerDatabase: {
               sleep(40)
               withEnv(["CF_APP=notify-delivery-worker-database"]) {
                 retry(3) {
-                  sh 'make cf-deploy-delivery-with-docker'
+                  sh 'make cf-deploy'
                 }
               }
             }, deployDeliveryWorkerResearch: {
               sleep(50)
               withEnv(["CF_APP=notify-delivery-worker-research"]) {
                 retry(3) {
-                  sh 'make cf-deploy-delivery-with-docker'
+                  sh 'make cf-deploy'
                 }
               }
             }

Makefile

@@ -23,6 +23,8 @@ CF_API ?= api.cloud.service.gov.uk
 CF_ORG ?= govuk-notify
 CF_SPACE ?= ${DEPLOY_ENV}
 
+CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml
+
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -254,38 +256,33 @@ cf-login: ## Log in to Cloud Foundry
 	@echo "Logging in to Cloud Foundry on ${CF_API}"
 	@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
 
-.PHONY: cf-deploy-api
+.PHONY: cf-deploy
-cf-deploy-api: ## Deploys the API to Cloud Foundry
+cf-deploy: ## Deploys the app to Cloud Foundry
-	$(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid notify-api) | jq -r ".entity.instances"))
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
-	@echo "Original instance count: ${ORIG_INSTANCES}"
+	$(if ${CF_APP},,$(error Must specify CF_APP))
-	cf check-manifest notify-api -f manifest-api-${CF_SPACE}.yml
+	@cf app --guid ${CF_APP} || exit 1
-	cf zero-downtime-push notify-api -f manifest-api-${CF_SPACE}.yml
+	cf rename ${CF_APP} ${CF_APP}-rollback
-	cf scale -i ${ORIG_INSTANCES} notify-api
+	cf push ${CF_APP} -f ${CF_MANIFEST_FILE}
-
+	cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP}
-.PHONY: cf-push-api
+	cf stop ${CF_APP}-rollback
-cf-push-api: ##
+	cf delete -f ${CF_APP}-rollback
-	cf push notify-api -f manifest-api-${CF_SPACE}.yml
 
 .PHONY: cf-deploy-api-db-migration
-cf-deploy-api-db-migration: ## Deploys the API db migration to Cloud Foundry
+cf-deploy-api-db-migration:
-	cf check-manifest notify-api-db-migration -f manifest-api-db-migration.yml
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
-	cf push notify-api-db-migration -f manifest-api-db-migration.yml
+	cf push notify-api-db-migration -f manifest-api-${CF_SPACE}.yml
+	cf run-task notify-api-db-migration "scripts/run_app_paas.sh python db.py db upgrade" --name api_db_migration
 
-cf-push-api-db-migration: cf-deploy-api-db-migration ## Deploys the API db migration to Cloud Foundry
+.PHONY: cf-rollback
-
+cf-rollback: ## Rollbacks the app to the previous release
-.PHONY: cf-deploy-delivery
-cf-deploy-delivery: ## Deploys a delivery app to Cloud Foundry
 	$(if ${CF_APP},,$(error Must specify CF_APP))
-	$(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid ${CF_APP}) | jq -r ".entity.instances"))
+	@cf app --guid ${CF_APP}-rollback || exit 1
-	@echo "Original instance count: ${ORIG_INSTANCES}"
+	cf delete -f ${CF_APP} || true
-	cf check-manifest ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml
+	cf rename ${CF_APP}-rollback ${CF_APP}
-	cf zero-downtime-push ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml
-	cf scale -i ${ORIG_INSTANCES} ${CF_APP}
 
-.PHONY: cf-push-delivery
+.PHONY: cf-push
-cf-push-delivery: ## Deploys a delivery app to Cloud Foundry
+cf-push:
-	$(if ${CF_APP},,$(error Must specify CF_APP))
+	cf push ${CF_APP} -f ${CF_MANIFEST_FILE}
-	cf push ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml
 
 define cf_deploy_with_docker
 	@docker run -i${DOCKER_TTY} --rm \
@@ -308,15 +305,14 @@ define cf_deploy_with_docker
 		${2}
 endef
 
-.PHONY: cf-deploy-api-with-docker
+.PHONY: cf-deploy-with-docker
-cf-deploy-api-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container
+cf-deploy-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container
-	$(call cf_deploy_with_docker,cf-deploy-api,make cf-login cf-deploy-api)
+	$(call cf_deploy_with_docker,cf-deploy,make cf-login cf-deploy)
 
 .PHONY: cf-deploy-api-db-migration-with-docker
 cf-deploy-api-db-migration-with-docker: prepare-docker-build-image ## Deploys the API db migration to Cloud Foundry from a Docker container
 	$(call cf_deploy_with_docker,cf-deploy-api-db-migration,make cf-login cf-deploy-api-db-migration)
 
-.PHONY: cf-deploy-delivery-with-docker
+.PHONY: cf-rollback-with-docker
-cf-deploy-delivery-with-docker: prepare-docker-build-image ## Deploys a delivery app to Cloud Foundry from a Docker container
+cf-rollback-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container
-	$(if ${CF_APP},,$(error Must specify CF_APP))
+	$(call cf_deploy_with_docker,cf-rollback,make cf-login cf-rollback)
-	$(call cf_deploy_with_docker,cf-deploy-delivery-${CF_APP},make cf-login cf-deploy-delivery)

docker/Dockerfile

@@ -33,10 +33,7 @@ RUN \
 RUN \
 	echo "Install Cloud Foundry CLI" \
 	&& curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \
-	&& dpkg -i /tmp/cloudfoundry-cli.deb \
+	&& dpkg -i /tmp/cloudfoundry-cli.deb
-	&& cf install-plugin -r CF-Community -f "autopilot" \
-	&& cf install-plugin -r CF-Community -f "blue-green-deploy" \
-	&& cf install-plugin -r CF-Community -f "antifreeze"
 
 COPY tianon.gpg /tmp/tianon.gpg
 

manifest-api-base.yml

@@ -0,0 +1,26 @@
+---
+
+buildpack: python_buildpack
+command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+services:
+  - notify-aws
+  - notify-config
+  - notify-db
+  - mmg
+  - firetext
+  - hosted-graphite
+env:
+  NOTIFY_APP_NAME: public-api
+  CW_APP_NAME: api
+instances: 1
+memory: 512M
+
+applications:
+  - name: notify-api
+
+  - name: notify-api-db-migration
+    command: sleep infinity
+    no-route: true
+    health-check-type: none
+    instances: 1
+    memory: 128M

manifest-api-db-migration.yml

@@ -1,19 +0,0 @@
----
-
-applications:
-  - name: notify-api-db-migration
-    buildpack: python_buildpack
-    command: python db.py db upgrade && sleep infinity
-    services:
-      - notify-aws
-      - notify-config
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    env:
-      NOTIFY_APP_NAME: public-api
-    no-route: true
-    health-check-type: none
-    instances: 1
-    memory: 128M

manifest-api-preview.yml

@@ -1,20 +1,7 @@
 ---
 
-applications:
+inherit: manifest-api-base.yml
-  - name: notify-api
+
-    buildpack: python_buildpack
+routes:
-    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+  - route: notify-api-preview.cloudapps.digital
-    services:
+  - route: api-paas.notify.works
-      - notify-aws
-      - notify-config
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    env:
-      NOTIFY_APP_NAME: public-api
-    routes:
-      - route: notify-api-preview.cloudapps.digital
-      - route: api-paas.notify.works
-    instances: 1
-    memory: 512M

manifest-api-production.yml

@@ -1,20 +1,9 @@
 ---
 
-applications:
+inherit: manifest-api-base.yml
-  - name: notify-api
+
-    buildpack: python_buildpack
+routes:
-    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+  - route: notify-api-production.cloudapps.digital
-    services:
+  - route: api-paas.notifications.service.gov.uk
-      - notify-aws
+instances: 2
-      - notify-config
+memory: 2048M
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    env:
-      NOTIFY_APP_NAME: public-api
-    routes:
-      - route: notify-api-production.cloudapps.digital
-      - route: api-paas.notifications.service.gov.uk
-    instances: 2
-    memory: 2048M

manifest-api-sandbox.yml

@@ -1,19 +1,6 @@
 ---
 
-applications:
+inherit: manifest-api-base.yml
-  - name: notify-api
+
-    buildpack: python_buildpack
+routes:
-    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+  - route: notify-api-sandbox.cloudapps.digital
-    services:
-      - notify-aws
-      - notify-config
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    env:
-      NOTIFY_APP_NAME: public-api
-    routes:
-      - route: notify-api-sandbox.cloudapps.digital
-    instances: 1
-    memory: 512M

manifest-api-staging.yml

@@ -1,20 +1,9 @@
 ---
 
-applications:
+inherit: manifest-api-base.yml
-  - name: notify-api
+
-    buildpack: python_buildpack
+routes:
-    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+  - route: notify-api-staging.cloudapps.digital
-    services:
+  - route: api-paas.staging-notify.works
-      - notify-aws
+instances: 2
-      - notify-config
+memory: 2048M
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    env:
-      NOTIFY_APP_NAME: public-api
-    routes:
-      - route: notify-api-staging.cloudapps.digital
-      - route: api-paas.staging-notify.works
-    instances: 2
-    memory: 2048M

manifest-delivery-base.yml

@@ -0,0 +1,40 @@
+---
+
+buildpack: python_buildpack
+health-check-type: none
+no-route: true
+services:
+  - notify-aws
+  - notify-config
+  - notify-db
+  - mmg
+  - firetext
+  - hosted-graphite
+instances: 1
+memory: 256M
+
+applications:
+  - name: notify-delivery-celery-beat
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery beat --loglevel=INFO
+    env:
+      NOTIFY_APP_NAME: delivery-celery-beat
+
+  - name: notify-delivery-worker-database
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email,db-letter
+    env:
+      NOTIFY_APP_NAME: delivery-worker-database
+
+  - name: notify-delivery-worker-research
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode
+    env:
+      NOTIFY_APP_NAME: delivery-worker-research
+
+  - name: notify-delivery-worker-sender
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email
+    env:
+      NOTIFY_APP_NAME: delivery-worker-sender
+
+  - name: notify-delivery-worker
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11
+    env:
+      NOTIFY_APP_NAME: delivery-worker

manifest-delivery-celery-beat.yml

@@ -1,19 +0,0 @@
----
-
-applications:
-  - name: notify-delivery-celery-beat
-    buildpack: python_buildpack
-    health-check-type: none
-    no-route: true
-    services:
-      - notify-aws
-      - notify-config
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    instances: 2
-    memory: 128M
-    command: celery -A aws_run_celery.notify_celery beat --loglevel=INFO
-    env:
-      NOTIFY_APP_NAME: delivery-celery-beat

manifest-delivery-preview.yml

@@ -0,0 +1,3 @@
+---
+
+inherit: manifest-delivery-base.yml

manifest-delivery-production.yml

@@ -0,0 +1,6 @@
+---
+
+inherit: manifest-delivery-base.yml
+
+instances: 2
+memory: 1024M

manifest-delivery-sandbox.yml

@@ -0,0 +1,3 @@
+---
+
+inherit: manifest-delivery-base.yml

manifest-delivery-staging.yml

@@ -0,0 +1,6 @@
+---
+
+inherit: manifest-delivery-base.yml
+
+instances: 2
+memory: 1024M

manifest-delivery-worker-database.yml

@@ -1,19 +0,0 @@
----
-
-applications:
-  - name: notify-delivery-worker-database
-    buildpack: python_buildpack
-    health-check-type: none
-    no-route: true
-    services:
-      - notify-aws
-      - notify-config
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    instances: 2
-    memory: 256M
-    command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email,db-letter
-    env:
-      NOTIFY_APP_NAME: delivery-worker-database

manifest-delivery-worker-research.yml

@@ -1,19 +0,0 @@
----
-
-applications:
-  - name: notify-delivery-worker-research
-    buildpack: python_buildpack
-    health-check-type: none
-    no-route: true
-    services:
-      - notify-aws
-      - notify-config
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    instances: 2
-    memory: 256M
-    command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode
-    env:
-      NOTIFY_APP_NAME: delivery-worker-research

manifest-delivery-worker-sender.yml

@@ -1,19 +0,0 @@
----
-
-applications:
-  - name: notify-delivery-worker-sender
-    buildpack: python_buildpack
-    health-check-type: none
-    no-route: true
-    services:
-      - notify-aws
-      - notify-config
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    instances: 2
-    memory: 256M
-    command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email
-    env:
-      NOTIFY_APP_NAME: delivery-worker-sender

manifest-delivery-worker.yml

@@ -1,19 +0,0 @@
----
-
-applications:
-  - name: notify-delivery-worker
-    buildpack: python_buildpack
-    health-check-type: none
-    no-route: true
-    services:
-      - notify-aws
-      - notify-config
-      - notify-db
-      - mmg
-      - firetext
-      - hosted-graphite
-    instances: 2
-    memory: 256M
-    command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11
-    env:
-      NOTIFY_APP_NAME: delivery-worker

requirements.txt

@@ -25,6 +25,10 @@ docopt==0.6.2
 # pin to minor version 3.1.x
 notifications-python-client>=3.1,<3.2
 
+# PaaS
+awscli>=1.11,<1.12
+awscli-cwlogs>=1.4,<1.5
+
 git+https://github.com/alphagov/notifications-utils.git@13.5.0#egg=notifications-utils==13.5.0
 
 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3

scripts/run_app_paas.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+function check_params {
+  if [ -z "${NOTIFY_APP_NAME}" ]; then
+    echo "You must set NOTIFY_APP_NAME"
+    exit 1
+  fi
+
+  if [ -z "${CW_APP_NAME}" ]; then
+    CW_APP_NAME=${NOTIFY_APP_NAME}
+  fi
+}
+
+function configure_aws_logs {
+  aws configure set plugins.cwlogs cwlogs
+
+  export AWS_ACCESS_KEY_ID=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_access_key_id')
+  export AWS_SECRET_ACCESS_KEY=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_secret_access_key')
+
+  cat > /home/vcap/app/awslogs.conf << EOF
+[general]
+state_file = /home/vcap/logs/awslogs-state
+
+[/home/vcap/logs/app-stdout.log]
+file = /home/vcap/logs/app-stdout.log*
+log_group_name = paas-${CW_APP_NAME}-application
+log_stream_name = {hostname}-stdout
+
+[/home/vcap/logs/app-stderr.log]
+file = /home/vcap/logs/app-stderr.log*
+log_group_name = paas-${CW_APP_NAME}-application
+log_stream_name = {hostname}-stderr
+EOF
+}
+
+function on_exit {
+  echo "Terminating application process with pid ${APP_PID}"
+  kill ${APP_PID} || true
+  while (kill -0 ${APP_PID} 2&>/dev/null); do
+    echo "Application is still running.."
+    sleep 1
+  done
+  echo "Application process exited, waiting 10 seconds"
+  sleep 10
+  echo "Terminating remaining subprocesses.."
+  kill 0
+}
+
+function start_appplication {
+  exec "$@" 2>/home/vcap/logs/app-stderr.log | while read line; do echo $line; echo $line >> /home/vcap/logs/app-stdout.log.`date +%Y-%m-%d`; done &
+  LOGGER_PID=$!
+  APP_PID=`jobs -p`
+  echo "Logger process pid: ${LOGGER_PID}"
+  echo "Application process pid: ${APP_PID}"
+}
+
+function start_aws_logs_agent {
+  exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf &
+  AWSLOGS_AGENT_PID=$!
+  echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
+}
+
+function run {
+  while true; do
+    kill -0 ${APP_PID} 2&>/dev/null || break
+    kill -0 ${LOGGER_PID} 2&>/dev/null || break
+    kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
+    sleep 1
+  done
+}
+
+echo "Run script pid: $$"
+
+check_params
+
+trap "on_exit" EXIT
+
+configure_aws_logs
+
+# The application has to start first!
+start_appplication "$@"
+
+start_aws_logs_agent
+
+run