Merge pull request #1130 from alphagov/paas_awslogs

Add awslogs logging for PaaS

Makefile

@@ -198,15 +198,23 @@ cf-login: ## Log in to Cloud Foundry
 
 .PHONY: cf-deploy
 cf-deploy: ## Deploys the app to Cloud Foundry
-	$(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid notify-admin) | jq -r ".entity.instances"))
+	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
-	@echo "Original instance count: ${ORIG_INSTANCES}"
+	@cf app --guid notify-admin || exit 1
-	cf check-manifest notify-admin -f manifest-${CF_SPACE}.yml
+	cf rename notify-admin notify-admin-rollback
-	cf zero-downtime-push notify-admin -f manifest-${CF_SPACE}.yml
+	cf push -f manifest-${CF_SPACE}.yml
-	cf scale -i ${ORIG_INSTANCES} notify-admin
+	cf scale -i $$(cf curl /v2/apps/$$(cf app --guid notify-admin) | jq -r ".entity.instances" 2>/dev/null || echo "1") notify-admin
+	cf stop notify-admin-rollback
+	cf delete -f notify-admin-rollback
+
+.PHONY: cf-rollback
+cf-rollback: ## Rollbacks the app to the previous release
+	@cf app --guid notify-admin-rollback || exit 1
+	cf delete -f notify-admin || true
+	cf rename notify-admin-rollback notify-admin
 
 .PHONY: cf-push
 cf-push:
-	cf push notify-admin -f manifest-${CF_SPACE}.yml
+	cf push -f manifest-${CF_SPACE}.yml
 
 .PHONY: cf-deploy-with-docker
 cf-deploy-with-docker: prepare-docker-build-image ## Deploys the app to Cloud Foundry from a new Docker container

docker/Dockerfile

@@ -58,10 +58,7 @@ RUN \
 RUN \
 	echo "Install Cloud Foundry CLI" \
 	&& curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \
-	&& dpkg -i /tmp/cloudfoundry-cli.deb \
+	&& dpkg -i /tmp/cloudfoundry-cli.deb
-	&& cf install-plugin -r CF-Community -f "autopilot" \
-	&& cf install-plugin -r CF-Community -f "blue-green-deploy" \
-	&& cf install-plugin -r CF-Community -f "antifreeze"
 
 COPY tianon.gpg /tmp/tianon.gpg
 

manifest-base.yml

@@ -0,0 +1,16 @@
+---
+
+buildpack: python_buildpack
+command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+services:
+  - notify-aws
+  - notify-config
+  - hosted-graphite
+  - deskpro
+instances: 1
+memory: 512M
+env:
+  NOTIFY_APP_NAME: admin
+
+applications:
+  - name: notify-admin

manifest-preview.yml

@@ -1,16 +1,7 @@
 ---
 
-applications:
+inherit: manifest-base.yml
-  - name: notify-admin
+
-    buildpack: python_buildpack
+routes:
-    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+  - route: notify-admin-preview.cloudapps.digital
-    services:
+  - route: admin-paas.notify.works
-      - notify-aws
-      - notify-config
-      - hosted-graphite
-      - deskpro
-    routes:
-      - route: notify-admin-preview.cloudapps.digital
-      - route: admin-paas.notify.works
-    instances: 1
-    memory: 512M

manifest-production.yml

@@ -1,16 +1,9 @@
 ---
 
-applications:
+inherit: manifest-base.yml
-  - name: notify-admin
+
-    buildpack: python_buildpack
+routes:
-    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+  - route: notify-admin-production.cloudapps.digital
-    services:
+  - route: admin-paas.notifications.service.gov.uk
-      - notify-aws
+instances: 2
-      - notify-config
+memory: 2048M
-      - hosted-graphite
-      - deskpro
-    routes:
-      - route: notify-admin-production.cloudapps.digital
-      - route: admin-paas.notifications.service.gov.uk
-    instances: 2
-    memory: 2048M

manifest-sandbox.yml

@@ -1,15 +1,6 @@
 ---
 
-applications:
+inherit: manifest-base.yml
-  - name: notify-admin
+
-    buildpack: python_buildpack
+routes:
-    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+  - route: notify-admin-sandbox.cloudapps.digital
-    services:
-      - notify-aws
-      - notify-config
-      - hosted-graphite
-      - deskpro
-    routes:
-      - route: notify-admin-sandbox.cloudapps.digital
-    instances: 1
-    memory: 512M

manifest-staging.yml

@@ -1,16 +1,9 @@
 ---
 
-applications:
+inherit: manifest-base.yml
-  - name: notify-admin
+
-    buildpack: python_buildpack
+routes:
-    command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
+  - route: notify-admin-staging.cloudapps.digital
-    services:
+  - route: admin-paas.staging-notify.works
-      - notify-aws
+instances: 2
-      - notify-config
+memory: 2048M
-      - hosted-graphite
-      - deskpro
-    routes:
-      - route: notify-admin-staging.cloudapps.digital
-      - route: admin-paas.staging-notify.works
-    instances: 2
-    memory: 2048M

requirements.txt

@@ -29,4 +29,8 @@ whitenoise==1.0.6  #manages static assets
 # pin to minor version 3.1.x
 notifications-python-client>=3.1,<3.2
 
+# PaaS
+awscli>=1.11,<1.12
+awscli-cwlogs>=1.4,<1.5
+
 git+https://github.com/alphagov/notifications-utils.git@13.5.0#egg=notifications-utils==13.5.0

scripts/run_app_paas.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+function check_params {
+  if [ -z "${NOTIFY_APP_NAME}" ]; then
+    echo "You must set NOTIFY_APP_NAME"
+    exit 1
+  fi
+
+  if [ -z "${CW_APP_NAME}" ]; then
+    CW_APP_NAME=${NOTIFY_APP_NAME}
+  fi
+}
+
+function configure_aws_logs {
+  aws configure set plugins.cwlogs cwlogs
+
+  export AWS_ACCESS_KEY_ID=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_access_key_id')
+  export AWS_SECRET_ACCESS_KEY=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_secret_access_key')
+
+  cat > /home/vcap/app/awslogs.conf << EOF
+[general]
+state_file = /home/vcap/logs/awslogs-state
+
+[/home/vcap/logs/app-stdout.log]
+file = /home/vcap/logs/app-stdout.log*
+log_group_name = paas-${CW_APP_NAME}-application
+log_stream_name = {hostname}-stdout
+
+[/home/vcap/logs/app-stderr.log]
+file = /home/vcap/logs/app-stderr.log*
+log_group_name = paas-${CW_APP_NAME}-application
+log_stream_name = {hostname}-stderr
+EOF
+}
+
+function on_exit {
+  echo "Terminating application process with pid ${APP_PID}"
+  kill ${APP_PID} || true
+  while (kill -0 ${APP_PID} 2&>/dev/null); do
+    echo "Application is still running.."
+    sleep 1
+  done
+  echo "Application process exited, waiting 10 seconds"
+  sleep 10
+  echo "Terminating remaining subprocesses.."
+  kill 0
+}
+
+function start_appplication {
+  exec "$@" 2>/home/vcap/logs/app-stderr.log | while read line; do echo $line; echo $line >> /home/vcap/logs/app-stdout.log.`date +%Y-%m-%d`; done &
+  LOGGER_PID=$!
+  APP_PID=`jobs -p`
+  echo "Logger process pid: ${LOGGER_PID}"
+  echo "Application process pid: ${APP_PID}"
+}
+
+function start_aws_logs_agent {
+  exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf &
+  AWSLOGS_AGENT_PID=$!
+  echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
+}
+
+function run {
+  while true; do
+    kill -0 ${APP_PID} 2&>/dev/null || break
+    kill -0 ${LOGGER_PID} 2&>/dev/null || break
+    kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
+    sleep 1
+  done
+}
+
+echo "Run script pid: $$"
+
+check_params
+
+trap "on_exit" EXIT
+
+configure_aws_logs
+
+# The application has to start first!
+start_appplication "$@"
+
+start_aws_logs_agent
+
+run