darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-01-31 23:26:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #828 from alphagov/paas_awslogs

Browse Source 
Add awslogs logging for PaaS
This commit is contained in:
Andras Ferencz-Szabo
2017-02-17 09:28:39 +00:00
committed by GitHub
parent d0424e319c 72dc93c6db
commit ce1fac63ec
21 changed files with 238 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
Jenkinsfile vendored
View File

@@ -37,42 +37,44 @@ def deploy(cfEnv) {
]) {
withEnv(["CF_SPACE=${cfEnv}"]) {
parallel deployApi: {
retry(3) {
sh 'make cf-deploy-api-with-docker'
withEnv(["CF_APP=notify-api"]) {
retry(3) {
sh 'make cf-deploy'
}
}
}, deployDeliveryCeleryBeat: {
sleep(10)
withEnv(["CF_APP=notify-delivery-celery-beat"]) {
retry(3) {
sh 'make cf-deploy-delivery-with-docker'
sh 'make cf-deploy'
}
}
}, deployDeliveryWorker: {
sleep(20)
withEnv(["CF_APP=notify-delivery-worker"]) {
retry(3) {
sh 'make cf-deploy-delivery-with-docker'
sh 'make cf-deploy'
}
}
}, deployDeliveryWorkerSender: {
sleep(30)
withEnv(["CF_APP=notify-delivery-worker-sender"]) {
retry(3) {
sh 'make cf-deploy-delivery-with-docker'
sh 'make cf-deploy'
}
}
}, deployDeliveryWorkerDatabase: {
sleep(40)
withEnv(["CF_APP=notify-delivery-worker-database"]) {
retry(3) {
sh 'make cf-deploy-delivery-with-docker'
sh 'make cf-deploy'
}
}
}, deployDeliveryWorkerResearch: {
sleep(50)
withEnv(["CF_APP=notify-delivery-worker-research"]) {
retry(3) {
sh 'make cf-deploy-delivery-with-docker'
sh 'make cf-deploy'
}
}
}

64
Makefile
View File

@@ -23,6 +23,8 @@ CF_API ?= api.cloud.service.gov.uk
CF_ORG ?= govuk-notify
CF_SPACE ?= ${DEPLOY_ENV}
CF_MANIFEST_FILE = manifest-$(firstword $(subst -, ,$(subst notify-,,${CF_APP})))-${CF_SPACE}.yml
.PHONY: help
help:
@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -254,38 +256,33 @@ cf-login: ## Log in to Cloud Foundry
@echo "Logging in to Cloud Foundry on ${CF_API}"
@cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}"
.PHONY: cf-deploy-api
cf-deploy-api: ## Deploys the API to Cloud Foundry
$(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid notify-api) | jq -r ".entity.instances"))
@echo "Original instance count: ${ORIG_INSTANCES}"
cf check-manifest notify-api -f manifest-api-${CF_SPACE}.yml
cf zero-downtime-push notify-api -f manifest-api-${CF_SPACE}.yml
cf scale -i ${ORIG_INSTANCES} notify-api
.PHONY: cf-push-api
cf-push-api: ##
cf push notify-api -f manifest-api-${CF_SPACE}.yml
.PHONY: cf-deploy
cf-deploy: ## Deploys the app to Cloud Foundry
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
$(if ${CF_APP},,$(error Must specify CF_APP))
@cf app --guid ${CF_APP} || exit 1
cf rename ${CF_APP} ${CF_APP}-rollback
cf push ${CF_APP} -f ${CF_MANIFEST_FILE}
cf scale -i $$(cf curl /v2/apps/$$(cf app --guid ${CF_APP}-rollback) | jq -r ".entity.instances" 2>/dev/null || echo "1") ${CF_APP}
cf stop ${CF_APP}-rollback
cf delete -f ${CF_APP}-rollback
.PHONY: cf-deploy-api-db-migration
cf-deploy-api-db-migration: ## Deploys the API db migration to Cloud Foundry
cf check-manifest notify-api-db-migration -f manifest-api-db-migration.yml
cf push notify-api-db-migration -f manifest-api-db-migration.yml
cf-deploy-api-db-migration:
$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
cf push notify-api-db-migration -f manifest-api-${CF_SPACE}.yml
cf run-task notify-api-db-migration "scripts/run_app_paas.sh python db.py db upgrade" --name api_db_migration
cf-push-api-db-migration: cf-deploy-api-db-migration ## Deploys the API db migration to Cloud Foundry
.PHONY: cf-deploy-delivery
cf-deploy-delivery: ## Deploys a delivery app to Cloud Foundry
.PHONY: cf-rollback
cf-rollback: ## Rollbacks the app to the previous release
$(if ${CF_APP},,$(error Must specify CF_APP))
$(eval export ORIG_INSTANCES=$(shell cf curl /v2/apps/$(shell cf app --guid ${CF_APP}) | jq -r ".entity.instances"))
@echo "Original instance count: ${ORIG_INSTANCES}"
cf check-manifest ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml
cf zero-downtime-push ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml
cf scale -i ${ORIG_INSTANCES} ${CF_APP}
@cf app --guid ${CF_APP}-rollback || exit 1
cf delete -f ${CF_APP} || true
cf rename ${CF_APP}-rollback ${CF_APP}
.PHONY: cf-push-delivery
cf-push-delivery: ## Deploys a delivery app to Cloud Foundry
$(if ${CF_APP},,$(error Must specify CF_APP))
cf push ${CF_APP} -f manifest-$(subst notify-,,${CF_APP}).yml
.PHONY: cf-push
cf-push:
cf push ${CF_APP} -f ${CF_MANIFEST_FILE}
define cf_deploy_with_docker
@docker run -i${DOCKER_TTY} --rm \
@@ -308,15 +305,14 @@ define cf_deploy_with_docker
${2}
endef
.PHONY: cf-deploy-api-with-docker
cf-deploy-api-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container
$(call cf_deploy_with_docker,cf-deploy-api,make cf-login cf-deploy-api)
.PHONY: cf-deploy-with-docker
cf-deploy-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container
$(call cf_deploy_with_docker,cf-deploy,make cf-login cf-deploy)
.PHONY: cf-deploy-api-db-migration-with-docker
cf-deploy-api-db-migration-with-docker: prepare-docker-build-image ## Deploys the API db migration to Cloud Foundry from a Docker container
$(call cf_deploy_with_docker,cf-deploy-api-db-migration,make cf-login cf-deploy-api-db-migration)
.PHONY: cf-deploy-delivery-with-docker
cf-deploy-delivery-with-docker: prepare-docker-build-image ## Deploys a delivery app to Cloud Foundry from a Docker container
$(if ${CF_APP},,$(error Must specify CF_APP))
$(call cf_deploy_with_docker,cf-deploy-delivery-${CF_APP},make cf-login cf-deploy-delivery)
.PHONY: cf-rollback-with-docker
cf-rollback-with-docker: prepare-docker-build-image ## Deploys the API to Cloud Foundry from a Docker container
$(call cf_deploy_with_docker,cf-rollback,make cf-login cf-rollback)

5
docker/Dockerfile
View File

@@ -33,10 +33,7 @@ RUN \
RUN \
echo "Install Cloud Foundry CLI" \
&& curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \
&& dpkg -i /tmp/cloudfoundry-cli.deb \
&& cf install-plugin -r CF-Community -f "autopilot" \
&& cf install-plugin -r CF-Community -f "blue-green-deploy" \
&& cf install-plugin -r CF-Community -f "antifreeze"
&& dpkg -i /tmp/cloudfoundry-cli.deb
COPY tianon.gpg /tmp/tianon.gpg

26
manifest-api-base.yml Normal file
View File

@@ -0,0 +1,26 @@
---
buildpack: python_buildpack
command: scripts/run_app_paas.sh gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
env:
NOTIFY_APP_NAME: public-api
CW_APP_NAME: api
instances: 1
memory: 512M
applications:
- name: notify-api
- name: notify-api-db-migration
command: sleep infinity
no-route: true
health-check-type: none
instances: 1
memory: 128M

19
manifest-api-db-migration.yml
View File

@@ -1,19 +0,0 @@
---
applications:
- name: notify-api-db-migration
buildpack: python_buildpack
command: python db.py db upgrade && sleep infinity
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
env:
NOTIFY_APP_NAME: public-api
no-route: true
health-check-type: none
instances: 1
memory: 128M

23
manifest-api-preview.yml
View File

@@ -1,20 +1,7 @@
---
applications:
- name: notify-api
buildpack: python_buildpack
command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
env:
NOTIFY_APP_NAME: public-api
routes:
- route: notify-api-preview.cloudapps.digital
- route: api-paas.notify.works
instances: 1
memory: 512M
inherit: manifest-api-base.yml
routes:
- route: notify-api-preview.cloudapps.digital
- route: api-paas.notify.works

25
manifest-api-production.yml
View File

@@ -1,20 +1,9 @@
---
applications:
- name: notify-api
buildpack: python_buildpack
command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
env:
NOTIFY_APP_NAME: public-api
routes:
- route: notify-api-production.cloudapps.digital
- route: api-paas.notifications.service.gov.uk
instances: 2
memory: 2048M
inherit: manifest-api-base.yml
routes:
- route: notify-api-production.cloudapps.digital
- route: api-paas.notifications.service.gov.uk
instances: 2
memory: 2048M

21
manifest-api-sandbox.yml
View File

@@ -1,19 +1,6 @@
---
applications:
- name: notify-api
buildpack: python_buildpack
command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
env:
NOTIFY_APP_NAME: public-api
routes:
- route: notify-api-sandbox.cloudapps.digital
instances: 1
memory: 512M
inherit: manifest-api-base.yml
routes:
- route: notify-api-sandbox.cloudapps.digital

25
manifest-api-staging.yml
View File

@@ -1,20 +1,9 @@
---
applications:
- name: notify-api
buildpack: python_buildpack
command: gunicorn -w 5 -b 0.0.0.0:$PORT wsgi
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
env:
NOTIFY_APP_NAME: public-api
routes:
- route: notify-api-staging.cloudapps.digital
- route: api-paas.staging-notify.works
instances: 2
memory: 2048M
inherit: manifest-api-base.yml
routes:
- route: notify-api-staging.cloudapps.digital
- route: api-paas.staging-notify.works
instances: 2
memory: 2048M

40
manifest-delivery-base.yml Normal file
View File

@@ -0,0 +1,40 @@
---
buildpack: python_buildpack
health-check-type: none
no-route: true
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
instances: 1
memory: 256M
applications:
- name: notify-delivery-celery-beat
command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery beat --loglevel=INFO
env:
NOTIFY_APP_NAME: delivery-celery-beat
- name: notify-delivery-worker-database
command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email,db-letter
env:
NOTIFY_APP_NAME: delivery-worker-database
- name: notify-delivery-worker-research
command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode
env:
NOTIFY_APP_NAME: delivery-worker-research
- name: notify-delivery-worker-sender
command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email
env:
NOTIFY_APP_NAME: delivery-worker-sender
- name: notify-delivery-worker
command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11
env:
NOTIFY_APP_NAME: delivery-worker

19
manifest-delivery-celery-beat.yml
View File

@@ -1,19 +0,0 @@
---
applications:
- name: notify-delivery-celery-beat
buildpack: python_buildpack
health-check-type: none
no-route: true
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
instances: 2
memory: 128M
command: celery -A aws_run_celery.notify_celery beat --loglevel=INFO
env:
NOTIFY_APP_NAME: delivery-celery-beat

3
manifest-delivery-preview.yml Normal file
View File

@@ -0,0 +1,3 @@
---
inherit: manifest-delivery-base.yml

6
manifest-delivery-production.yml Normal file
View File

@@ -0,0 +1,6 @@
---
inherit: manifest-delivery-base.yml
instances: 2
memory: 1024M

3
manifest-delivery-sandbox.yml Normal file
View File

@@ -0,0 +1,3 @@
---
inherit: manifest-delivery-base.yml

6
manifest-delivery-staging.yml Normal file
View File

@@ -0,0 +1,6 @@
---
inherit: manifest-delivery-base.yml
instances: 2
memory: 1024M

19
manifest-delivery-worker-database.yml
View File

@@ -1,19 +0,0 @@
---
applications:
- name: notify-delivery-worker-database
buildpack: python_buildpack
health-check-type: none
no-route: true
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
instances: 2
memory: 256M
command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email,db-letter
env:
NOTIFY_APP_NAME: delivery-worker-database

19
manifest-delivery-worker-research.yml
View File

@@ -1,19 +0,0 @@
---
applications:
- name: notify-delivery-worker-research
buildpack: python_buildpack
health-check-type: none
no-route: true
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
instances: 2
memory: 256M
command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode
env:
NOTIFY_APP_NAME: delivery-worker-research

19
manifest-delivery-worker-sender.yml
View File

@@ -1,19 +0,0 @@
---
applications:
- name: notify-delivery-worker-sender
buildpack: python_buildpack
health-check-type: none
no-route: true
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
instances: 2
memory: 256M
command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email
env:
NOTIFY_APP_NAME: delivery-worker-sender

19
manifest-delivery-worker.yml
View File

@@ -1,19 +0,0 @@
---
applications:
- name: notify-delivery-worker
buildpack: python_buildpack
health-check-type: none
no-route: true
services:
- notify-aws
- notify-config
- notify-db
- mmg
- firetext
- hosted-graphite
instances: 2
memory: 256M
command: celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11
env:
NOTIFY_APP_NAME: delivery-worker

4
requirements.txt
View File

@@ -25,6 +25,10 @@ docopt==0.6.2
# pin to minor version 3.1.x
notifications-python-client>=3.1,<3.2
# PaaS
awscli>=1.11,<1.12
awscli-cwlogs>=1.4,<1.5
git+https://github.com/alphagov/notifications-utils.git@13.5.0#egg=notifications-utils==13.5.0
git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3

87
scripts/run_app_paas.sh Executable file
View File

@@ -0,0 +1,87 @@
#!/bin/bash
set -e -o pipefail
function check_params {
if [ -z "${NOTIFY_APP_NAME}" ]; then
echo "You must set NOTIFY_APP_NAME"
exit 1
fi
if [ -z "${CW_APP_NAME}" ]; then
CW_APP_NAME=${NOTIFY_APP_NAME}
fi
}
function configure_aws_logs {
aws configure set plugins.cwlogs cwlogs
export AWS_ACCESS_KEY_ID=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_access_key_id')
export AWS_SECRET_ACCESS_KEY=$(echo ${VCAP_SERVICES} | jq -r '.["user-provided"][]|select(.name=="notify-aws")|.credentials.aws_secret_access_key')
cat > /home/vcap/app/awslogs.conf << EOF
[general]
state_file = /home/vcap/logs/awslogs-state
[/home/vcap/logs/app-stdout.log]
file = /home/vcap/logs/app-stdout.log*
log_group_name = paas-${CW_APP_NAME}-application
log_stream_name = {hostname}-stdout
[/home/vcap/logs/app-stderr.log]
file = /home/vcap/logs/app-stderr.log*
log_group_name = paas-${CW_APP_NAME}-application
log_stream_name = {hostname}-stderr
EOF
}
function on_exit {
echo "Terminating application process with pid ${APP_PID}"
kill ${APP_PID} || true
while (kill -0 ${APP_PID} 2&>/dev/null); do
echo "Application is still running.."
sleep 1
done
echo "Application process exited, waiting 10 seconds"
sleep 10
echo "Terminating remaining subprocesses.."
kill 0
}
function start_appplication {
exec "$@" 2>/home/vcap/logs/app-stderr.log | while read line; do echo $line; echo $line >> /home/vcap/logs/app-stdout.log.`date +%Y-%m-%d`; done &
LOGGER_PID=$!
APP_PID=`jobs -p`
echo "Logger process pid: ${LOGGER_PID}"
echo "Application process pid: ${APP_PID}"
}
function start_aws_logs_agent {
exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf &
AWSLOGS_AGENT_PID=$!
echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
}
function run {
while true; do
kill -0 ${APP_PID} 2&>/dev/null || break
kill -0 ${LOGGER_PID} 2&>/dev/null || break
kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
sleep 1
done
}
echo "Run script pid: $$"
check_params
trap "on_exit" EXIT
configure_aws_logs
# The application has to start first!
start_appplication "$@"
start_aws_logs_agent
run