diff --git a/Makefile b/Makefile index e774907aa..735b5e3ce 100644 --- a/Makefile +++ b/Makefile @@ -199,7 +199,10 @@ clean-docker-containers: ## Clean up any remaining docker containers .PHONY: clean clean: - rm -rf node_modules cache target venv .coverage build tests/.cache + rm -rf node_modules cache target venv .coverage build tests/.cache scripts/statsd_exporter + +scripts/statsd_exporter: + curl -sSL https://github.com/prometheus/statsd_exporter/releases/download/v0.9.0/statsd_exporter-0.9.0.linux-amd64.tar.gz | tar zx --strip-components=1 -C scripts/ statsd_exporter-0.9.0.linux-amd64/statsd_exporter .PHONY: cf-login cf-login: ## Log in to Cloud Foundry @@ -223,7 +226,7 @@ generate-manifest: <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1 .PHONY: cf-deploy -cf-deploy: ## Deploys the app to Cloud Foundry +cf-deploy: scripts/statsd_exporter ## Deploys the app to Cloud Foundry $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) $(if ${CF_APP},,$(error Must specify CF_APP)) cf target -o ${CF_ORG} -s ${CF_SPACE} @@ -253,7 +256,7 @@ cf-rollback: ## Rollbacks the app to the previous release cf v3-cancel-zdt-push ${CF_APP} .PHONY: cf-push -cf-push: +cf-push: scripts/statsd_exporter $(if ${CF_APP},,$(error Must specify CF_APP)) cf target -o ${CF_ORG} -s ${CF_SPACE} cf push ${CF_APP} -f <(make -s generate-manifest) diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 6a61b8ac9..3d8b43470 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -10,8 +10,8 @@ 'notify-delivery-celery-beat': {'memory': '128M'}, 'notify-delivery-worker-jobs': {}, - 'notify-delivery-worker-research': {}, - 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'}, + 'notify-delivery-worker-research': {'local_statsd': ['preview', 'staging']}, + 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G', 'local_statsd': ['preview', 'staging']}, 'notify-delivery-worker-periodic': {}, 'notify-delivery-worker-priority': {}, 'notify-delivery-worker-letters': {}, @@ -49,6 +49,9 @@ applications: services: - notify-db - logit-ssl-syslog-drain + {% if environment in app.get('local_statsd', []) -%} + - notify-prometheus + {% endif %} env: NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }} @@ -74,8 +77,8 @@ applications: AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}' AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}' - {% if environment == 'preview' %} - STATSD_HOST: "notify-statsd-exporter.apps.internal" + {% if environment in app.get('local_statsd', []) %} + STATSD_HOST: "localhost" STATSD_PREFIX: "" {% else %} STATSD_HOST: "statsd.hostedgraphite.com" diff --git a/scripts/run_app_paas.sh b/scripts/run_app_paas.sh index a4416e56f..201bc186e 100755 --- a/scripts/run_app_paas.sh +++ b/scripts/run_app_paas.sh @@ -69,10 +69,20 @@ function start_aws_logs_agent { echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" } +function start_statsd_exporter { + echo "Starting statsd exporter..." + exec ./scripts/statsd_exporter --web.listen-address=":${PORT}" --statsd.listen-udp=":8125" --statsd.mapping-config=statsd_mapping.yml & + STATSD_EXPORTER_PID=$! + echo "Statsd exporter pid: ${STATSD_EXPORTER_PID}" +} + function run { while true; do kill -0 ${APP_PID} 2&>/dev/null || break kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent + if [ "${STATSD_HOST}" == "localhost" ]; then + kill -0 ${STATSD_EXPORTER_PID} 2&>/dev/null || start_statsd_exporter + fi sleep 1 done } @@ -84,6 +94,9 @@ check_params trap "on_exit" EXIT configure_aws_logs +if [ "${STATSD_HOST}" == "localhost" ]; then + start_statsd_exporter +fi # The application has to start first! start_application "$@" diff --git a/scripts/run_multi_worker_app_paas.sh b/scripts/run_multi_worker_app_paas.sh index cab2307b4..25a95c452 100755 --- a/scripts/run_multi_worker_app_paas.sh +++ b/scripts/run_multi_worker_app_paas.sh @@ -104,6 +104,13 @@ function start_logs_tail { echo "tail pid: ${LOGS_TAIL_PID}" } +function start_statsd_exporter { + echo "Starting statsd exporter..." + exec ./scripts/statsd_exporter --web.listen-address=":${PORT}" --statsd.listen-udp=":8125" --statsd.mapping-config=statsd_mapping.yml & + STATSD_EXPORTER_PID=$! + echo "Statsd exporter pid: ${STATSD_EXPORTER_PID}" +} + function ensure_celery_is_running { if [ "${APP_PIDS}" = "" ]; then echo "There are no celery processes running, this container is bad" @@ -119,6 +126,7 @@ function ensure_celery_is_running { echo "Killing awslogs_agent and tail" kill -9 ${AWSLOGS_AGENT_PID} kill -9 ${LOGS_TAIL_PID} + kill -9 ${STATSD_EXPORTER_PID} exit 1 fi @@ -135,6 +143,9 @@ function run { done kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail + if [ "${STATSD_HOST}" == "localhost" ]; then + kill -0 ${STATSD_EXPORTER_PID} 2&>/dev/null || start_statsd_exporter + fi sleep 1 done } @@ -146,6 +157,9 @@ check_params trap "on_exit" EXIT configure_aws_logs +if [ "${STATSD_HOST}" == "localhost" ]; then + start_statsd_exporter +fi # The application has to start first! start_application "$@"