From 655d5a4e16371917977567dd3a075735f48b0614 Mon Sep 17 00:00:00 2001 From: Andy Paine Date: Wed, 22 May 2019 13:53:05 +0100 Subject: [PATCH] AUTO-413: Use an internal app for statsd preview - We are running statsd exporter as an app with a public route for Prometheus to scrape - This updates preview to send statsd metrics over the CF internal networking to the statsd exporter - Removes the sidecar statsd exporters too --- Makefile | 9 +++------ manifest.yml.j2 | 11 ++++------- scripts/run_app_paas.sh | 13 ------------- scripts/run_multi_worker_app_paas.sh | 14 -------------- 4 files changed, 7 insertions(+), 40 deletions(-) diff --git a/Makefile b/Makefile index 735b5e3ce..e774907aa 100644 --- a/Makefile +++ b/Makefile @@ -199,10 +199,7 @@ clean-docker-containers: ## Clean up any remaining docker containers .PHONY: clean clean: - rm -rf node_modules cache target venv .coverage build tests/.cache scripts/statsd_exporter - -scripts/statsd_exporter: - curl -sSL https://github.com/prometheus/statsd_exporter/releases/download/v0.9.0/statsd_exporter-0.9.0.linux-amd64.tar.gz | tar zx --strip-components=1 -C scripts/ statsd_exporter-0.9.0.linux-amd64/statsd_exporter + rm -rf node_modules cache target venv .coverage build tests/.cache .PHONY: cf-login cf-login: ## Log in to Cloud Foundry @@ -226,7 +223,7 @@ generate-manifest: <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1 .PHONY: cf-deploy -cf-deploy: scripts/statsd_exporter ## Deploys the app to Cloud Foundry +cf-deploy: ## Deploys the app to Cloud Foundry $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) $(if ${CF_APP},,$(error Must specify CF_APP)) cf target -o ${CF_ORG} -s ${CF_SPACE} @@ -256,7 +253,7 @@ cf-rollback: ## Rollbacks the app to the previous release cf v3-cancel-zdt-push ${CF_APP} .PHONY: cf-push -cf-push: scripts/statsd_exporter +cf-push: $(if ${CF_APP},,$(error Must specify CF_APP)) cf target -o ${CF_ORG} -s ${CF_SPACE} cf push ${CF_APP} -f <(make -s generate-manifest) diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 3d8b43470..0da756d85 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -10,8 +10,8 @@ 'notify-delivery-celery-beat': {'memory': '128M'}, 'notify-delivery-worker-jobs': {}, - 'notify-delivery-worker-research': {'local_statsd': ['preview', 'staging']}, - 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G', 'local_statsd': ['preview', 'staging']}, + 'notify-delivery-worker-research': {}, + 'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'}, 'notify-delivery-worker-periodic': {}, 'notify-delivery-worker-priority': {}, 'notify-delivery-worker-letters': {}, @@ -49,9 +49,6 @@ applications: services: - notify-db - logit-ssl-syslog-drain - {% if environment in app.get('local_statsd', []) -%} - - notify-prometheus - {% endif %} env: NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }} @@ -77,8 +74,8 @@ applications: AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}' AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}' - {% if environment in app.get('local_statsd', []) %} - STATSD_HOST: "localhost" + {% if environment == 'preview' %} + STATSD_HOST: "notify-statsd-exporter-{{ environment }}.apps.internal" STATSD_PREFIX: "" {% else %} STATSD_HOST: "statsd.hostedgraphite.com" diff --git a/scripts/run_app_paas.sh b/scripts/run_app_paas.sh index 201bc186e..a4416e56f 100755 --- a/scripts/run_app_paas.sh +++ b/scripts/run_app_paas.sh @@ -69,20 +69,10 @@ function start_aws_logs_agent { echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" } -function start_statsd_exporter { - echo "Starting statsd exporter..." - exec ./scripts/statsd_exporter --web.listen-address=":${PORT}" --statsd.listen-udp=":8125" --statsd.mapping-config=statsd_mapping.yml & - STATSD_EXPORTER_PID=$! - echo "Statsd exporter pid: ${STATSD_EXPORTER_PID}" -} - function run { while true; do kill -0 ${APP_PID} 2&>/dev/null || break kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent - if [ "${STATSD_HOST}" == "localhost" ]; then - kill -0 ${STATSD_EXPORTER_PID} 2&>/dev/null || start_statsd_exporter - fi sleep 1 done } @@ -94,9 +84,6 @@ check_params trap "on_exit" EXIT configure_aws_logs -if [ "${STATSD_HOST}" == "localhost" ]; then - start_statsd_exporter -fi # The application has to start first! start_application "$@" diff --git a/scripts/run_multi_worker_app_paas.sh b/scripts/run_multi_worker_app_paas.sh index 25a95c452..cab2307b4 100755 --- a/scripts/run_multi_worker_app_paas.sh +++ b/scripts/run_multi_worker_app_paas.sh @@ -104,13 +104,6 @@ function start_logs_tail { echo "tail pid: ${LOGS_TAIL_PID}" } -function start_statsd_exporter { - echo "Starting statsd exporter..." - exec ./scripts/statsd_exporter --web.listen-address=":${PORT}" --statsd.listen-udp=":8125" --statsd.mapping-config=statsd_mapping.yml & - STATSD_EXPORTER_PID=$! - echo "Statsd exporter pid: ${STATSD_EXPORTER_PID}" -} - function ensure_celery_is_running { if [ "${APP_PIDS}" = "" ]; then echo "There are no celery processes running, this container is bad" @@ -126,7 +119,6 @@ function ensure_celery_is_running { echo "Killing awslogs_agent and tail" kill -9 ${AWSLOGS_AGENT_PID} kill -9 ${LOGS_TAIL_PID} - kill -9 ${STATSD_EXPORTER_PID} exit 1 fi @@ -143,9 +135,6 @@ function run { done kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail - if [ "${STATSD_HOST}" == "localhost" ]; then - kill -0 ${STATSD_EXPORTER_PID} 2&>/dev/null || start_statsd_exporter - fi sleep 1 done } @@ -157,9 +146,6 @@ check_params trap "on_exit" EXIT configure_aws_logs -if [ "${STATSD_HOST}" == "localhost" ]; then - start_statsd_exporter -fi # The application has to start first! start_application "$@"