AUTO-413: Use an internal app for statsd preview

- We are running statsd exporter as an app with a public route for
  Prometheus to scrape
- This updates preview to send statsd metrics over the CF internal
  networking to the statsd exporter
- Removes the sidecar statsd exporters too

Makefile

@@ -199,10 +199,7 @@ clean-docker-containers: ## Clean up any remaining docker containers
 
 .PHONY: clean
 clean:
-	rm -rf node_modules cache target venv .coverage build tests/.cache scripts/statsd_exporter
+	rm -rf node_modules cache target venv .coverage build tests/.cache
-
-scripts/statsd_exporter:
-	curl -sSL https://github.com/prometheus/statsd_exporter/releases/download/v0.9.0/statsd_exporter-0.9.0.linux-amd64.tar.gz | tar zx --strip-components=1 -C scripts/ statsd_exporter-0.9.0.linux-amd64/statsd_exporter
 
 .PHONY: cf-login
 cf-login: ## Log in to Cloud Foundry
@@ -226,7 +223,7 @@ generate-manifest:
 	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
 
 .PHONY: cf-deploy
-cf-deploy: scripts/statsd_exporter ## Deploys the app to Cloud Foundry
+cf-deploy: ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	$(if ${CF_APP},,$(error Must specify CF_APP))
 	cf target -o ${CF_ORG} -s ${CF_SPACE}
@@ -256,7 +253,7 @@ cf-rollback: ## Rollbacks the app to the previous release
 	cf v3-cancel-zdt-push ${CF_APP}
 
 .PHONY: cf-push
-cf-push: scripts/statsd_exporter
+cf-push:
 	$(if ${CF_APP},,$(error Must specify CF_APP))
 	cf target -o ${CF_ORG} -s ${CF_SPACE}
 	cf push ${CF_APP} -f <(make -s generate-manifest)

manifest.yml.j2

@@ -10,8 +10,8 @@
 
   'notify-delivery-celery-beat': {'memory': '128M'},
   'notify-delivery-worker-jobs': {},
-  'notify-delivery-worker-research': {'local_statsd': ['preview', 'staging']},
+  'notify-delivery-worker-research': {},
-  'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G', 'local_statsd': ['preview', 'staging']},
+  'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'},
   'notify-delivery-worker-periodic': {},
   'notify-delivery-worker-priority': {},
   'notify-delivery-worker-letters': {},
@@ -49,9 +49,6 @@ applications:
     services:
       - notify-db
       - logit-ssl-syslog-drain
-      {% if environment in app.get('local_statsd', []) -%}
-      - notify-prometheus
-      {% endif %}
 
     env:
       NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }}
@@ -77,8 +74,8 @@ applications:
       AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}'
       AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}'
 
-      {% if environment in app.get('local_statsd', []) %}
+      {% if environment == 'preview' %}
-      STATSD_HOST: "localhost"
+      STATSD_HOST: "notify-statsd-exporter-{{ environment }}.apps.internal"
       STATSD_PREFIX: ""
       {% else %}
       STATSD_HOST: "statsd.hostedgraphite.com"

scripts/run_app_paas.sh

@@ -69,20 +69,10 @@ function start_aws_logs_agent {
   echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
 }
 
-function start_statsd_exporter {
-  echo "Starting statsd exporter..."
-  exec ./scripts/statsd_exporter --web.listen-address=":${PORT}" --statsd.listen-udp=":8125" --statsd.mapping-config=statsd_mapping.yml &
-  STATSD_EXPORTER_PID=$!
-  echo "Statsd exporter pid: ${STATSD_EXPORTER_PID}"
-}
-
 function run {
   while true; do
     kill -0 ${APP_PID} 2&>/dev/null || break
     kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
-    if [ "${STATSD_HOST}" == "localhost" ]; then
-        kill -0 ${STATSD_EXPORTER_PID} 2&>/dev/null || start_statsd_exporter
-    fi
     sleep 1
   done
 }
@@ -94,9 +84,6 @@ check_params
 trap "on_exit" EXIT
 
 configure_aws_logs
-if [ "${STATSD_HOST}" == "localhost" ]; then
-    start_statsd_exporter
-fi
 
 # The application has to start first!
 start_application "$@"

scripts/run_multi_worker_app_paas.sh

@@ -104,13 +104,6 @@ function start_logs_tail {
   echo "tail pid: ${LOGS_TAIL_PID}"
 }
 
-function start_statsd_exporter {
-  echo "Starting statsd exporter..."
-  exec ./scripts/statsd_exporter --web.listen-address=":${PORT}" --statsd.listen-udp=":8125" --statsd.mapping-config=statsd_mapping.yml &
-  STATSD_EXPORTER_PID=$!
-  echo "Statsd exporter pid: ${STATSD_EXPORTER_PID}"
-}
-
 function ensure_celery_is_running {
   if [ "${APP_PIDS}" = "" ]; then
     echo "There are no celery processes running, this container is bad"
@@ -126,7 +119,6 @@ function ensure_celery_is_running {
     echo "Killing awslogs_agent and tail"
     kill -9 ${AWSLOGS_AGENT_PID}
     kill -9 ${LOGS_TAIL_PID}
-    kill -9 ${STATSD_EXPORTER_PID}
 
     exit 1
   fi
@@ -143,9 +135,6 @@ function run {
     done
     kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
     kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail
-    if [ "${STATSD_HOST}" == "localhost" ]; then
-        kill -0 ${STATSD_EXPORTER_PID} 2&>/dev/null || start_statsd_exporter
-    fi
     sleep 1
   done
 }
@@ -157,9 +146,6 @@ check_params
 trap "on_exit" EXIT
 
 configure_aws_logs
-if [ "${STATSD_HOST}" == "localhost" ]; then
-    start_statsd_exporter
-fi
 
 # The application has to start first!
 start_application "$@"