Merge pull request #2468 from alphagov/local-statsd-exporter

Local statsd exporter

Makefile

@@ -199,7 +199,10 @@ clean-docker-containers: ## Clean up any remaining docker containers
 
 .PHONY: clean
 clean:
-	rm -rf node_modules cache target venv .coverage build tests/.cache
+	rm -rf node_modules cache target venv .coverage build tests/.cache scripts/statsd_exporter
+
+scripts/statsd_exporter:
+	curl -sSL https://github.com/prometheus/statsd_exporter/releases/download/v0.9.0/statsd_exporter-0.9.0.linux-amd64.tar.gz | tar zx --strip-components=1 -C scripts/ statsd_exporter-0.9.0.linux-amd64/statsd_exporter
 
 .PHONY: cf-login
 cf-login: ## Log in to Cloud Foundry
@@ -223,7 +226,7 @@ generate-manifest:
 	    <(${DECRYPT_CMD} ${NOTIFY_CREDENTIALS}/credentials/${CF_SPACE}/paas/environment-variables.gpg) 2>&1
 
 .PHONY: cf-deploy
-cf-deploy: ## Deploys the app to Cloud Foundry
+cf-deploy: scripts/statsd_exporter ## Deploys the app to Cloud Foundry
 	$(if ${CF_SPACE},,$(error Must specify CF_SPACE))
 	$(if ${CF_APP},,$(error Must specify CF_APP))
 	@cf app --guid ${CF_APP} || exit 1
@@ -261,7 +264,7 @@ cf-rollback: ## Rollbacks the app to the previous release
 	cf rename ${CF_APP}-rollback ${CF_APP}
 
 .PHONY: cf-push
-cf-push:
+cf-push: scripts/statsd_exporter
 	$(if ${CF_APP},,$(error Must specify CF_APP))
 	cf target -o ${CF_ORG} -s ${CF_SPACE}
 	cf push ${CF_APP} -f <(make -s generate-manifest)

app/config.py

@@ -281,9 +281,9 @@ class Config(object):
     NOTIFICATIONS_ALERT = 5  # five mins
     FROM_NUMBER = 'development'
 
-    STATSD_ENABLED = False
-    STATSD_HOST = "statsd.hostedgraphite.com"
+    STATSD_HOST = os.getenv('STATSD_HOST')
     STATSD_PORT = 8125
+    STATSD_ENABLED = bool(STATSD_HOST)
 
     SENDING_NOTIFICATIONS_TIMEOUT_PERIOD = 259200  # 3 days
 
@@ -356,11 +356,6 @@ class Development(Config):
     SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/notification_api'
     REDIS_URL = 'redis://localhost:6379/0'
 
-    STATSD_ENABLED = False
-    STATSD_HOST = "localhost"
-    STATSD_PORT = 1000
-    STATSD_PREFIX = "stats-prefix"
-
     ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
 
     for queue in QueueNames.all_queues():
@@ -431,7 +426,6 @@ class Staging(Config):
     LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
     LETTERS_SCAN_BUCKET_NAME = 'staging-letters-scan'
     INVALID_PDF_BUCKET_NAME = 'staging-letters-invalid-pdf'
-    STATSD_ENABLED = True
     FROM_NUMBER = 'stage'
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = True
@@ -447,7 +441,6 @@ class Live(Config):
     LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf'
     LETTERS_SCAN_BUCKET_NAME = 'production-letters-scan'
     INVALID_PDF_BUCKET_NAME = 'production-letters-invalid-pdf'
-    STATSD_ENABLED = True
     FROM_NUMBER = 'GOVUK'
     PERFORMANCE_PLATFORM_ENABLED = True
     API_RATE_LIMIT_ENABLED = True

manifest.yml.j2

@@ -9,8 +9,8 @@
 
   'notify-delivery-celery-beat': {'memory': '128M'},
   'notify-delivery-worker-database': {},
-  'notify-delivery-worker-research': {},
-  'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G'},
+  'notify-delivery-worker-research': {'local_statsd': ['preview', 'staging']},
+  'notify-delivery-worker-sender': {'disk_quota': '2G', 'memory': '3G', 'local_statsd': ['preview', 'staging']},
   'notify-delivery-worker-periodic': {},
   'notify-delivery-worker-priority': {},
   'notify-delivery-worker': {},
@@ -37,6 +37,10 @@ applications:
       {%- for route in app['routes'][environment] %}
       - route: {{ route }}
       {%- endfor -%}
+    {%- elif environment in app.get('local_statsd', []) -%}
+    health-check-type: none
+    routes:
+      - route: {{ CF_APP }}-{{ environment }}.cloudapps.digital
     {%- else -%}
     health-check-type: none
     no-route: true
@@ -45,6 +49,7 @@ applications:
     services:
       - notify-db
       - logit-ssl-syslog-drain
+      {% if environment in app.get('local_statsd', []) %}- notify-prometheus{% endif %}
 
     env:
       NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }}
@@ -70,7 +75,13 @@ applications:
       AWS_ACCESS_KEY_ID: '{{ AWS_ACCESS_KEY_ID }}'
       AWS_SECRET_ACCESS_KEY: '{{ AWS_SECRET_ACCESS_KEY }}'
 
+      {% if environment in app.get('local_statsd', []) %}
+      STATSD_HOST: "localhost"
+      STATSD_PREFIX: ""
+      {% else %}
+      STATSD_HOST: "statsd.hostedgraphite.com"
       STATSD_PREFIX: '{{ STATSD_PREFIX }}'
+      {% endif %}
 
       ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}'
 

scripts/run_app_paas.sh

@@ -69,10 +69,20 @@ function start_aws_logs_agent {
   echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
 }
 
+function start_statsd_exporter {
+  echo "Starting statsd exporter..."
+  exec ./scripts/statsd_exporter --web.listen-address=":${PORT}" --statsd.listen-udp=":8125" --statsd.mapping-config=statsd_mapping.yml &
+  STATSD_EXPORTER_PID=$!
+  echo "Statsd exporter pid: ${STATSD_EXPORTER_PID}"
+}
+
 function run {
   while true; do
     kill -0 ${APP_PID} 2&>/dev/null || break
     kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
+    if [ "${STATSD_HOST}" == "localhost" ]; then
+        kill -0 ${STATSD_EXPORTER_PID} 2&>/dev/null || start_statsd_exporter
+    fi
     sleep 1
   done
 }
@@ -84,6 +94,9 @@ check_params
 trap "on_exit" EXIT
 
 configure_aws_logs
+if [ "${STATSD_HOST}" == "localhost" ]; then
+    start_statsd_exporter
+fi
 
 # The application has to start first!
 start_application "$@"

scripts/run_multi_worker_app_paas.sh

@@ -84,23 +84,33 @@ function send_signal_to_celery_processes {
 }
 
 function start_application {
+  echo "Starting application..."
   eval "$@"
   get_celery_pids
   echo "Application process pids: "${APP_PIDS}
 }
 
 function start_aws_logs_agent {
+  echo "Starting aws logs agent..."
   exec aws logs push --region eu-west-1 --config-file /home/vcap/app/awslogs.conf &
   AWSLOGS_AGENT_PID=$!
   echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
 }
 
 function start_logs_tail {
+  echo "Starting logs tail..."
   exec tail -n0 -f ${LOGS_DIR}/app.log.json &
   LOGS_TAIL_PID=$!
   echo "tail pid: ${LOGS_TAIL_PID}"
 }
 
+function start_statsd_exporter {
+  echo "Starting statsd exporter..."
+  exec ./scripts/statsd_exporter --web.listen-address=":${PORT}" --statsd.listen-udp=":8125" --statsd.mapping-config=statsd_mapping.yml &
+  STATSD_EXPORTER_PID=$!
+  echo "Statsd exporter pid: ${STATSD_EXPORTER_PID}"
+}
+
 function ensure_celery_is_running {
   if [ "${APP_PIDS}" = "" ]; then
     echo "There are no celery processes running, this container is bad"
@@ -116,6 +126,7 @@ function ensure_celery_is_running {
     echo "Killing awslogs_agent and tail"
     kill -9 ${AWSLOGS_AGENT_PID}
     kill -9 ${LOGS_TAIL_PID}
+    kill -9 ${STATSD_EXPORTER_PID}
 
     exit 1
   fi
@@ -132,6 +143,9 @@ function run {
     done
     kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
     kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail
+    if [ "${STATSD_HOST}" == "localhost" ]; then
+        kill -0 ${STATSD_EXPORTER_PID} 2&>/dev/null || start_statsd_exporter
+    fi
     sleep 1
   done
 }
@@ -143,6 +157,9 @@ check_params
 trap "on_exit" EXIT
 
 configure_aws_logs
+if [ "${STATSD_HOST}" == "localhost" ]; then
+    start_statsd_exporter
+fi
 
 # The application has to start first!
 start_application "$@"

statsd_mapping.yml

@@ -0,0 +1,11 @@
+defaults:
+  timer_type: histogram
+  buckets: [.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25]
+  ttl: 0 # metrics do not expire
+
+mappings:
+- match: (\w+)\.notifications\.(.+)
+  match_type: regex
+  name: "notifications_${2}"
+  labels:
+    space: "$1"