Revert "Revert "Statsd to prometheus""

app/__init__.py

@@ -24,6 +24,7 @@ from flask.globals import _lookup_req_object, _request_ctx_stack
 from flask_login import LoginManager, current_user
 from flask_wtf import CSRFProtect
 from flask_wtf.csrf import CSRFError
+from gds_metrics import GDSMetrics
 from govuk_frontend_jinja.flask_ext import init_govuk_frontend
 from itsdangerous import BadSignature
 from notifications_python_client.errors import HTTPError
@@ -45,12 +46,7 @@ from app import proxy_fix
 from app.asset_fingerprinter import asset_fingerprinter
 from app.commands import setup_commands
 from app.config import configs
-from app.extensions import (
-    antivirus_client,
-    redis_client,
-    statsd_client,
-    zendesk_client,
-)
+from app.extensions import antivirus_client, redis_client, zendesk_client
 from app.models.organisation import Organisation
 from app.models.service import Service
 from app.models.user import AnonymousUser, User
@@ -99,6 +95,7 @@ from app.utils import format_thousands, get_logo_cdn_domain, id_safe
 
 login_manager = LoginManager()
 csrf = CSRFProtect()
+metrics = GDSMetrics()
 
 
 # The current service attached to the request stack.
@@ -137,6 +134,7 @@ def create_app(application):
         # Gubbins
         csrf,
         login_manager,
+        metrics,
         proxy_fix,
         request_helper,
 
@@ -166,14 +164,13 @@ def create_app(application):
 
         # External API clients
         antivirus_client,
-        statsd_client,
+        redis_client,
         zendesk_client,
-        redis_client
 
     ):
         client.init_app(application)
 
-    logging.init_app(application, statsd_client)
+    logging.init_app(application)
 
     login_manager.login_view = 'main.sign_in'
     login_manager.login_message_category = 'default'

app/config.py

@@ -20,11 +20,6 @@ class Config(object):
     TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')
     TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key')
 
-    # Hosted graphite statsd prefix
-    STATSD_ENABLED = False
-    STATSD_HOST = os.getenv('STATSD_HOST')
-    STATSD_PORT = 8125
-
     # Logging
     DEBUG = False
     NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH')
@@ -97,7 +92,6 @@ class Development(Config):
     DEBUG = True
     SESSION_COOKIE_SECURE = False
     SESSION_PROTECTION = None
-    STATSD_ENABLED = False
     CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
     CONTACT_LIST_UPLOAD_BUCKET_NAME = 'development-contact-list'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-tools'
@@ -120,7 +114,6 @@ class Development(Config):
 class Test(Development):
     DEBUG = True
     TESTING = True
-    STATSD_ENABLED = False
     WTF_CSRF_ENABLED = False
     CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
     CONTACT_LIST_UPLOAD_BUCKET_NAME = 'test-contact-list'
@@ -141,7 +134,6 @@ class Test(Development):
 class Preview(Config):
     HTTP_PROTOCOL = 'https'
     HEADER_COLOUR = '#F499BE'  # $baby-pink
-    STATSD_ENABLED = True
     CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
     CONTACT_LIST_UPLOAD_BUCKET_NAME = 'preview-contact-list'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview'
@@ -160,7 +152,6 @@ class Staging(Config):
     SHOW_STYLEGUIDE = False
     HTTP_PROTOCOL = 'https'
     HEADER_COLOUR = '#6F72AF'  # $mauve
-    STATSD_ENABLED = True
     CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload'
     CONTACT_LIST_UPLOAD_BUCKET_NAME = 'staging-contact-list'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging'
@@ -176,7 +167,6 @@ class Live(Config):
     SHOW_STYLEGUIDE = False
     HEADER_COLOUR = '#005EA5'  # $govuk-blue
     HTTP_PROTOCOL = 'https'
-    STATSD_ENABLED = True
     CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
     CONTACT_LIST_UPLOAD_BUCKET_NAME = 'production-contact-list'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-production'
@@ -196,7 +186,6 @@ class CloudFoundryConfig(Config):
 class Sandbox(CloudFoundryConfig):
     HTTP_PROTOCOL = 'https'
     HEADER_COLOUR = '#F499BE'  # $baby-pink
-    STATSD_ENABLED = True
     CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
     LOGO_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-logo-upload'
     NOTIFY_ENVIRONMENT = 'sandbox'

app/extensions.py

@@ -2,10 +2,8 @@ from notifications_utils.clients.antivirus.antivirus_client import (
     AntivirusClient,
 )
 from notifications_utils.clients.redis.redis_client import RedisClient
-from notifications_utils.clients.statsd.statsd_client import StatsdClient
 from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
 
 antivirus_client = AntivirusClient()
-statsd_client = StatsdClient()
 zendesk_client = ZendeskClient()
 redis_client = RedisClient()

app/navigation.py

@@ -28,7 +28,7 @@ class Navigation:
     def endpoints_without_navigation(self):
         return tuple(
             self._get_endpoint_with_blueprint(endpoint) for endpoint in self.exclude
-        ) + ('static', 'status.show_status')
+        ) + ('static', 'status.show_status', 'metrics')
 
     def is_selected(self, navigation_item):
         if request.endpoint in self.mapping[navigation_item]:

gunicorn_config.py

@@ -4,6 +4,8 @@ import traceback
 
 import gunicorn
 
+from gds_metrics.gunicorn import child_exit  # noqa
+
 workers = 5
 worker_class = "eventlet"
 errorlog = "/home/vcap/logs/gunicorn_error.log"

manifest.yml.j2

@@ -31,6 +31,7 @@ applications:
 
     services:
       - logit-ssl-syslog-drain
+      - notify-prometheus
 
     env:
       NOTIFY_APP_NAME: admin
@@ -50,8 +51,6 @@ applications:
       ANTIVIRUS_API_HOST: '{{ ANTIVIRUS_API_HOST }}'
       ANTIVIRUS_API_KEY: '{{ ANTIVIRUS_API_KEY }}'
 
-      STATSD_HOST: 'notify-statsd-exporter-{{ environment }}.apps.internal'
-
       ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}'
 
       TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}'

requirements-app.txt

@@ -25,3 +25,7 @@ itsdangerous==1.1.0
 
 git+https://github.com/alphagov/notifications-utils.git@39.7.0#egg=notifications-utils==39.7.0
 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha
+
+# gds-metrics requires prometheseus 0.2.0, override that requirement as later versions bring significant performance gains
+prometheus-client==0.8.0
+gds-metrics==0.2.0

requirements.txt

@@ -28,6 +28,10 @@ itsdangerous==1.1.0
 git+https://github.com/alphagov/notifications-utils.git@39.7.0#egg=notifications-utils==39.7.0
 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha
 
+# gds-metrics requires prometheseus 0.2.0, override that requirement as later versions bring significant performance gains
+prometheus-client==0.8.0
+gds-metrics==0.2.0
+
 ## The following requirements were added by pip freeze:
 awscli==1.18.90
 bleach==3.1.4