From 6958c0d677aa74ff12034bc7fd4155e6f622ee31 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 30 Jun 2020 11:08:11 +0100 Subject: [PATCH 1/2] Remove statsd We don't expose these metrics anywhere anyway and we want to move to prometheus too (which will be done in the next commit) --- app/__init__.py | 12 +++--------- app/config.py | 11 ----------- app/extensions.py | 2 -- manifest.yml.j2 | 2 -- 4 files changed, 3 insertions(+), 24 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index eb2775acd..bf26e100a 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -45,12 +45,7 @@ from app import proxy_fix from app.asset_fingerprinter import asset_fingerprinter from app.commands import setup_commands from app.config import configs -from app.extensions import ( - antivirus_client, - redis_client, - statsd_client, - zendesk_client, -) +from app.extensions import antivirus_client, redis_client, zendesk_client from app.models.organisation import Organisation from app.models.service import Service from app.models.user import AnonymousUser, User @@ -166,14 +161,13 @@ def create_app(application): # External API clients antivirus_client, - statsd_client, + redis_client, zendesk_client, - redis_client ): client.init_app(application) - logging.init_app(application, statsd_client) + logging.init_app(application) login_manager.login_view = 'main.sign_in' login_manager.login_message_category = 'default' diff --git a/app/config.py b/app/config.py index 9d470974e..a03d4aabd 100644 --- a/app/config.py +++ b/app/config.py @@ -20,11 +20,6 @@ class Config(object): TEMPLATE_PREVIEW_API_HOST = os.environ.get('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013') TEMPLATE_PREVIEW_API_KEY = os.environ.get('TEMPLATE_PREVIEW_API_KEY', 'my-secret-key') - # Hosted graphite statsd prefix - STATSD_ENABLED = False - STATSD_HOST = os.getenv('STATSD_HOST') - STATSD_PORT = 8125 - # Logging DEBUG = False NOTIFY_LOG_PATH = os.getenv('NOTIFY_LOG_PATH') @@ -97,7 +92,6 @@ class Development(Config): DEBUG = True SESSION_COOKIE_SECURE = False SESSION_PROTECTION = None - STATSD_ENABLED = False CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' CONTACT_LIST_UPLOAD_BUCKET_NAME = 'development-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-tools' @@ -120,7 +114,6 @@ class Development(Config): class Test(Development): DEBUG = True TESTING = True - STATSD_ENABLED = False WTF_CSRF_ENABLED = False CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' CONTACT_LIST_UPLOAD_BUCKET_NAME = 'test-contact-list' @@ -141,7 +134,6 @@ class Test(Development): class Preview(Config): HTTP_PROTOCOL = 'https' HEADER_COLOUR = '#F499BE' # $baby-pink - STATSD_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' CONTACT_LIST_UPLOAD_BUCKET_NAME = 'preview-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview' @@ -160,7 +152,6 @@ class Staging(Config): SHOW_STYLEGUIDE = False HTTP_PROTOCOL = 'https' HEADER_COLOUR = '#6F72AF' # $mauve - STATSD_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload' CONTACT_LIST_UPLOAD_BUCKET_NAME = 'staging-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging' @@ -176,7 +167,6 @@ class Live(Config): SHOW_STYLEGUIDE = False HEADER_COLOUR = '#005EA5' # $govuk-blue HTTP_PROTOCOL = 'https' - STATSD_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload' CONTACT_LIST_UPLOAD_BUCKET_NAME = 'production-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-production' @@ -196,7 +186,6 @@ class CloudFoundryConfig(Config): class Sandbox(CloudFoundryConfig): HTTP_PROTOCOL = 'https' HEADER_COLOUR = '#F499BE' # $baby-pink - STATSD_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload' LOGO_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-logo-upload' NOTIFY_ENVIRONMENT = 'sandbox' diff --git a/app/extensions.py b/app/extensions.py index 935dfd7c4..0d676c8e6 100644 --- a/app/extensions.py +++ b/app/extensions.py @@ -2,10 +2,8 @@ from notifications_utils.clients.antivirus.antivirus_client import ( AntivirusClient, ) from notifications_utils.clients.redis.redis_client import RedisClient -from notifications_utils.clients.statsd.statsd_client import StatsdClient from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient antivirus_client = AntivirusClient() -statsd_client = StatsdClient() zendesk_client = ZendeskClient() redis_client = RedisClient() diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 93ed81c3b..996e1821e 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -50,8 +50,6 @@ applications: ANTIVIRUS_API_HOST: '{{ ANTIVIRUS_API_HOST }}' ANTIVIRUS_API_KEY: '{{ ANTIVIRUS_API_KEY }}' - STATSD_HOST: 'notify-statsd-exporter-{{ environment }}.apps.internal' - ZENDESK_API_KEY: '{{ ZENDESK_API_KEY }}' TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}' From 043e6ac69c18557b72c759b9287030829ff3778c Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 30 Jun 2020 11:31:56 +0100 Subject: [PATCH 2/2] Add GDS metrics package to admin app Follows the code from the API --- app/__init__.py | 3 +++ app/navigation.py | 2 +- gunicorn_config.py | 2 ++ manifest.yml.j2 | 1 + requirements-app.txt | 4 ++++ requirements.txt | 12 ++++++++---- 6 files changed, 19 insertions(+), 5 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index bf26e100a..daaaae04e 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -24,6 +24,7 @@ from flask.globals import _lookup_req_object, _request_ctx_stack from flask_login import LoginManager, current_user from flask_wtf import CSRFProtect from flask_wtf.csrf import CSRFError +from gds_metrics import GDSMetrics from govuk_frontend_jinja.flask_ext import init_govuk_frontend from itsdangerous import BadSignature from notifications_python_client.errors import HTTPError @@ -94,6 +95,7 @@ from app.utils import format_thousands, get_logo_cdn_domain, id_safe login_manager = LoginManager() csrf = CSRFProtect() +metrics = GDSMetrics() # The current service attached to the request stack. @@ -132,6 +134,7 @@ def create_app(application): # Gubbins csrf, login_manager, + metrics, proxy_fix, request_helper, diff --git a/app/navigation.py b/app/navigation.py index d328d9b4b..1aa043ef2 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -28,7 +28,7 @@ class Navigation: def endpoints_without_navigation(self): return tuple( self._get_endpoint_with_blueprint(endpoint) for endpoint in self.exclude - ) + ('static', 'status.show_status') + ) + ('static', 'status.show_status', 'metrics') def is_selected(self, navigation_item): if request.endpoint in self.mapping[navigation_item]: diff --git a/gunicorn_config.py b/gunicorn_config.py index 4ac2d0ff6..6f870d672 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -4,6 +4,8 @@ import traceback import gunicorn +from gds_metrics.gunicorn import child_exit # noqa + workers = 5 worker_class = "eventlet" errorlog = "/home/vcap/logs/gunicorn_error.log" diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 996e1821e..dab36fbab 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -31,6 +31,7 @@ applications: services: - logit-ssl-syslog-drain + - notify-prometheus env: NOTIFY_APP_NAME: admin diff --git a/requirements-app.txt b/requirements-app.txt index a26c2a81a..e4364d0c9 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -25,3 +25,7 @@ itsdangerous==1.1.0 git+https://github.com/alphagov/notifications-utils.git@39.7.0#egg=notifications-utils==39.7.0 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha + +# gds-metrics requires prometheseus 0.2.0, override that requirement as later versions bring significant performance gains +prometheus-client==0.8.0 +gds-metrics==0.2.0 diff --git a/requirements.txt b/requirements.txt index e56517437..1b6b76dba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,11 +28,15 @@ itsdangerous==1.1.0 git+https://github.com/alphagov/notifications-utils.git@39.7.0#egg=notifications-utils==39.7.0 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha +# gds-metrics requires prometheseus 0.2.0, override that requirement as later versions bring significant performance gains +prometheus-client==0.8.0 +gds-metrics==0.2.0 + ## The following requirements were added by pip freeze: -awscli==1.18.84 +awscli==1.18.90 bleach==3.1.4 boto3==1.10.38 -botocore==1.17.7 +botocore==1.17.13 cachetools==4.1.0 certifi==2020.6.20 chardet==3.0.4 @@ -45,7 +49,7 @@ et-xmlfile==1.0.1 flask-redis==0.4.0 future==0.18.2 greenlet==0.4.16 -idna==2.9 +idna==2.10 jdcal==1.4.1 Jinja2==2.11.2 jmespath==0.10.0 @@ -54,7 +58,7 @@ lxml==4.5.1 MarkupSafe==1.1.1 mistune==0.8.4 monotonic==1.5 -openpyxl==3.0.3 +openpyxl==3.0.4 orderedset==2.0.1 phonenumbers==8.11.2 pyasn1==0.4.8