Merge pull request #309 from alphagov/statsd-integration

Statsd integration

app/__init__.py

@@ -1,5 +1,7 @@
 import uuid
 import os
+
+import statsd
 from flask import request, url_for, g
 from flask import Flask, _request_ctx_stack
 from flask.ext.sqlalchemy import SQLAlchemy
@@ -10,10 +12,10 @@ from notifications_utils import logging
 from app.celery.celery import NotifyCelery
 from app.clients import Clients
 from app.clients.sms.mmg import MMGClient
-from app.clients.sms.twilio import TwilioClient
 from app.clients.sms.firetext import FiretextClient
 from app.clients.sms.loadtesting import LoadtestingClient
 from app.clients.email.aws_ses import AwsSesClient
+from app.clients.statsd.statsd_client import StatsdClient
 from app.encryption import Encryption
 
 DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
@@ -22,12 +24,12 @@ DATE_FORMAT = "%Y-%m-%d"
 db = SQLAlchemy()
 ma = Marshmallow()
 notify_celery = NotifyCelery()
-twilio_client = TwilioClient()
 firetext_client = FiretextClient()
 loadtest_client = LoadtestingClient()
 mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
 encryption = Encryption()
+statsd_client = StatsdClient()
 
 clients = Clients()
 
@@ -47,11 +49,11 @@ def create_app(app_name=None):
     ma.init_app(application)
     init_app(application)
     logging.init_app(application)
-    twilio_client.init_app(application)
-    firetext_client.init_app(application)
-    loadtest_client.init_app(application)
-    mmg_client.init_app(application.config)
-    aws_ses_client.init_app(application.config['AWS_REGION'])
+    statsd_client.init_app(application)
+    firetext_client.init_app(application, statsd_client=statsd_client)
+    loadtest_client.init_app(application, statsd_client=statsd_client)
+    mmg_client.init_app(application.config, statsd_client=statsd_client)
+    aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
     notify_celery.init_app(application)
     encryption.init_app(application)
     clients.init_app(sms_clients=[firetext_client, mmg_client, loadtest_client], email_clients=[aws_ses_client])

app/celery/tasks.py

@@ -2,8 +2,9 @@ import itertools
 from datetime import datetime
 
 from flask import current_app
+from monotonic import monotonic
 from sqlalchemy.exc import SQLAlchemyError
-from app import clients
+from app import clients, statsd_client
 from app.clients.email import EmailClientException
 from app.clients.sms import SmsClientException
 from app.dao.services_dao import dao_fetch_service_by_id
@@ -112,6 +113,7 @@ def delete_invitations():
 
 @notify_celery.task(name="process-job")
 def process_job(job_id):
+    task_start = monotonic()
     start = datetime.utcnow()
     job = dao_get_job_by_id(job_id)
 
@@ -191,6 +193,8 @@ def process_job(job_id):
     current_app.logger.info(
         "Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished)
     )
+    statsd_client.incr("notifications.tasks.process-job")
+    statsd_client.timing("notifications.tasks.process-job.task-time", monotonic() - task_start)
 
 
 @notify_celery.task(name="remove-job")
@@ -202,6 +206,7 @@ def remove_job(job_id):
 
 @notify_celery.task(name="send-sms")
 def send_sms(service_id, notification_id, encrypted_notification, created_at):
+    task_start = monotonic()
     notification = encryption.decrypt(encrypted_notification)
     service = dao_fetch_service_by_id(service_id)
 
@@ -237,7 +242,11 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
             sent_by=provider.get_name(),
             content_char_count=template.replaced_content_count
         )
-
+        statsd_client.timing_with_dates(
+            "notifications.tasks.send-sms.queued-for",
+            sent_at,
+            datetime.strptime(created_at, DATETIME_FORMAT)
+        )
         dao_create_notification(notification_db_object, TEMPLATE_TYPE_SMS, provider.get_name())
 
         if restricted:
@@ -261,12 +270,15 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
         current_app.logger.info(
             "SMS {} created at {} sent at {}".format(notification_id, created_at, sent_at)
         )
+        statsd_client.incr("notifications.tasks.send-sms")
+        statsd_client.timing("notifications.tasks.send-sms.task-time", monotonic() - task_start)
     except SQLAlchemyError as e:
         current_app.logger.exception(e)
 
 
 @notify_celery.task(name="send-email")
 def send_email(service_id, notification_id, from_address, encrypted_notification, created_at):
+    task_start = monotonic()
     notification = encryption.decrypt(encrypted_notification)
     service = dao_fetch_service_by_id(service_id)
 
@@ -296,6 +308,11 @@ def send_email(service_id, notification_id, from_address, encrypted_notification
         )
 
         dao_create_notification(notification_db_object, TEMPLATE_TYPE_EMAIL, provider.get_name())
+        statsd_client.timing_with_dates(
+            "notifications.tasks.send-email.queued-for",
+            sent_at,
+            datetime.strptime(created_at, DATETIME_FORMAT)
+        )
 
         if restricted:
             return
@@ -312,7 +329,9 @@ def send_email(service_id, notification_id, from_address, encrypted_notification
                 body=template.replaced_govuk_escaped,
                 html_body=template.as_HTML_email,
             )
+
             update_notification_reference_by_id(notification_id, reference)
+
         except EmailClientException as e:
             current_app.logger.exception(e)
             notification_db_object.status = 'failed'
@@ -321,6 +340,8 @@ def send_email(service_id, notification_id, from_address, encrypted_notification
         current_app.logger.info(
             "Email {} created at {} sent at {}".format(notification_id, created_at, sent_at)
         )
+        statsd_client.incr("notifications.tasks.send-email")
+        statsd_client.timing("notifications.tasks.send-email.task-time", monotonic() - task_start)
     except SQLAlchemyError as e:
         current_app.logger.exception(e)
 

app/clients/email/aws_ses.py

@@ -39,10 +39,11 @@ class AwsSesClient(EmailClient):
     Amazon SES email client.
     '''
 
-    def init_app(self, region, *args, **kwargs):
+    def init_app(self, region, statsd_client, *args, **kwargs):
         self._client = boto3.client('ses', region_name=region)
         super(AwsSesClient, self).__init__(*args, **kwargs)
         self.name = 'ses'
+        self.statsd_client = statsd_client
 
     def get_name(self):
         return self.name
@@ -88,7 +89,9 @@ class AwsSesClient(EmailClient):
                 ReplyToAddresses=reply_to_addresses)
             elapsed_time = monotonic() - start_time
             current_app.logger.info("AWS SES request finished in {}".format(elapsed_time))
+            self.statsd_client.timing("notifications.clients.ses.request-time", elapsed_time)
             return response['MessageId']
         except Exception as e:
             # TODO logging exceptions
+            self.statsd_client.incr("notifications.clients.ses.error")
             raise AwsSesClientException(str(e))

app/clients/sms/firetext.py

@@ -50,11 +50,12 @@ class FiretextClient(SmsClient):
     FireText sms client.
     '''
 
-    def init_app(self, config, *args, **kwargs):
+    def init_app(self, config, statsd_client, *args, **kwargs):
         super(SmsClient, self).__init__(*args, **kwargs)
         self.api_key = config.config.get('FIRETEXT_API_KEY')
         self.from_number = config.config.get('FIRETEXT_NUMBER')
         self.name = 'firetext'
+        self.statsd_client = statsd_client
 
     def get_name(self):
         return self.name
@@ -90,8 +91,10 @@ class FiretextClient(SmsClient):
                     api_error.message
                 )
             )
+            self.statsd_client.incr("notifications.clients.firetext.error")
             raise api_error
         finally:
             elapsed_time = monotonic() - start_time
             current_app.logger.info("Firetext request finished in {}".format(elapsed_time))
+            self.statsd_client.timing("notifications.clients.firetext.request-time", elapsed_time)
         return response

app/clients/sms/loadtesting.py

@@ -11,8 +11,9 @@ class LoadtestingClient(FiretextClient):
     Loadtest sms client.
     '''
 
-    def init_app(self, config, *args, **kwargs):
+    def init_app(self, config, statsd_client, *args, **kwargs):
         super(FiretextClient, self).__init__(*args, **kwargs)
         self.api_key = config.config.get('LOADTESTING_API_KEY')
         self.from_number = config.config.get('LOADTESTING_NUMBER')
         self.name = 'loadtesting'
+        self.statsd_client = statsd_client

app/clients/sms/mmg.py

@@ -39,11 +39,12 @@ class MMGClient(SmsClient):
     MMG sms client
     '''
 
-    def init_app(self, config, *args, **kwargs):
+    def init_app(self, config, statsd_client, *args, **kwargs):
         super(SmsClient, self).__init__(*args, **kwargs)
         self.api_key = config.get('MMG_API_KEY')
         self.from_number = config.get('MMG_FROM_NUMBER')
         self.name = 'mmg'
+        self.statsd_client = statsd_client
 
     def get_name(self):
         return self.name
@@ -78,8 +79,10 @@ class MMGClient(SmsClient):
                     api_error.message
                 )
             )
+            self.statsd_client.incr("notifications.clients.mmg.error")
             raise api_error
         finally:
             elapsed_time = monotonic() - start_time
+            self.statsd_client.timing("notifications.clients.mmg.request-time", elapsed_time)
             current_app.logger.info("MMG request finished in {}".format(elapsed_time))
         return response

app/clients/sms/twilio.py

@@ -1,54 +0,0 @@
-from monotonic import monotonic
-from app.clients.sms import (
-    SmsClient, SmsClientException)
-from twilio.rest import TwilioRestClient
-from twilio import TwilioRestException
-from flask import current_app
-
-
-class TwilioClientException(SmsClientException):
-    pass
-
-
-class TwilioClient(SmsClient):
-    '''
-    Twilio sms client.
-    '''
-    def init_app(self, config, *args, **kwargs):
-        super(TwilioClient, self).__init__(*args, **kwargs)
-        self.client = TwilioRestClient(
-            config.config.get('TWILIO_ACCOUNT_SID'),
-            config.config.get('TWILIO_AUTH_TOKEN'))
-        self.from_number = config.config.get('TWILIO_NUMBER')
-        self.name = 'twilio'
-
-    def get_name(self):
-        return self.name
-
-    def send_sms(self, to, content):
-        start_time = monotonic()
-        try:
-            response = self.client.messages.create(
-                body=content,
-                to=to,
-                from_=self.from_number
-            )
-            return response.sid
-        except TwilioRestException as e:
-            current_app.logger.exception(e)
-            raise TwilioClientException(e)
-        finally:
-            elapsed_time = monotonic() - start_time
-            current_app.logger.info("Twilio request finished in {}".format(elapsed_time))
-
-    def status(self, message_id):
-        try:
-            response = self.client.messages.get(message_id)
-            if response.status in ('delivered', 'failed'):
-                return response.status
-            elif response.status == 'undelivered':
-                return 'sending'
-            return None
-        except TwilioRestException as e:
-            current_app.logger.exception(e)
-            raise TwilioClientException(e)

app/clients/statsd/statsd_client.py

@@ -0,0 +1,25 @@
+from statsd import StatsClient
+
+
+class StatsdClient(StatsClient):
+    def init_app(self, app, *args, **kwargs):
+        StatsClient.__init__(
+            self,
+            app.config.get('STATSD_HOST'),
+            app.config.get('STATSD_PORT'),
+            prefix=app.config.get('STATSD_PREFIX')
+        )
+        self.active = app.config.get('STATSD_ENABLED')
+
+    def incr(self, stat, count=1, rate=1):
+        if self.active:
+            super(StatsClient, self).incr(stat, count, rate)
+
+    def timing(self, stat, delta, rate=1):
+        if self.active:
+            super(StatsClient, self).timing(stat, delta, rate)
+
+    def timing_with_dates(self, stat, start, end, rate=1):
+        if self.active:
+            delta = (start - end).total_seconds() * 1000
+            super(StatsClient, self).timing(stat, delta, rate)

app/notifications/process_client_response.py

@@ -1,5 +1,7 @@
 import uuid
 from flask import current_app
+
+from app import statsd_client
 from app.dao import notifications_dao
 from app.clients.sms.firetext import get_firetext_responses
 from app.clients.sms.mmg import get_mmg_responses
@@ -67,5 +69,6 @@ def process_sms_client_response(status, reference, client_name):
                                                                                     reference,
                                                                                     notification_status_message))
 
+    statsd_client.incr('notifications.callback.{}.{}'.format(client_name.lower(), notification_statistics_status))
     success = "{} callback succeeded. reference {} updated".format(client_name, reference)
     return success, errors

app/notifications/rest.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-
+import statsd
 import itertools
 from flask import (
     Blueprint,
@@ -12,7 +12,7 @@ from flask import (
 from notifications_utils.recipients import allowed_to_send_to, first_column_heading
 from notifications_utils.template import Template
 from app.clients.email.aws_ses import get_aws_responses
-from app import api_user, encryption, create_uuid, DATETIME_FORMAT, DATE_FORMAT
+from app import api_user, encryption, create_uuid, DATETIME_FORMAT, DATE_FORMAT, statsd_client
 from app.authentication.auth import require_admin
 from app.dao import (
     templates_dao,
@@ -103,6 +103,7 @@ def process_ses_response():
                     )
                 )
 
+            statsd_client.incr('notifications.callback.ses.{}'.format(notification_statistics_status))
             return jsonify(
                 result="success", message="SES callback succeeded"
             ), 200
@@ -374,4 +375,5 @@ def send_notification(notification_type):
             datetime.utcnow().strftime(DATETIME_FORMAT)
         ), queue='email')
 
+    statsd_client.incr('notifications.api.{}'.format(notification_type))
     return jsonify(data={"notification": {"id": notification_id}}), 201