Merge pull request #804 from alphagov/feat-add-perf-platform-client-and-job

Add client and job to update the performance platform daily

app/__init__.py

@@ -1,6 +1,5 @@
 import os
 import uuid
-import json
 
 from flask import Flask, _request_ctx_stack
 from flask import request, url_for, g, jsonify
@@ -18,6 +17,7 @@ from app.clients.email.aws_ses import AwsSesClient
 from app.clients.sms.firetext import FiretextClient
 from app.clients.sms.loadtesting import LoadtestingClient
 from app.clients.sms.mmg import MMGClient
+from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
 from app.encryption import Encryption
 
 
@@ -34,6 +34,7 @@ aws_ses_client = AwsSesClient()
 encryption = Encryption()
 statsd_client = StatsdClient()
 redis_store = RedisClient()
+performance_platform_client = PerformancePlatformClient()
 
 clients = Clients()
 

app/celery/scheduled_tasks.py

@@ -5,10 +5,13 @@ from sqlalchemy.exc import SQLAlchemyError
 
 from app.aws import s3
 from app import notify_celery
+from app import performance_platform_client
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
 from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than
-from app.dao.notifications_dao import (delete_notifications_created_more_than_a_week_ago,
-                                       dao_timeout_notifications)
+from app.dao.notifications_dao import (
+    delete_notifications_created_more_than_a_week_ago,
+    dao_timeout_notifications
+)
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
 from app.statsd_decorators import statsd
 from app.celery.tasks import process_job
@@ -109,3 +112,24 @@ def timeout_notifications():
     if updated:
         current_app.logger.info(
             "Timeout period reached for {} notifications, status has been updated.".format(updated))
+
+
+@notify_celery.task(name='send-daily-performance-platform-stats')
+@statsd(namespace="tasks")
+def send_daily_performance_stats():
+    count_dict = performance_platform_client.get_total_sent_notifications_yesterday()
+    start_date = count_dict.get('start_date')
+
+    performance_platform_client.send_performance_stats(
+        start_date,
+        'sms',
+        count_dict.get('sms').get('count'),
+        'day'
+    )
+
+    performance_platform_client.send_performance_stats(
+        start_date,
+        'email',
+        count_dict.get('email').get('count'),
+        'day'
+    )

app/clients/performance_platform/performance_platform_client.py

@@ -0,0 +1,80 @@
+import base64
+import json
+from datetime import datetime
+from requests import request
+
+from flask import current_app
+
+from app.utils import (
+    get_midnight_for_day_before,
+    get_london_midnight_in_utc
+)
+
+
+class PerformancePlatformClient:
+
+    def init_app(self, app):
+        self.active = app.config.get('PERFORMANCE_PLATFORM_ENABLED')
+        if self.active:
+            self.bearer_token = app.config.get('PERFORMANCE_PLATFORM_TOKEN')
+            self.performance_platform_url = current_app.config.get('PERFORMANCE_PLATFORM_URL')
+
+    def send_performance_stats(self, date, channel, count, period):
+        if self.active:
+            payload = {
+                '_timestamp': date,
+                'service': 'govuk-notify',
+                'channel': channel,
+                'count': count,
+                'dataType': 'notifications',
+                'period': period
+            }
+            self._add_id_for_payload(payload)
+            self._send_stats_to_performance_platform(payload)
+
+    def get_total_sent_notifications_yesterday(self):
+        today = datetime.utcnow()
+        start_date = get_midnight_for_day_before(today)
+        end_date = get_london_midnight_in_utc(today)
+
+        from app.dao.notifications_dao import get_total_sent_notifications_in_date_range
+        return {
+            "start_date": start_date,
+            "end_date": end_date,
+            "email": {
+                "count": get_total_sent_notifications_in_date_range(start_date, end_date, 'email')
+            },
+            "sms": {
+                "count": get_total_sent_notifications_in_date_range(start_date, end_date, 'sms')
+            }
+        }
+
+    def _send_stats_to_performance_platform(self, payload):
+        headers = {
+            'Content-Type': "application/json",
+            'Authorization': 'Bearer {}'.format(self.bearer_token)
+        }
+        resp = request(
+            "POST",
+            self.performance_platform_url,
+            data=json.dumps(payload),
+            headers=headers
+        )
+
+        if resp.status_code != 200:
+            current_app.logger.error(
+                "Performance platform update request failed with {} '{}'".format(
+                    resp.status_code,
+                    resp.json())
+            )
+
+    def _add_id_for_payload(self, payload):
+        payload_string = '{}{}{}{}{}'.format(
+            payload['_timestamp'],
+            payload['service'],
+            payload['channel'],
+            payload['dataType'],
+            payload['period']
+        )
+        _id = base64.b64encode(payload_string.encode('utf-8'))
+        payload.update({'_id': _id.decode('utf-8')})

app/config.py

@@ -50,6 +50,11 @@ class Config(object):
     REDIS_URL = os.getenv('REDIS_URL')
     REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
 
+    # Performance platform
+    PERFORMANCE_PLATFORM_ENABLED = os.getenv('PERFORMANCE_PLATFORM_ENABLED') == '1'
+    PERFORMANCE_PLATFORM_URL = 'https://www.performance.service.gov.uk/data/govuk-notify/notifications'
+    PERFORMANCE_PLATFORM_TOKEN = os.getenv('PERFORMANCE_PLATFORM_TOKEN')
+
     # Logging
     DEBUG = False
     LOGGING_STDOUT_JSON = os.getenv('LOGGING_STDOUT_JSON') == '1'
@@ -119,6 +124,11 @@ class Config(object):
             'schedule': crontab(minute=0, hour='0,1,2'),
             'options': {'queue': 'periodic'}
         },
+        'send-daily-performance-platform-stats': {
+            'task': 'send-daily-performance-platform-stats',
+            'schedule': crontab(minute=30, hour=0),  # 00:30
+            'options': {'queue': 'periodic'}
+        },
         'timeout-sending-notifications': {
             'task': 'timeout-sending-notifications',
             'schedule': crontab(minute=0, hour='0,1,2'),

app/dao/notifications_dao.py

@@ -1,4 +1,3 @@
-
 import pytz
 from datetime import (
     datetime,
@@ -7,7 +6,7 @@ from datetime import (
 
 from flask import current_app
 from werkzeug.datastructures import MultiDict
-from sqlalchemy import (desc, func, or_, and_, asc, extract)
+from sqlalchemy import (desc, func, or_, and_, asc)
 from sqlalchemy.orm import joinedload
 
 from app import db, create_uuid
@@ -23,7 +22,8 @@ from app.models import (
     NOTIFICATION_PENDING,
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
-    KEY_TYPE_NORMAL, KEY_TYPE_TEST)
+    KEY_TYPE_NORMAL, KEY_TYPE_TEST
+)
 
 from app.dao.dao_utils import transactional
 from app.statsd_decorators import statsd
@@ -408,3 +408,16 @@ def get_april_fools(year):
     """
     return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(
         tzinfo=None)
+
+
+def get_total_sent_notifications_in_date_range(start_date, end_date, notification_type):
+    result = db.session.query(
+        func.count(NotificationHistory.id).label('count')
+    ).filter(
+        NotificationHistory.key_type != KEY_TYPE_TEST,
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at <= end_date,
+        NotificationHistory.notification_type == notification_type
+    ).scalar()
+
+    return result or 0

app/service/rest.py

@@ -53,7 +53,7 @@ from app.schemas import (
     notifications_filter_schema,
     detailed_service_schema
 )
-from app.utils import pagination_links
+from app.utils import pagination_links, get_london_midnight_in_utc
 
 service_blueprint = Blueprint('service', __name__)
 register_errors(service_blueprint)
@@ -287,8 +287,9 @@ def get_detailed_services(start_date, end_date, only_active=False, include_from_
     if start_date == datetime.utcnow().date():
         stats = dao_fetch_todays_stats_for_all_services(include_from_test_key=include_from_test_key)
     else:
-        stats = fetch_stats_by_date_range_for_all_services(start_date=start_date,
-                                                           end_date=end_date,
+
+        stats = fetch_stats_by_date_range_for_all_services(start_date=get_london_midnight_in_utc(start_date),
+                                                           end_date=get_london_midnight_in_utc(end_date),
                                                            include_from_test_key=include_from_test_key)
 
     for service_id, rows in itertools.groupby(stats, lambda x: x.service_id):

app/utils.py

@@ -1,5 +1,7 @@
+from datetime import datetime, timedelta
+
+import pytz
 from flask import url_for
-from app.models import SMS_TYPE, EMAIL_TYPE
 from notifications_utils.template import SMSMessageTemplate, PlainTextEmailTemplate
 
 
@@ -23,6 +25,24 @@ def url_with_token(data, url, config):
 
 
 def get_template_instance(template, values):
+    from app.models import SMS_TYPE, EMAIL_TYPE
     return {
         SMS_TYPE: SMSMessageTemplate, EMAIL_TYPE: PlainTextEmailTemplate
     }[template['template_type']](template, values)
+
+
+def get_london_midnight_in_utc(date):
+    """
+     This function converts date to midnight as BST (British Standard Time) to UTC,
+     the tzinfo is lastly removed from the datetime because the database stores the timestamps without timezone.
+     :param date: the day to calculate the London midnight in UTC for
+     :return: the datetime of London midnight in UTC, for example 2016-06-17 = 2016-06-17 23:00:00
+    """
+    return pytz.timezone('Europe/London').localize(datetime.combine(date, datetime.min.time())).astimezone(
+        pytz.UTC).replace(
+        tzinfo=None)
+
+
+def get_midnight_for_day_before(date):
+    day_before = date - timedelta(1)
+    return get_london_midnight_in_utc(day_before)