Merge pull request #1222 from alphagov/get-performance-stats

Get performance stats

app/celery/scheduled_tasks.py

@@ -8,7 +8,7 @@ from sqlalchemy.exc import SQLAlchemyError
 
 from app.aws import s3
 from app import notify_celery
-from app.performance_platform import total_sent_notifications
+from app.performance_platform import total_sent_notifications, processing_time
 from app import performance_platform_client
 from app.dao.date_util import get_month_start_and_end_date_in_utc
 from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
@@ -176,27 +176,32 @@ def timeout_notifications():
 @statsd(namespace="tasks")
 def send_daily_performance_platform_stats():
     if performance_platform_client.active:
-        count_dict = total_sent_notifications.get_total_sent_notifications_yesterday()
-        email_sent_count = count_dict.get('email').get('count')
-        sms_sent_count = count_dict.get('sms').get('count')
-        start_date = count_dict.get('start_date')
+        send_total_sent_notifications_to_performance_platform()
+        processing_time.send_processing_time_to_performance_platform()
 
-        current_app.logger.info(
-            "Attempting to update performance platform for date {} with email count {} and sms count {}"
-            .format(start_date, email_sent_count, sms_sent_count)
-        )
 
-        total_sent_notifications.send_total_notifications_sent_for_day_stats(
-            start_date,
-            'sms',
-            sms_sent_count
-        )
+def send_total_sent_notifications_to_performance_platform():
+    count_dict = total_sent_notifications.get_total_sent_notifications_yesterday()
+    email_sent_count = count_dict.get('email').get('count')
+    sms_sent_count = count_dict.get('sms').get('count')
+    start_date = count_dict.get('start_date')
 
-        total_sent_notifications.send_total_notifications_sent_for_day_stats(
-            start_date,
-            'email',
-            email_sent_count
-        )
+    current_app.logger.info(
+        "Attempting to update performance platform for date {} with email count {} and sms count {}"
+        .format(start_date, email_sent_count, sms_sent_count)
+    )
+
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_date,
+        'sms',
+        sms_sent_count
+    )
+
+    total_sent_notifications.send_total_notifications_sent_for_day_stats(
+        start_date,
+        'email',
+        email_sent_count
+    )
 
 
 @notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')

app/dao/notifications_dao.py

@@ -16,6 +16,8 @@ from notifications_utils.recipients import (
 from werkzeug.datastructures import MultiDict
 from sqlalchemy import (desc, func, or_, and_, asc)
 from sqlalchemy.orm import joinedload
+from sqlalchemy.sql.expression import case
+from sqlalchemy.sql import functions
 from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
 
 from app import db, create_uuid
@@ -42,7 +44,6 @@ from app.models import (
 
 from app.dao.dao_utils import transactional
 from app.statsd_decorators import statsd
-from app.utils import get_london_month_from_utc_column
 
 
 def dao_get_notification_statistics_for_service_and_day(service_id, day):
@@ -519,3 +520,38 @@ def set_scheduled_notification_to_processed(notification_id):
         {'pending': False}
     )
     db.session.commit()
+
+
+def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date):
+    """
+    SELECT
+    count(notification_history),
+    coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0)
+    FROM notification_history
+    WHERE
+    created_at > 'START DATE' AND
+    created_at < 'END DATE' AND
+    api_key_id IS NOT NULL AND
+    key_type != 'test' AND
+    notification_type != 'letter';
+    """
+    under_10_secs = NotificationHistory.sent_at - NotificationHistory.created_at <= timedelta(seconds=10)
+    sum_column = functions.coalesce(functions.sum(
+        case(
+            [
+                (under_10_secs, 1)
+            ],
+            else_=0
+        )
+    ), 0)
+
+    return db.session.query(
+        func.count(NotificationHistory.id).label('messages_total'),
+        sum_column.label('messages_within_10_secs')
+    ).filter(
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at < end_date,
+        NotificationHistory.api_key_id.isnot(None),
+        NotificationHistory.key_type != KEY_TYPE_TEST,
+        NotificationHistory.notification_type != LETTER_TYPE
+    ).one()

app/performance_platform/processing_time.py

@@ -0,0 +1,36 @@
+from datetime import datetime
+
+from flask import current_app
+
+from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
+from app.dao.notifications_dao import dao_get_total_notifications_sent_per_day_for_performance_platform
+from app import performance_platform_client
+
+
+def send_processing_time_to_performance_platform():
+    today = datetime.utcnow()
+    start_date = get_midnight_for_day_before(today)
+    end_date = get_london_midnight_in_utc(today)
+
+    result = dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date)
+
+    current_app.logger.info(
+        'Sending processing-time to performance platform for date {}. Total: {}, under 10 secs {}'.format(
+            start_date, result.messages_total, result.messages_within_10_secs
+        )
+    )
+
+    send_processing_time_data(start_date, 'messages-total', result.messages_total)
+    send_processing_time_data(start_date, 'messages-within-10-secs', result.messages_within_10_secs)
+
+
+def send_processing_time_data(date, status, count):
+    payload = performance_platform_client.format_payload(
+        dataset='processing-time',
+        date=date,
+        group_name='status',
+        group_value=status,
+        count=count
+    )
+
+    performance_platform_client.send_stats_to_performance_platform(payload)