Merge pull request #2249 from alphagov/improve-performance-of-platform-admin-page

Improve performance of platform stats page

app/dao/fact_notification_status_dao.py

@@ -138,3 +138,51 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
         all_stats_table.c.notification_type,
         all_stats_table.c.status,
     ).all()
+
+
+def fetch_notification_status_totals_for_all_services(start_date, end_date):
+    stats = db.session.query(
+        FactNotificationStatus.notification_type.label('notification_type'),
+        FactNotificationStatus.notification_status.label('status'),
+        FactNotificationStatus.key_type.label('key_type'),
+        func.sum(FactNotificationStatus.notification_count).label('count')
+    ).filter(
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.bst_date <= end_date
+    ).group_by(
+        FactNotificationStatus.notification_type,
+        FactNotificationStatus.notification_status,
+        FactNotificationStatus.key_type,
+    ).order_by(
+        FactNotificationStatus.notification_type
+    )
+    today = get_london_midnight_in_utc(datetime.utcnow())
+    if start_date <= today.date() <= end_date:
+        stats_for_today = db.session.query(
+            Notification.notification_type.cast(db.Text).label('notification_type'),
+            Notification.status,
+            Notification.key_type,
+            func.count().label('count')
+        ).filter(
+            Notification.created_at >= today
+        ).group_by(
+            Notification.notification_type.cast(db.Text),
+            Notification.status,
+            Notification.key_type,
+        )
+        all_stats_table = stats.union_all(stats_for_today).subquery()
+        query = db.session.query(
+            all_stats_table.c.notification_type,
+            all_stats_table.c.status,
+            all_stats_table.c.key_type,
+            func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
+        ).group_by(
+            all_stats_table.c.notification_type,
+            all_stats_table.c.status,
+            all_stats_table.c.key_type,
+        ).order_by(
+            all_stats_table.c.notification_type
+        )
+    else:
+        query = stats
+    return query.all()

app/dao/notifications_dao.py

@@ -7,26 +7,25 @@ from datetime import (
 
 from boto.exception import BotoClientError
 from flask import current_app
-
+from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
 from notifications_utils.recipients import (
     validate_and_format_email_address,
     InvalidEmailError,
     try_validate_and_format_phone_number
 )
 from notifications_utils.statsd_decorators import statsd
-from werkzeug.datastructures import MultiDict
+from notifications_utils.timezones import convert_utc_to_bst
 from sqlalchemy import (desc, func, or_, asc)
 from sqlalchemy.orm import joinedload
-from sqlalchemy.sql.expression import case
 from sqlalchemy.sql import functions
-from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
-from notifications_utils.timezones import convert_utc_to_bst
+from sqlalchemy.sql.expression import case
+from werkzeug.datastructures import MultiDict
 
 from app import db, create_uuid
 from app.aws.s3 import remove_s3_object, get_s3_bucket_objects
-from app.letters.utils import LETTERS_PDF_FILE_LOCATION_STRUCTURE
-from app.utils import midnight_n_days_ago, escape_special_characters
+from app.dao.dao_utils import transactional
 from app.errors import InvalidRequest
+from app.letters.utils import LETTERS_PDF_FILE_LOCATION_STRUCTURE
 from app.models import (
     Notification,
     NotificationHistory,
@@ -47,9 +46,8 @@ from app.models import (
     EMAIL_TYPE,
     ServiceDataRetention
 )
-
-from app.dao.dao_utils import transactional
 from app.utils import get_london_midnight_in_utc
+from app.utils import midnight_n_days_ago, escape_special_characters
 
 
 @statsd(namespace="dao")
@@ -645,31 +643,3 @@ def guess_notification_type(search_term):
         return EMAIL_TYPE
     else:
         return SMS_TYPE
-
-
-@statsd(namespace='dao')
-def fetch_aggregate_stats_by_date_range_for_all_services(start_date, end_date):
-    start_date = get_london_midnight_in_utc(start_date)
-    end_date = get_london_midnight_in_utc(end_date + timedelta(days=1))
-    table = NotificationHistory
-
-    if start_date >= datetime.utcnow() - timedelta(days=7):
-        table = Notification
-
-    query = db.session.query(
-        table.notification_type,
-        table.status,
-        table.key_type,
-        func.count(table.id).label('count')
-    ).filter(
-        table.created_at >= start_date,
-        table.created_at < end_date
-    ).group_by(
-        table.notification_type,
-        table.key_type,
-        table.status
-    ).order_by(
-        table.notification_type,
-    )
-
-    return query.all()

app/platform_stats/rest.py

@@ -2,7 +2,7 @@ from datetime import datetime
 
 from flask import Blueprint, jsonify, request
 
-from app.dao.notifications_dao import fetch_aggregate_stats_by_date_range_for_all_services
+from app.dao.fact_notification_status_dao import fetch_notification_status_totals_for_all_services
 from app.errors import register_errors
 from app.platform_stats.platform_stats_schema import platform_stats_request
 from app.service.statistics import format_admin_stats
@@ -23,7 +23,7 @@ def get_platform_stats():
 
     start_date = datetime.strptime(request.args.get('start_date', today), '%Y-%m-%d').date()
     end_date = datetime.strptime(request.args.get('end_date', today), '%Y-%m-%d').date()
-    data = fetch_aggregate_stats_by_date_range_for_all_services(start_date=start_date, end_date=end_date)
+    data = fetch_notification_status_totals_for_all_services(start_date=start_date, end_date=end_date)
     stats = format_admin_stats(data)
 
     return jsonify(stats)