Optimise the query for getting the platform statistics for all services. The page should render for all time after this change.

This is one step closer to eliminating the need to read from NotificationHistory.

app/dao/fact_notification_status_dao.py

@@ -199,7 +199,7 @@ def fetch_notification_statuses_for_job(job_id):
     ).all()
 
 
-def fetch_stats_for_all_services_by_date_range(start_date, end_date,include_from_test_key=True):
+def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_from_test_key=True):
     stats = db.session.query(
         FactNotificationStatus.service_id.label('service_id'),
         Service.name.label('name'),
@@ -230,8 +230,8 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date,include_from
     if not include_from_test_key:
         stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST)
 
-    today = get_london_midnight_in_utc(datetime.utcnow())
+    if start_date <= datetime.utcnow().date() <= end_date:
-    if start_date <= today.date() <= end_date:
+        today = get_london_midnight_in_utc(datetime.utcnow())
         subquery = db.session.query(
             Notification.notification_type.cast(db.Text).label('notification_type'),
             Notification.status.label('status'),
@@ -284,7 +284,9 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date,include_from
             all_stats_table.c.notification_type,
             all_stats_table.c.status,
         ).order_by(
-            all_stats_table.c.service_id
+            all_stats_table.c.name,
+            all_stats_table.c.notification_type,
+            all_stats_table.c.status
         )
     else:
         query = stats

app/service/rest.py

@@ -23,8 +23,8 @@ from app.dao.api_key_dao import (
 from app.dao.fact_notification_status_dao import (
     fetch_notification_status_for_service_by_month,
     fetch_notification_status_for_service_for_day,
-    fetch_notification_status_for_service_for_today_and_7_previous_days
+    fetch_notification_status_for_service_for_today_and_7_previous_days,
-)
+    fetch_stats_for_all_services_by_date_range)
 from app.dao.inbound_numbers_dao import dao_allocate_number_for_service
 from app.dao.organisation_dao import dao_get_organisation_by_service_id
 from app.dao.service_data_retention_dao import (
@@ -56,7 +56,6 @@ from app.dao.services_dao import (
     dao_remove_user_from_service,
     dao_suspend_service,
     dao_update_service,
-    fetch_stats_by_date_range_for_all_services
 )
 from app.dao.service_whitelist_dao import (
     dao_fetch_service_whitelist,
@@ -472,10 +471,10 @@ def get_detailed_services(start_date, end_date, only_active=False, include_from_
                                                         only_active=only_active)
     else:
 
-        stats = fetch_stats_by_date_range_for_all_services(start_date=start_date,
+        stats = fetch_stats_for_all_services_by_date_range(start_date=start_date,
                                                            end_date=end_date,
                                                            include_from_test_key=include_from_test_key,
-                                                           only_active=only_active)
+                                                           )
     results = []
     for service_id, rows in itertools.groupby(stats, lambda x: x.service_id):
         rows = list(rows)

tests/app/dao/test_fact_notification_status_dao.py

@@ -289,6 +289,7 @@ def set_up_data():
     create_notification(sms_template, created_at=datetime(2018, 10, 31, 11, 0, 0))
     create_notification(sms_template, created_at=datetime(2018, 10, 31, 12, 0, 0), status='delivered')
     create_notification(email_template, created_at=datetime(2018, 10, 31, 13, 0, 0), status='delivered')
+    return service_1, service_2
 
 
 def test_fetch_notification_statuses_for_job(sample_template):
@@ -308,8 +309,32 @@ def test_fetch_notification_statuses_for_job(sample_template):
 
 @freeze_time('2018-10-31 14:00')
 def test_fetch_stats_for_all_services_by_date_range(notify_db_session):
-    set_up_data()
+    service_1, service_2 = set_up_data()
-    results = fetch_stats_for_all_services_by_date_range( start_date=date(2018, 10, 29),
+    results = fetch_stats_for_all_services_by_date_range(start_date=date(2018, 10, 29),
-                                                          end_date=date(2018, 10, 31))
+                                                         end_date=date(2018, 10, 31))
-    print(results)
+    assert len(results) == 5
-    assert len(results) == 2
+
+    assert results[0].service_id == service_1.id
+    assert results[0].notification_type == 'email'
+    assert results[0].status == 'delivered'
+    assert results[0].count == 4
+
+    assert results[1].service_id == service_1.id
+    assert results[1].notification_type == 'sms'
+    assert results[1].status == 'created'
+    assert results[1].count == 2
+
+    assert results[2].service_id == service_1.id
+    assert results[2].notification_type == 'sms'
+    assert results[2].status == 'delivered'
+    assert results[2].count == 11
+
+    assert results[3].service_id == service_2.id
+    assert results[3].notification_type == 'letter'
+    assert results[3].status == 'delivered'
+    assert results[3].count == 10
+
+    assert results[4].service_id == service_2.id
+    assert not results[4].notification_type
+    assert not results[4].status
+    assert not results[4].count

tests/app/service/test_rest.py

@@ -1642,31 +1642,37 @@ def test_get_detailed_services_only_includes_todays_notifications(notify_db, not
     }
 
 
-@pytest.mark.parametrize(
+@pytest.mark.parametrize("start_date_delta, end_date_delta",
-    'set_time',
+                         [(2, 1),
-    ['2017-03-28T12:00:00', '2017-01-28T12:00:00', '2017-01-02T12:00:00', '2017-10-31T12:00:00']
+                          (3, 2),
-)
+                          (1, 0)
-def test_get_detailed_services_for_date_range(notify_db, notify_db_session, set_time):
+                          ])
+@freeze_time('2017-03-28T12:00:00')
+def test_get_detailed_services_for_date_range(sample_template, start_date_delta, end_date_delta):
     from app.service.rest import get_detailed_services
 
-    with freeze_time(set_time):
+    create_ft_notification_status(bst_date=(datetime.utcnow() - timedelta(days=3)).date(),
-        create_sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow() - timedelta(days=3))
+                                  service=sample_template.service,
-        create_sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow() - timedelta(days=2))
+                                  notification_type='sms')
-        create_sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow() - timedelta(days=1))
+    create_ft_notification_status(bst_date=(datetime.utcnow() - timedelta(days=2)).date(),
-        create_sample_notification(notify_db, notify_db_session, created_at=datetime.utcnow())
+                                  service=sample_template.service,
+                                  notification_type='sms')
+    create_ft_notification_status(bst_date=(datetime.utcnow() - timedelta(days=1)).date(),
+                                  service=sample_template.service,
+                                  notification_type='sms')
 
-        start_date = (datetime.utcnow() - timedelta(days=2)).date()
+    create_notification(template=sample_template, created_at=datetime.utcnow(), status='delivered')
-        end_date = (datetime.utcnow() - timedelta(days=1)).date()
+
+    start_date = (datetime.utcnow() - timedelta(days=start_date_delta)).date()
+    end_date = (datetime.utcnow() - timedelta(days=end_date_delta)).date()
 
     data = get_detailed_services(only_active=False, include_from_test_key=True,
                                  start_date=start_date, end_date=end_date)
 
     assert len(data) == 1
-    assert data[0]['statistics'] == {
+    assert data[0]['statistics'][EMAIL_TYPE] == {'delivered': 0, 'failed': 0, 'requested': 0}
-        EMAIL_TYPE: {'delivered': 0, 'failed': 0, 'requested': 0},
+    assert data[0]['statistics'][SMS_TYPE] == {'delivered': 2, 'failed': 0, 'requested': 2}
-        SMS_TYPE: {'delivered': 0, 'failed': 0, 'requested': 2},
+    assert data[0]['statistics'][LETTER_TYPE] == {'delivered': 0, 'failed': 0, 'requested': 0}
-        LETTER_TYPE: {'delivered': 0, 'failed': 0, 'requested': 0}
-    }
 
 
 def test_search_for_notification_by_to_field(client, sample_template, sample_email_template):