Make perf platform processing stats query the NotificationHistory table

app/dao/notifications_dao.py

@@ -44,7 +44,6 @@ from app.models import (
 
 from app.dao.dao_utils import transactional
 from app.statsd_decorators import statsd
-from app.utils import get_london_month_from_utc_column
 
 
 def dao_get_notification_statistics_for_service_and_day(service_id, day):
@@ -523,12 +522,12 @@ def set_scheduled_notification_to_processed(notification_id):
     db.session.commit()
 
 
-def dao_get_total_notifications_sent_per_day_for_perfomance_platform(start_date, end_date):
+def dao_get_total_notifications_sent_per_day_for_performance_platform(start_date, end_date):
     """
     SELECT
-    count(notifications),
+    count(notification_history),
     coalesce(sum(CASE WHEN sent_at - created_at <= interval '10 seconds' THEN 1 ELSE 0 END), 0)
-    FROM notifications
+    FROM notification_history
     WHERE
     created_at > 'START DATE' AND
     created_at < 'END DATE' AND
@@ -536,7 +535,7 @@ def dao_get_total_notifications_sent_per_day_for_perfomance_platform(start_date,
     key_type != 'test' AND
     notification_type != 'letter';
     """
-    under_10_secs = Notification.sent_at - Notification.created_at <= timedelta(seconds=10)
+    under_10_secs = NotificationHistory.sent_at - NotificationHistory.created_at <= timedelta(seconds=10)
     sum_column = functions.coalesce(functions.sum(
         case(
             [
@@ -545,13 +544,14 @@ def dao_get_total_notifications_sent_per_day_for_perfomance_platform(start_date,
             else_=0
         )
     ), 0)
+
     return db.session.query(
-        func.count(Notification.id).label('messages_total'),
+        func.count(NotificationHistory.id).label('messages_total'),
         sum_column.label('messages_within_10_secs')
     ).filter(
-        Notification.created_at >= start_date,
+        NotificationHistory.created_at >= start_date,
-        Notification.created_at < end_date,
+        NotificationHistory.created_at < end_date,
-        Notification.api_key_id.isnot(None),
+        NotificationHistory.api_key_id.isnot(None),
-        Notification.key_type != KEY_TYPE_TEST,
+        NotificationHistory.key_type != KEY_TYPE_TEST,
-        Notification.notification_type != LETTER_TYPE
+        NotificationHistory.notification_type != LETTER_TYPE
     ).one()

tests/app/conftest.py

@@ -136,24 +136,30 @@ def sample_service(
     restricted=False,
     limit=1000,
     email_from=None,
-    permissions=[SMS_TYPE, EMAIL_TYPE]
+    permissions=[SMS_TYPE, EMAIL_TYPE],
+    research_mode=None
 ):
     if user is None:
         user = create_user()
     if email_from is None:
         email_from = service_name.lower().replace(' ', '.')
+
     data = {
         'name': service_name,
         'message_limit': limit,
         'restricted': restricted,
         'email_from': email_from,
         'created_by': user,
-        'letter_contact_block': 'London,\nSW1A 1AA',
+        'letter_contact_block': 'London,\nSW1A 1AA'
     }
     service = Service.query.filter_by(name=service_name).first()
     if not service:
         service = Service(**data)
         dao_create_service(service, user, service_permissions=permissions)
+
+        if research_mode:
+            service.research_mode = research_mode
+
     else:
         if user not in service.users:
             dao_add_user_to_service(service, user)
@@ -206,6 +212,7 @@ def sample_template(
         })
     template = Template(**data)
     dao_create_template(template)
+
     return template
 
 
@@ -631,14 +638,22 @@ def sample_notification_history(
     status='created',
     created_at=None,
     notification_type=None,
-    key_type=KEY_TYPE_NORMAL
+    key_type=KEY_TYPE_NORMAL,
+    sent_at=None,
+    api_key=None
 ):
     if created_at is None:
         created_at = datetime.utcnow()
 
+    if sent_at is None:
+        sent_at = datetime.utcnow()
+
     if notification_type is None:
         notification_type = sample_template.template_type
 
+    if not api_key:
+        api_key = create_api_key(sample_template.service, key_type=key_type)
+
     notification_history = NotificationHistory(
         id=uuid.uuid4(),
         service=sample_template.service,
@@ -647,7 +662,10 @@ def sample_notification_history(
         status=status,
         created_at=created_at,
         notification_type=notification_type,
-        key_type=key_type
+        key_type=key_type,
+        api_key=api_key,
+        api_key_id=api_key and api_key.id,
+        sent_at=sent_at
     )
     notify_db.session.add(notification_history)
     notify_db.session.commit()

tests/app/dao/notification_dao/test_notification_dao_performance_platform.py

@@ -6,13 +6,20 @@ from app.dao.notifications_dao import dao_get_total_notifications_sent_per_day_f
 from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST
 
 from tests.app.db import create_notification
+from tests.app.conftest import (
+    sample_notification_history,
+    sample_service,
+    sample_template
+)
 
 
 def test_get_total_notifications_filters_on_date(sample_template):
     create_notification(sample_template, created_at=datetime(2016, 10, 17, 10, 0))
     create_notification(sample_template, created_at=datetime(2016, 10, 18, 10, 0))
     create_notification(sample_template, created_at=datetime(2016, 10, 19, 10, 0))
+
     result = dao_get_total_notifications_sent_per_day_for_performance_platform(date(2016, 10, 18), date(2016, 10, 19))
+
     assert result.messages_total == 1
 
 
@@ -21,7 +28,9 @@ def test_get_total_notifications_filters_on_date_at_midnight(sample_template):
     create_notification(sample_template, created_at=datetime(2016, 10, 18, 0, 0))
     create_notification(sample_template, created_at=datetime(2016, 10, 18, 23, 59, 59))
     create_notification(sample_template, created_at=datetime(2016, 10, 19, 0, 0))
+
     result = dao_get_total_notifications_sent_per_day_for_performance_platform(date(2016, 10, 18), date(2016, 10, 19))
+
     assert result.messages_total == 2
 
 
@@ -32,7 +41,9 @@ def test_get_total_notifications_only_counts_api_notifications(sample_template,
     create_notification(sample_template, job=sample_job)
     create_notification(sample_template, job=sample_job)
     create_notification(sample_template, api_key=sample_api_key)
+
     result = dao_get_total_notifications_sent_per_day_for_performance_platform(date(2016, 10, 18), date(2016, 10, 19))
+
     assert result.messages_total == 1
 
 
@@ -45,7 +56,9 @@ def test_get_total_notifications_ignores_test_keys(sample_template):
     create_notification(sample_template, key_type=KEY_TYPE_TEAM)
     create_notification(sample_template, key_type=KEY_TYPE_TEAM)
     create_notification(sample_template, key_type=KEY_TYPE_TEST)
+
     result = dao_get_total_notifications_sent_per_day_for_performance_platform(date(2016, 10, 18), date(2016, 10, 19))
+
     assert result.messages_total == 4
 
 
@@ -62,7 +75,9 @@ def test_get_total_notifications_ignores_letters(
     create_notification(sample_email_template)
     create_notification(sample_email_template)
     create_notification(sample_letter_template)
+
     result = dao_get_total_notifications_sent_per_day_for_performance_platform(date(2016, 10, 18), date(2016, 10, 19))
+
     assert result.messages_total == 4
 
 
@@ -75,6 +90,7 @@ def test_get_total_notifications_counts_messages_within_10_seconds(sample_templa
     create_notification(sample_template, sent_at=created_at + timedelta(seconds=15))
 
     result = dao_get_total_notifications_sent_per_day_for_performance_platform(date(2016, 10, 18), date(2016, 10, 19))
+
     assert result.messages_total == 3
     assert result.messages_within_10_secs == 2
 
@@ -82,7 +98,9 @@ def test_get_total_notifications_counts_messages_within_10_seconds(sample_templa
 @freeze_time('2016-10-18T10:00')
 def test_get_total_notifications_counts_messages_that_have_not_sent(sample_template):
     create_notification(sample_template, status='created', sent_at=None)
+
     result = dao_get_total_notifications_sent_per_day_for_performance_platform(date(2016, 10, 18), date(2016, 10, 19))
+
     assert result.messages_total == 1
     assert result.messages_within_10_secs == 0
 
@@ -90,5 +108,35 @@ def test_get_total_notifications_counts_messages_that_have_not_sent(sample_templ
 @freeze_time('2016-10-18T10:00')
 def test_get_total_notifications_returns_zero_if_no_data(notify_db_session):
     result = dao_get_total_notifications_sent_per_day_for_performance_platform(date(2016, 10, 18), date(2016, 10, 19))
+
     assert result.messages_total == 0
     assert result.messages_within_10_secs == 0
+
+
+@freeze_time('2016-10-18T10:00')
+def test_get_total_notifications_counts_ignores_research_mode(notify_db, notify_db_session):
+    created_at = datetime.utcnow()
+    service = sample_service(notify_db, notify_db_session, research_mode=True)
+    template = sample_template(notify_db, notify_db_session, service=service)
+
+    create_notification(template, status='created', sent_at=None)
+
+    sample_notification_history(
+        notify_db,
+        notify_db_session,
+        template,
+        notification_type='email',
+        sent_at=created_at + timedelta(seconds=5)
+    )
+    sample_notification_history(
+        notify_db,
+        notify_db_session,
+        template,
+        notification_type='sms',
+        sent_at=created_at + timedelta(seconds=5)
+    )
+
+    result = dao_get_total_notifications_sent_per_day_for_performance_platform(date(2016, 10, 18), date(2016, 10, 19))
+
+    assert result.messages_total == 2
+    assert result.messages_within_10_secs == 2