work out which table to get notification status data from

previously we checked notifications table, and if the results were
zero, checked the notification history table to see if there's data
in there. When we know that data isn't in notifications, we're still
checking. These queries take half a second per service, and we're
doing at least ten for each of the five thousand services we have in
notify. Most of these services have no data in either table for any
given day, and we can reduce the amount of queries we do by only
checking one table.

Check the data retention for a service, and then if the date is older
than the retention, get from history table.

NOTE: This requires that the delete tasks haven't run yet for the day!
If your retention is three days, this will look in the Notification
table for data from three days ago - expecting that shortly after the
task finishes, we'll delete that data.

tests/app/dao/test_fact_notification_status_dao.py

@@ -1,4 +1,4 @@
-from datetime import timedelta, datetime, date
+from datetime import timedelta, datetime, date, time
 from uuid import UUID
 
 import pytest
@@ -36,7 +36,8 @@ from freezegun import freeze_time
 
 from tests.app.db import (
     create_notification, create_service, create_template, create_ft_notification_status,
-    create_job, create_notification_history
+    create_job, create_notification_history,
+    create_service_data_retention
 )
 
 
@@ -48,24 +49,32 @@ def test_update_fact_notification_status(notify_db_session):
     third_service = create_service(service_name='third Service')
     third_template = create_template(service=third_service, template_type='letter')
 
-    create_notification(template=first_template, status='delivered')
-    create_notification(template=first_template, created_at=datetime.utcnow() - timedelta(days=1))
-    # simulate a service with data retention - data has been moved to history and does not exist in notifications
-    create_notification_history(template=second_template, status='temporary-failure')
-    create_notification_history(template=second_template, created_at=datetime.utcnow() - timedelta(days=1))
-    create_notification(template=third_template, status='created')
-    create_notification(template=third_template, created_at=datetime.utcnow() - timedelta(days=1))
+    create_service_data_retention(second_service, 'email', days_of_retention=3)
+
+    process_day = date.today() - timedelta(days=5)
+    with freeze_time(datetime.combine(process_day, time.min)):
+        create_notification(template=first_template, status='delivered')
+
+        # 2nd service email has 3 day data retention - data has been moved to history and doesn't exist in notifications
+        create_notification_history(template=second_template, status='temporary-failure')
+
+        create_notification(template=third_template, status='sending')
+
+    # these created notifications from a different day get ignored
+    with freeze_time(datetime.combine(date.today() - timedelta(days=4), time.min)):
+        create_notification(template=first_template)
+        create_notification_history(template=second_template)
+        create_notification(template=third_template)
 
-    process_day = datetime.utcnow()
     data = fetch_notification_status_for_day(process_day=process_day)
-    update_fact_notification_status(data=data, process_day=process_day.date())
+    update_fact_notification_status(data=data, process_day=process_day)
 
     new_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
                                                           FactNotificationStatus.notification_type
                                                           ).all()
 
     assert len(new_fact_data) == 3
-    assert new_fact_data[0].bst_date == process_day.date()
+    assert new_fact_data[0].bst_date == process_day
     assert new_fact_data[0].template_id == second_template.id
     assert new_fact_data[0].service_id == second_service.id
     assert new_fact_data[0].job_id == UUID('00000000-0000-0000-0000-000000000000')
@@ -73,15 +82,15 @@ def test_update_fact_notification_status(notify_db_session):
     assert new_fact_data[0].notification_status == 'temporary-failure'
     assert new_fact_data[0].notification_count == 1
 
-    assert new_fact_data[1].bst_date == process_day.date()
+    assert new_fact_data[1].bst_date == process_day
     assert new_fact_data[1].template_id == third_template.id
     assert new_fact_data[1].service_id == third_service.id
     assert new_fact_data[1].job_id == UUID('00000000-0000-0000-0000-000000000000')
     assert new_fact_data[1].notification_type == 'letter'
-    assert new_fact_data[1].notification_status == 'created'
+    assert new_fact_data[1].notification_status == 'sending'
     assert new_fact_data[1].notification_count == 1
 
-    assert new_fact_data[2].bst_date == process_day.date()
+    assert new_fact_data[2].bst_date == process_day
     assert new_fact_data[2].template_id == first_template.id
     assert new_fact_data[2].service_id == first_service.id
     assert new_fact_data[2].job_id == UUID('00000000-0000-0000-0000-000000000000')
@@ -95,9 +104,9 @@ def test__update_fact_notification_status_updates_row(notify_db_session):
     first_template = create_template(service=first_service)
     create_notification(template=first_template, status='delivered')
 
-    process_day = datetime.utcnow()
+    process_day = date.today()
     data = fetch_notification_status_for_day(process_day=process_day)
-    update_fact_notification_status(data=data, process_day=process_day.date())
+    update_fact_notification_status(data=data, process_day=process_day)
 
     new_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
                                                           FactNotificationStatus.notification_type
@@ -108,7 +117,7 @@ def test__update_fact_notification_status_updates_row(notify_db_session):
     create_notification(template=first_template, status='delivered')
 
     data = fetch_notification_status_for_day(process_day=process_day)
-    update_fact_notification_status(data=data, process_day=process_day.date())
+    update_fact_notification_status(data=data, process_day=process_day)
 
     updated_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
                                                               FactNotificationStatus.notification_type

tests/app/test_utils.py

@@ -7,7 +7,11 @@ from app.utils import (
     get_london_midnight_in_utc,
     get_midnight_for_day_before,
     midnight_n_days_ago,
+    get_notification_table_to_use,
 )
+from app.models import Notification, NotificationHistory
+
+from tests.app.db import create_service_data_retention
 
 
 @pytest.mark.parametrize('date, expected_date', [
@@ -51,3 +55,46 @@ def test_get_midnight_for_day_before_returns_expected_date(date, expected_date):
 def test_midnight_n_days_ago(current_time, arg, expected_datetime):
     with freeze_time(current_time):
         assert midnight_n_days_ago(arg) == expected_datetime
+
+
+@freeze_time('2019-01-10 00:30')
+def test_get_notification_table_to_use(sample_service):
+    # it's currently early morning of Thurs 10th Jan.
+    # When the delete task runs a bit later, it'll delete data from last wednesday 2nd.
+    assert get_notification_table_to_use(sample_service, 'sms', date(2018, 12, 31)) == NotificationHistory
+    assert get_notification_table_to_use(sample_service, 'sms', date(2019, 1, 1)) == NotificationHistory
+    assert get_notification_table_to_use(sample_service, 'sms', date(2019, 1, 2)) == Notification
+    assert get_notification_table_to_use(sample_service, 'sms', date(2019, 1, 3)) == Notification
+
+
+@freeze_time('2019-01-10 00:30')
+@pytest.mark.parametrize('has_delete_task_run, table', [
+    (True, NotificationHistory),
+    (False, Notification),
+])
+def test_get_notification_table_to_use_knows_if_delete_task_has_run(sample_service, has_delete_task_run, table):
+    # it's currently early morning of Thurs 10th Jan.
+    # The delete task deletes/moves data from last wednesday 2nd.
+    assert get_notification_table_to_use(sample_service, 'sms', date(2019, 1, 2), has_delete_task_run) == table
+
+
+@freeze_time('2019-06-09 23:30')
+def test_get_notification_table_to_use_respects_daylight_savings_time(sample_service):
+    # current time is 12:30am on 10th july in BST
+    assert get_notification_table_to_use(sample_service, 'sms', date(2019, 6, 1)) == NotificationHistory
+    assert get_notification_table_to_use(sample_service, 'sms', date(2019, 6, 2)) == Notification
+
+
+@freeze_time('2019-01-10 00:30')
+def test_get_notification_table_to_use_checks_service_data_retention(sample_service):
+    create_service_data_retention(sample_service, 'email', days_of_retention=3)
+
+    # it's currently early morning of Thurs 10th Jan.
+    # three days retention means we'll delete sunday 6th's data when the delete task runs (so there's still three full
+    # days of monday, tuesday and wednesday left over)
+    assert get_notification_table_to_use(sample_service, 'email', date(2019, 1, 5)) == NotificationHistory
+    assert get_notification_table_to_use(sample_service, 'email', date(2019, 1, 6)) == Notification
+
+    # falls back to 7 days if not specified
+    assert get_notification_table_to_use(sample_service, 'sms', date(2019, 1, 1)) == NotificationHistory
+    assert get_notification_table_to_use(sample_service, 'sms', date(2019, 1, 2)) == Notification