work out which table to get notification status data from

previously we checked notifications table, and if the results were
zero, checked the notification history table to see if there's data
in there. When we know that data isn't in notifications, we're still
checking. These queries take half a second per service, and we're
doing at least ten for each of the five thousand services we have in
notify. Most of these services have no data in either table for any
given day, and we can reduce the amount of queries we do by only
checking one table.

Check the data retention for a service, and then if the date is older
than the retention, get from history table.

NOTE: This requires that the delete tasks haven't run yet for the day!
If your retention is three days, this will look in the Notification
table for data from three days ago - expecting that shortly after the
task finishes, we'll delete that data.

app/dao/fact_billing_dao.py

@@ -325,7 +325,7 @@ def fetch_billing_data_for_day(process_day, service_id=None):
                     service_id=id_of_service
                 )
 
-            transit_data = transit_data + results
+            transit_data += results
 
     return transit_data
 

app/dao/fact_notification_status_dao.py

@@ -14,7 +14,6 @@ from app.models import (
     KEY_TYPE_TEST,
     LETTER_TYPE,
     Notification,
-    NotificationHistory,
     NOTIFICATION_CANCELLED,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
@@ -28,41 +27,39 @@ from app.models import (
     SMS_TYPE,
     Template,
 )
-from app.utils import get_london_midnight_in_utc, midnight_n_days_ago, get_london_month_from_utc_column
+from app.utils import (
+    get_london_midnight_in_utc,
+    midnight_n_days_ago,
+    get_london_month_from_utc_column,
+    get_notification_table_to_use,
+)
 
 
 def fetch_notification_status_for_day(process_day):
     start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
     end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
-    # use notification_history if process day is older than 7 days
-    # this is useful if we need to rebuild the ft_billing table for a date older than 7 days ago.
+
     current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))
 
     all_data_for_process_day = []
-    service_ids = [x.id for x in Service.query.all()]
+    services = Service.query.all()
     # for each service
     # for each notification type
     # query notifications for day
     # if no rows try notificationHistory
-    for service_id in service_ids:
+    for service in services:
         for notification_type in [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE]:
+            table = get_notification_table_to_use(service, notification_type, process_day)
+
             data_for_service_and_type = query_for_fact_status_data(
-                table=Notification,
+                table=table,
                 start_date=start_date,
                 end_date=end_date,
                 notification_type=notification_type,
-                service_id=service_id
+                service_id=service.id
             )
 
-            if len(data_for_service_and_type) == 0:
-                data_for_service_and_type = query_for_fact_status_data(
-                    table=NotificationHistory,
-                    start_date=start_date,
-                    end_date=end_date,
-                    notification_type=notification_type,
-                    service_id=service_id
-                )
-            all_data_for_process_day = all_data_for_process_day + data_for_service_and_type
+            all_data_for_process_day += data_for_service_and_type
 
     return all_data_for_process_day
 

app/models.py

@@ -11,6 +11,7 @@ from sqlalchemy.dialects.postgresql import (
     JSON,
     JSONB,
 )
+from sqlalchemy.orm.collections import attribute_mapped_collection
 from sqlalchemy import UniqueConstraint, CheckConstraint, Index
 from notifications_utils.columns import Columns
 from notifications_utils.recipients import (
@@ -2044,7 +2045,13 @@ class ServiceDataRetention(db.Model):
 
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), unique=False, index=True, nullable=False)
-    service = db.relationship(Service, backref=db.backref('service_data_retention'))
+    service = db.relationship(
+        Service,
+        backref=db.backref(
+            'data_retention',
+            collection_class=attribute_mapped_collection('notification_type')
+        )
+    )
     notification_type = db.Column(notification_types, nullable=False)
     days_of_retention = db.Column(db.Integer, nullable=False)
     created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)

app/schemas.py

@@ -238,6 +238,7 @@ class ServiceSchema(BaseSchema):
             'reply_to_email_addresses',
             'letter_contacts',
             'complaints',
+            'data_retention',
         )
         strict = True
 

app/utils.py

@@ -3,6 +3,7 @@ from datetime import datetime, timedelta
 import pytz
 from flask import url_for
 from sqlalchemy import func
+from notifications_utils.timezones import convert_utc_to_bst
 from notifications_utils.template import SMSMessageTemplate, WithSubjectTemplate, get_html_email_body
 
 local_timezone = pytz.timezone("Europe/London")
@@ -112,3 +113,26 @@ def email_address_is_nhs(email_address):
     return email_address.lower().endswith((
         '@nhs.uk', '@nhs.net', '.nhs.uk', '.nhs.net',
     ))
+
+
+def get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False):
+    """
+    Work out what table will contain notification data for a service by looking up their data retention.
+
+    Make sure that when you run this you think about whether the delete task has run for that day! If it's run, the
+    notifications from that day will have moved to NotificationHistory. By default we assume it hasn't run, since
+    """
+    from app.models import Notification, NotificationHistory
+
+    data_retention = service.data_retention.get(notification_type)
+    days_of_retention = data_retention.days_of_retention if data_retention else 7
+
+    todays_bst_date = convert_utc_to_bst(datetime.utcnow()).date()
+    days_ago = todays_bst_date - process_day
+
+    if not has_delete_task_run:
+        # if the task hasn't run yet, we've got an extra day of data in the notification table so can go back an extra
+        # day before looking at NotificationHistory
+        days_of_retention += 1
+
+    return Notification if days_ago <= timedelta(days=days_of_retention) else NotificationHistory