In order to re-run the create_nightly_billing for dates in the past, we added a condition on which table is used.

This will allow us to re-run nightly billing for those 2 days where it failed.
For the majority of time the query will run on Notiifcations.

app/dao/fact_billing_dao.py

@@ -14,7 +14,8 @@ from app.models import (
     LETTER_TYPE,
     SMS_TYPE,
     Rate,
-    LetterRate
+    LetterRate,
+    NotificationHistory
 )
 from app.utils import convert_utc_to_bst, convert_bst_to_utc
 
@@ -84,43 +85,48 @@ def fetch_monthly_billing_for_year(service_id, year):
 def fetch_billing_data_for_day(process_day, service_id=None):
     start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
     end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
+    # use notification_history if process day is older than 7 days
+    # this is useful if we need to rebuild the ft_billing table for a date older than 7 days ago.
+    table = Notification
+    if start_date < datetime.utcnow() - timedelta(days=7):
+        table = NotificationHistory
 
     transit_data = db.session.query(
-        Notification.template_id,
+        table.template_id,
-        Notification.service_id,
+        table.service_id,
-        Notification.notification_type,
+        table.notification_type,
-        func.coalesce(Notification.sent_by,
+        func.coalesce(table.sent_by,
                       case(
                           [
-                              (Notification.notification_type == 'letter', 'dvla'),
+                              (table.notification_type == 'letter', 'dvla'),
-                              (Notification.notification_type == 'sms', 'unknown'),
+                              (table.notification_type == 'sms', 'unknown'),
-                              (Notification.notification_type == 'email', 'ses')
+                              (table.notification_type == 'email', 'ses')
                           ]),
                       ).label('sent_by'),
-        func.coalesce(Notification.rate_multiplier, 1).label('rate_multiplier'),
+        func.coalesce(table.rate_multiplier, 1).label('rate_multiplier'),
-        func.coalesce(Notification.international, False).label('international'),
+        func.coalesce(table.international, False).label('international'),
-        func.sum(Notification.billable_units).label('billable_units'),
+        func.sum(table.billable_units).label('billable_units'),
         func.count().label('notifications_sent'),
         Service.crown,
     ).filter(
-        Notification.status != NOTIFICATION_CREATED,  # at created status, provider information is not available
+        table.status != NOTIFICATION_CREATED,  # at created status, provider information is not available
-        Notification.status != NOTIFICATION_TECHNICAL_FAILURE,
+        table.status != NOTIFICATION_TECHNICAL_FAILURE,
-        Notification.key_type != KEY_TYPE_TEST,
+        table.key_type != KEY_TYPE_TEST,
-        Notification.created_at >= start_date,
+        table.created_at >= start_date,
-        Notification.created_at < end_date
+        table.created_at < end_date
     ).group_by(
-        Notification.template_id,
+        table.template_id,
-        Notification.service_id,
+        table.service_id,
-        Notification.notification_type,
+        table.notification_type,
         'sent_by',
-        Notification.rate_multiplier,
+        table.rate_multiplier,
-        Notification.international,
+        table.international,
         Service.crown
     ).join(
         Service
     )
     if service_id:
-        transit_data = transit_data.filter(Notification.service_id == service_id)
+        transit_data = transit_data.filter(table.service_id == service_id)
 
     return transit_data.all()
 

tests/app/dao/test_ft_billing_dao.py

@@ -10,7 +10,7 @@ from app.dao.fact_billing_dao import (
     get_rate,
     fetch_billing_totals_for_year,
 )
-from app.models import FactBilling
+from app.models import FactBilling, Notification
 from app.utils import convert_utc_to_bst
 from tests.app.db import (
     create_ft_billing,
@@ -187,6 +187,19 @@ def test_fetch_billing_data_for_day_returns_empty_list(notify_db_session):
     assert results == []
 
 
+def test_fetch_billing_data_for_day_uses_notification_history(notify_db_session):
+    service = create_service()
+    sms_template = create_template(service=service, template_type='sms')
+    create_notification(template=sms_template, status='delivered', created_at=datetime.utcnow() - timedelta(days=8))
+    create_notification(template=sms_template, status='delivered', created_at=datetime.utcnow() - timedelta(days=8))
+
+    Notification.query.delete()
+    db.session.commit()
+    results = fetch_billing_data_for_day(process_day=datetime.utcnow() - timedelta(days=8), service_id=service.id)
+    assert len(results) == 1
+    assert results[0].notifications_sent == 2
+
+
 def test_fetch_billing_data_for_day_returns_list_for_given_service(notify_db_session):
     service = create_service()
     service_2 = create_service(service_name='Service 2')