Update the ft_billing query to only include billable notification status types.

Update test_provider_statistics dao - this is really irrelevant since the endpoint using the query is not being used. We have a PR coming to delete the unused code.
Update rate_multiplier to always be an integer

app/commands.py

@@ -446,6 +446,8 @@ def migrate_data_to_ft_billing(start_date, end_date):
                     left join services s on s.id = n.service_id
                     where n.notification_status!='technical-failure'
                         and n.key_type!='test'
+                        and n.notification_status in
+                        ('sending', 'sent', 'delivered', 'temporary-failure', 'permanent-failure', 'failed')
                         and n.notification_status!='created'
                         and n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
                         at time zone 'UTC'
@@ -462,8 +464,8 @@ def migrate_data_to_ft_billing(start_date, end_date):
 
         result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
         db.session.commit()
-        current_app.logger.info('ft_billing: --- Completed took {}ms. Migrated {} rows.'.format(datetime.now() - start_time,
-                                                                              result.rowcount))
+        current_app.logger.info('ft_billing: --- Completed took {}ms. Migrated {} rows for {}'.format(
+            datetime.now() - start_time, result.rowcount, process_date))
 
         process_date += timedelta(days=1)
 

app/dao/fact_billing_dao.py

@@ -1,7 +1,7 @@
 from datetime import datetime, timedelta, time
 
 from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy import func, case, desc, Date
+from sqlalchemy import func, case, desc, Date, Integer
 
 from app import db
 from app.dao.date_util import get_financial_year
@@ -9,14 +9,13 @@ from app.models import (
     FactBilling,
     Notification,
     Service,
-    NOTIFICATION_CREATED,
-    NOTIFICATION_TECHNICAL_FAILURE,
     KEY_TYPE_TEST,
     LETTER_TYPE,
     SMS_TYPE,
     Rate,
     LetterRate,
-    NotificationHistory
+    NotificationHistory,
+    NOTIFICATION_STATUS_TYPES_BILLABLE
 )
 from app.utils import convert_utc_to_bst, convert_bst_to_utc
 
@@ -104,14 +103,13 @@ def fetch_billing_data_for_day(process_day, service_id=None):
                               (table.notification_type == 'email', 'ses')
                           ]),
                       ).label('sent_by'),
-        func.coalesce(table.rate_multiplier, 1).label('rate_multiplier'),
+        func.coalesce(table.rate_multiplier, 1).cast(Integer).label('rate_multiplier'),
         func.coalesce(table.international, False).label('international'),
         func.sum(table.billable_units).label('billable_units'),
         func.count().label('notifications_sent'),
         Service.crown,
     ).filter(
-        table.status != NOTIFICATION_CREATED,  # at created status, provider information is not available
-        table.status != NOTIFICATION_TECHNICAL_FAILURE,
+        table.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE),
         table.key_type != KEY_TYPE_TEST,
         table.created_at >= start_date,
         table.created_at < end_date

app/models.py

@@ -1068,7 +1068,6 @@ NOTIFICATION_STATUS_TYPES_BILLABLE = [
     NOTIFICATION_SENT,
     NOTIFICATION_DELIVERED,
     NOTIFICATION_FAILED,
-    NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
     NOTIFICATION_PERMANENT_FAILURE,
 ]

tests/app/dao/test_provider_statistics_dao.py

@@ -4,7 +4,10 @@ import uuid
 import pytest
 from freezegun import freeze_time
 
-from app.models import NotificationHistory, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST, NOTIFICATION_STATUS_TYPES
+from app.models import (
+    NotificationHistory, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST,
+    NOTIFICATION_STATUS_TYPES_BILLABLE
+)
 from app.dao.provider_statistics_dao import get_fragment_count
 
 
@@ -24,10 +27,10 @@ def test_get_fragment_count_separates_sms_and_email(notify_db, sample_template,
 
 
 def test_get_fragment_count_filters_on_status(notify_db, sample_template):
-    for status in NOTIFICATION_STATUS_TYPES:
+    for status in NOTIFICATION_STATUS_TYPES_BILLABLE:
         noti_hist(notify_db, sample_template, status=status)
-    # sending, sent, delivered, failed, technical-failure, temporary-failure, permanent-failure
-    assert get_fragment_count(sample_template.service_id)['sms_count'] == 7
+    # sending, sent, delivered, failed, temporary-failure, permanent-failure
+    assert get_fragment_count(sample_template.service_id)['sms_count'] == 6
 
 
 def test_get_fragment_count_filters_on_service_id(notify_db, sample_template, service_factory):