Merge pull request #2192 from alphagov/exclude-failed-letters-from-billing

Update billing to ignore failed letters.

app/dao/fact_billing_dao.py

@@ -17,7 +17,8 @@ from app.models import (
     LetterRate,
     NOTIFICATION_STATUS_TYPES_BILLABLE,
     NotificationHistory,
-    EMAIL_TYPE
+    EMAIL_TYPE,
+    NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS
 )
 from app.utils import convert_utc_to_bst, convert_bst_to_utc
 
@@ -150,52 +151,61 @@ def fetch_billing_data_for_day(process_day, service_id=None):
     table = Notification
     if start_date < datetime.utcnow() - timedelta(days=7):
         table = NotificationHistory
+    transit_data = []
+    for notification_type in (SMS_TYPE, EMAIL_TYPE, LETTER_TYPE):
+        billable_type_list = {
+            SMS_TYPE: NOTIFICATION_STATUS_TYPES_BILLABLE,
+            EMAIL_TYPE: NOTIFICATION_STATUS_TYPES_BILLABLE,
+            LETTER_TYPE: NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS
+        }
+        query = db.session.query(
+            table.template_id,
+            table.service_id,
+            table.notification_type,
+            func.coalesce(table.sent_by,
+                          case(
+                              [
+                                  (table.notification_type == 'letter', 'dvla'),
+                                  (table.notification_type == 'sms', 'unknown'),
+                                  (table.notification_type == 'email', 'ses')
+                              ]),
+                          ).label('sent_by'),
+            func.coalesce(table.rate_multiplier, 1).cast(Integer).label('rate_multiplier'),
+            func.coalesce(table.international, False).label('international'),
+            case(
+                [
+                    (table.notification_type == 'letter', table.billable_units),
+                ]
+            ).label('letter_page_count'),
+            func.sum(table.billable_units).label('billable_units'),
+            func.count().label('notifications_sent'),
+            Service.crown,
+            func.coalesce(table.postage, 'none').label('postage')
+        ).filter(
+            table.status.in_(billable_type_list[notification_type]),
+            table.key_type != KEY_TYPE_TEST,
+            table.created_at >= start_date,
+            table.created_at < end_date,
+            table.notification_type == notification_type
+        ).group_by(
+            table.template_id,
+            table.service_id,
+            table.notification_type,
+            'sent_by',
+            'letter_page_count',
+            table.rate_multiplier,
+            table.international,
+            Service.crown,
+            table.postage,
+        ).join(
+            Service
+        )
+        if service_id:
+            query = query.filter(table.service_id == service_id)
 
-    transit_data = db.session.query(
+        transit_data = transit_data + query.all()
-        table.template_id,
-        table.service_id,
-        table.notification_type,
-        func.coalesce(table.sent_by,
-                      case(
-                          [
-                              (table.notification_type == 'letter', 'dvla'),
-                              (table.notification_type == 'sms', 'unknown'),
-                              (table.notification_type == 'email', 'ses')
-                          ]),
-                      ).label('sent_by'),
-        func.coalesce(table.rate_multiplier, 1).cast(Integer).label('rate_multiplier'),
-        func.coalesce(table.international, False).label('international'),
-        case(
-            [
-                (table.notification_type == 'letter', table.billable_units),
-            ]
-        ).label('letter_page_count'),
-        func.sum(table.billable_units).label('billable_units'),
-        func.count().label('notifications_sent'),
-        Service.crown,
-        func.coalesce(table.postage, 'none').label('postage')
-    ).filter(
-        table.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE),
-        table.key_type != KEY_TYPE_TEST,
-        table.created_at >= start_date,
-        table.created_at < end_date
-    ).group_by(
-        table.template_id,
-        table.service_id,
-        table.notification_type,
-        'sent_by',
-        'letter_page_count',
-        table.rate_multiplier,
-        table.international,
-        Service.crown,
-        table.postage,
-    ).join(
-        Service
-    )
-    if service_id:
-        transit_data = transit_data.filter(table.service_id == service_id)
 
-    return transit_data.all()
+    return transit_data
 
 
 def get_rates_for_billing():

app/models.py

@@ -1113,6 +1113,12 @@ NOTIFICATION_STATUS_SUCCESS = [
     NOTIFICATION_DELIVERED
 ]
 
+NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS = [
+    NOTIFICATION_SENDING,
+    NOTIFICATION_DELIVERED,
+    NOTIFICATION_RETURNED_LETTER,
+]
+
 NOTIFICATION_STATUS_TYPES_BILLABLE = [
     NOTIFICATION_SENDING,
     NOTIFICATION_SENT,

tests/app/dao/test_ft_billing_dao.py

@@ -15,7 +15,11 @@ from app.dao.fact_billing_dao import (
     get_rate,
     get_rates_for_billing,
 )
-from app.models import FactBilling, Notification
+from app.models import (
+    FactBilling,
+    Notification,
+    NOTIFICATION_STATUS_TYPES,
+)
 from app.utils import convert_utc_to_bst
 from tests.app.db import (
     create_ft_billing,
@@ -253,6 +257,26 @@ def test_fetch_billing_data_for_day_returns_list_for_given_service(notify_db_ses
     assert results[0].service_id == service.id
 
 
+def test_fetch_billing_data_for_day_bills_correctly_for_status(notify_db_session):
+    service = create_service()
+    sms_template = create_template(service=service, template_type='sms')
+    email_template = create_template(service=service, template_type='email')
+    letter_template = create_template(service=service, template_type='letter')
+    for status in NOTIFICATION_STATUS_TYPES:
+        create_notification(template=sms_template, status=status)
+        create_notification(template=email_template, status=status)
+        create_notification(template=letter_template, status=status)
+    today = convert_utc_to_bst(datetime.utcnow())
+    results = fetch_billing_data_for_day(process_day=today, service_id=service.id)
+
+    sms_results = [x for x in results if x[2] == 'sms']
+    email_results = [x for x in results if x[2] == 'email']
+    letter_results = [x for x in results if x[2] == 'letter']
+    assert 7 == sms_results[0][7]
+    assert 7 == email_results[0][7]
+    assert 3 == letter_results[0][7]
+
+
 def test_get_rates_for_billing(notify_db_session):
     create_rate(start_date=datetime.utcnow(), value=12, notification_type='email')
     create_rate(start_date=datetime.utcnow(), value=22, notification_type='sms')