Add "free_chargeable_units" to service usage APIs

This represents the number of chargeable_units that were actually
free due to the free allowance - they won't be included in "cost".
Although the existing calculations in Admin [^1][^2] will still be
correct with a change in SMS rates - it's cost that's the problem
- it makes sense to have all the knowledge about calculating usage
consistently in these two APIs.

[^1]: 474d7dfda8/app/main/views/dashboard.py (L490)
[^2]: c63660d56d/app/main/views/dashboard.py (L350)

app/billing/billing_schemas.py

@@ -23,6 +23,7 @@ def serialize_ft_billing_remove_emails(rows):
             "rate": float(row.rate),
             "postage": row.postage,
             "cost": float(row.cost),
+            "free_chargeable_units": row.free_chargeable_units,
         }
         for row in rows
         if row.notification_type != 'email'
@@ -40,6 +41,7 @@ def serialize_ft_billing_yearly_totals(rows):
             # TEMPORARY: while we migrate to "cost" in the Admin app
             "letter_total": float(row.billable_units * row.rate) if row.notification_type == 'letter' else 0,
             "cost": float(row.cost),
+            "free_chargeable_units": row.free_chargeable_units,
         }
         for row in rows
     ]

app/dao/fact_billing_dao.py

@@ -207,6 +207,7 @@ def fetch_billing_totals_for_year(service_id, year):
                 query.c.rate.label("rate"),
                 query.c.notification_type.label("notification_type"),
                 func.sum(query.c.cost).label("cost"),
+                func.sum(query.c.free_chargeable_units).label("free_chargeable_units"),
             ).group_by(
                 query.c.rate,
                 query.c.notification_type
@@ -245,6 +246,7 @@ def fetch_monthly_billing_for_year(service_id, year):
                 query.c.postage.label("postage"),
                 query.c.notification_type.label("notification_type"),
                 func.sum(query.c.cost).label("cost"),
+                func.sum(query.c.free_chargeable_units).label("free_chargeable_units"),
             ).group_by(
                 query.c.rate,
                 query.c.notification_type,
@@ -274,6 +276,7 @@ def query_service_email_usage_for_year(service_id, year):
         FactBilling.notifications_sent.label("chargeable_units"),
         FactBilling.rate,
         FactBilling.notification_type,
+        literal(0).label("free_chargeable_units"),
         literal(0).label("cost"),
     ).filter(
         FactBilling.service_id == service_id,
@@ -293,6 +296,7 @@ def query_service_letter_usage_for_year(service_id, year):
         FactBilling.notifications_sent.label("chargeable_units"),
         FactBilling.rate,
         FactBilling.notification_type,
+        literal(0).label("free_chargeable_units"),
         (FactBilling.notifications_sent * FactBilling.rate).label("cost"),
     ).filter(
         FactBilling.service_id == service_id,
@@ -333,6 +337,10 @@ def query_service_sms_usage_for_year(service_id, year):
         chargeable_units.label("chargeable_units"),
         FactBilling.rate,
         FactBilling.notification_type,
+        func.least(
+            cumulative_free_remainder,
+            chargeable_units,
+        ).cast(Integer).label("free_chargeable_units"),  # for some reason the result is a Decimal
         (
             func.greatest(chargeable_units - cumulative_free_remainder, literal(0)) *
             FactBilling.rate

tests/app/billing/test_rest.py

@@ -176,6 +176,7 @@ def test_get_yearly_usage_by_monthly_from_ft_billing(admin_request, notify_db_se
     assert letter_row["rate"] == 0.33
     assert letter_row["postage"] == "second"
     assert letter_row["cost"] == 9.9
+    assert letter_row["free_chargeable_units"] == 0
 
     assert sms_row["month"] == "April"
     assert sms_row["notification_type"] == "sms"
@@ -185,6 +186,7 @@ def test_get_yearly_usage_by_monthly_from_ft_billing(admin_request, notify_db_se
     assert sms_row["postage"] == "none"
     # free allowance is 4, so (30 - 4) * 0.162
     assert sms_row["cost"] == 4.212
+    assert sms_row["free_chargeable_units"] == 4
 
 
 def set_up_yearly_data():
@@ -257,6 +259,7 @@ def test_get_yearly_billing_usage_summary_from_ft_billing(admin_request, notify_
     assert json_response[0]['rate'] == 0
     assert json_response[0]['letter_total'] == 0
     assert json_response[0]['cost'] == 0
+    assert json_response[0]['free_chargeable_units'] == 0
 
     assert json_response[1]['notification_type'] == 'letter'
     assert json_response[1]['billing_units'] == 275
@@ -264,6 +267,7 @@ def test_get_yearly_billing_usage_summary_from_ft_billing(admin_request, notify_
     assert json_response[1]['rate'] == 0.33
     assert json_response[1]['letter_total'] == 90.75
     assert json_response[1]['cost'] == 90.75
+    assert json_response[1]['free_chargeable_units'] == 0
 
     assert json_response[2]['notification_type'] == 'sms'
     assert json_response[2]['billing_units'] == 825
@@ -271,3 +275,4 @@ def test_get_yearly_billing_usage_summary_from_ft_billing(admin_request, notify_
     assert json_response[2]['rate'] == 0.0162
     assert json_response[2]['letter_total'] == 0
     assert json_response[2]['cost'] == 13.3002
+    assert json_response[2]['free_chargeable_units'] == 4

tests/app/dao/test_ft_billing_dao.py

@@ -429,6 +429,7 @@ def test_fetch_monthly_billing_for_year(notify_db_session):
     assert results[0].chargeable_units == 30
     assert results[0].rate == Decimal('0')
     assert results[0].cost == Decimal('0')
+    assert results[0].free_chargeable_units == 0
 
     assert str(results[1].month) == "2016-04-01"
     assert results[1].notification_type == 'letter'
@@ -437,6 +438,7 @@ def test_fetch_monthly_billing_for_year(notify_db_session):
     assert results[1].chargeable_units == 30
     assert results[1].rate == Decimal('0.30')
     assert results[1].cost == Decimal('9')
+    assert results[1].free_chargeable_units == 0
 
     assert str(results[1].month) == "2016-04-01"
     assert results[2].notification_type == 'letter'
@@ -445,6 +447,7 @@ def test_fetch_monthly_billing_for_year(notify_db_session):
     assert results[2].chargeable_units == 30
     assert results[2].rate == Decimal('0.33')
     assert results[2].cost == Decimal('9.9')
+    assert results[2].free_chargeable_units == 0
 
     assert str(results[3].month) == "2016-04-01"
     assert results[3].notification_type == 'sms'
@@ -454,6 +457,7 @@ def test_fetch_monthly_billing_for_year(notify_db_session):
     assert results[3].rate == Decimal('0.162')
     # free allowance is 10, so (30 - 10) * 0.162
     assert results[3].cost == Decimal('3.24')
+    assert results[3].free_chargeable_units == 10
 
     assert str(results[4].month) == "2016-05-01"
     assert str(results[47].month) == "2017-03-01"
@@ -474,6 +478,7 @@ def test_fetch_monthly_billing_for_year_variable_rates(notify_db_session):
     assert results[0].chargeable_units == 1
     assert results[0].rate == Decimal('0.33')
     assert results[0].cost == Decimal('0.33')
+    assert results[0].free_chargeable_units == 0
 
     assert str(results[1].month) == "2018-05-01"
     assert results[1].notification_type == 'letter'
@@ -482,6 +487,7 @@ def test_fetch_monthly_billing_for_year_variable_rates(notify_db_session):
     assert results[1].chargeable_units == 1
     assert results[1].rate == Decimal('0.36')
     assert results[1].cost == Decimal('0.36')
+    assert results[1].free_chargeable_units == 0
 
     assert str(results[2].month) == "2018-05-01"
     assert results[2].notification_type == 'sms'
@@ -491,6 +497,7 @@ def test_fetch_monthly_billing_for_year_variable_rates(notify_db_session):
     assert results[2].rate == Decimal('0.015')
     # 4 free units sent on the 16th, 0 on the 17th
     assert results[2].cost == Decimal('0')
+    assert results[2].free_chargeable_units == 4
 
     assert str(results[3].month) == "2018-05-01"
     assert results[3].notification_type == 'sms'
@@ -500,6 +507,7 @@ def test_fetch_monthly_billing_for_year_variable_rates(notify_db_session):
     assert results[3].rate == Decimal('0.162')
     # 1 free unit on the 16th, 1 on the 17th (+ 3 paid)
     assert results[3].cost == Decimal('0.486')
+    assert results[3].free_chargeable_units == 2
 
 
 @freeze_time('2018-08-01 13:30:00')
@@ -534,6 +542,7 @@ def test_fetch_billing_totals_for_year(notify_db_session):
     assert results[0].chargeable_units == 365
     assert results[0].rate == Decimal('0')
     assert results[0].cost == Decimal('0')
+    assert results[0].free_chargeable_units == 0
 
     assert results[1].notification_type == 'letter'
     assert results[1].notifications_sent == 365
@@ -541,6 +550,7 @@ def test_fetch_billing_totals_for_year(notify_db_session):
     assert results[1].chargeable_units == 365
     assert results[1].rate == Decimal('0.3')
     assert results[1].cost == Decimal('109.5')
+    assert results[1].free_chargeable_units == 0
 
     assert results[2].notification_type == 'letter'
     assert results[2].notifications_sent == 365
@@ -548,6 +558,7 @@ def test_fetch_billing_totals_for_year(notify_db_session):
     assert results[2].chargeable_units == 365
     assert results[2].rate == Decimal('0.33')
     assert results[2].cost == Decimal('120.45')
+    assert results[2].free_chargeable_units == 0
 
     assert results[3].notification_type == 'sms'
     assert results[3].notifications_sent == 365
@@ -555,6 +566,7 @@ def test_fetch_billing_totals_for_year(notify_db_session):
     assert results[3].chargeable_units == 365
     assert results[3].rate == Decimal('0.162')
     assert results[3].cost == Decimal('0')
+    assert results[3].free_chargeable_units == 365
 
 
 def test_fetch_billing_totals_for_year_uses_current_annual_billing(notify_db_session):
@@ -584,6 +596,7 @@ def test_fetch_billing_totals_for_year_variable_rates(notify_db_session):
     assert results[0].chargeable_units == 1
     assert results[0].rate == Decimal('0.33')
     assert results[0].cost == Decimal('0.33')
+    assert results[0].free_chargeable_units == 0
 
     assert results[1].notification_type == 'letter'
     assert results[1].notifications_sent == 1
@@ -591,6 +604,7 @@ def test_fetch_billing_totals_for_year_variable_rates(notify_db_session):
     assert results[1].chargeable_units == 1
     assert results[1].rate == Decimal('0.36')
     assert results[1].cost == Decimal('0.36')
+    assert results[1].free_chargeable_units == 0
 
     assert results[2].notification_type == 'sms'
     assert results[2].notifications_sent == 1
@@ -599,6 +613,7 @@ def test_fetch_billing_totals_for_year_variable_rates(notify_db_session):
     assert results[2].rate == Decimal('0.015')
     # 4 units sent on the 16th, 0 on the 17th
     assert results[2].cost == Decimal('0')
+    assert results[2].free_chargeable_units == 4
 
     assert results[3].notification_type == 'sms'
     assert results[3].notifications_sent == 2
@@ -607,6 +622,7 @@ def test_fetch_billing_totals_for_year_variable_rates(notify_db_session):
     assert results[3].rate == Decimal('0.162')
     # 1 free unit on the 16th, 1 on the 17th (+ 3 paid)
     assert results[3].cost == Decimal('0.486')  # (5 - 2) * 0.162
+    assert results[3].free_chargeable_units == 2
 
 
 def test_delete_billing_data(notify_db_session):