Add costs to each row in yearly usage API

This will replace the manual calculation in Admin [^1] for SMS and
also in API [^2] for letters.

Doing the calculation here also means we correctly attribute free
allowance to the earliest rows in the billing table - Admin doesn't
know when a given rate was applied so can't do this with the data
currently returned from the API.

Since the calculation now depends on annual billing, we need to
change all the tests to make sure a suitable row exists.

Note about "OVER" clause
========================

Using "rows=" ("ROWS BETWEEN") makes more sense than "range=" as
we want the remainder to be incremental within each group in a
"GROUP BY" clause, as well as between groups i.e

  # ROWS BETWEEN (arbitrary numbers to illustrate)
  date=2021-04-03, units=3, cost=3.29
  date=2021-04-03, units=2, cost=4.17
  date=2021-04-04, units=2, cost=5.10

  vs.

  # RANGE BETWEEN
  date=2021-04-03, units=3, cost=4.17
  date=2021-04-03, units=2, cost=4.17
  date=2021-04-04, units=2, cost=5.10

See [^3] for more details and examples.

[^1]: https://github.com/alphagov/notifications-admin/blob/master/app/templates/views/usage.html#L60
[^2]: 072c3b2079/app/billing/billing_schemas.py (L37)
[^3]: https://learnsql.com/blog/difference-between-rows-range-window-functions/

app/billing/billing_schemas.py

@@ -34,7 +34,9 @@ def serialize_ft_billing_yearly_totals(rows):
             "billing_units": row.billable_units,
             "chargeable_units": row.chargeable_units,
             "rate": float(row.rate),
+            # TEMPORARY: while we migrate to "cost" in the Admin app
             "letter_total": float(row.billable_units * row.rate) if row.notification_type == 'letter' else 0,
+            "cost": float(row.cost),
         }
         for row in rows
     ]

app/dao/fact_billing_dao.py

@@ -206,6 +206,7 @@ def fetch_billing_totals_for_year(service_id, year):
                 func.sum(query.c.chargeable_units).label("chargeable_units"),
                 query.c.rate.label("rate"),
                 query.c.notification_type.label("notification_type"),
+                func.sum(query.c.cost).label("cost"),
             ).group_by(
                 query.c.rate,
                 query.c.notification_type
@@ -287,6 +288,7 @@ def query_service_email_usage_for_year(service_id, year):
         FactBilling.notifications_sent.label("chargeable_units"),
         FactBilling.rate,
         FactBilling.notification_type,
+        literal(0).label("cost"),
     ).filter(
         FactBilling.service_id == service_id,
         FactBilling.bst_date >= year_start,
@@ -303,6 +305,7 @@ def query_service_letter_usage_for_year(service_id, year):
         FactBilling.notifications_sent.label("chargeable_units"),
         FactBilling.rate,
         FactBilling.notification_type,
+        (FactBilling.notifications_sent * FactBilling.rate).label("cost"),
     ).filter(
         FactBilling.service_id == service_id,
         FactBilling.bst_date >= year_start,
@@ -315,16 +318,44 @@ def query_service_sms_usage_for_year(service_id, year):
     year_start, year_end = get_financial_year_dates(year)
     chargeable_units = FactBilling.billable_units * FactBilling.rate_multiplier
 
+    # Subquery for the number of chargeable units in all rows preceding this one,
+    # which might be none if this is the first row (hence the "coalesce").
+    cumulative_chargeable_units = func.coalesce(
+        func.sum(chargeable_units).over(
+            order_by=[
+                FactBilling.bst_date,  # order is "ASC" by default
+                FactBilling.rate  # ensures test stability for rows on the same day
+            ],
+            rows=(None, -1)  # ROWS BETWEEN UNBOUNDED PRECEDING AND 1 ROW PRECEDING
+        ),
+        literal(0)
+    )
+
+    # Subquery for how much free allowance we have left before the current row,
+    # so we can work out the cost for this row after taking it into account.
+    cumulative_free_remainder = func.greatest(
+        AnnualBilling.free_sms_fragment_limit - cumulative_chargeable_units,
+        0
+    )
+
     return db.session.query(
         FactBilling.notifications_sent,
         chargeable_units.label("chargeable_units"),
         FactBilling.rate,
         FactBilling.notification_type,
+        (
+            func.greatest(chargeable_units - cumulative_free_remainder, literal(0)) *
+            FactBilling.rate
+        ).label("cost")
+    ).outerjoin(
+        AnnualBilling,
+        AnnualBilling.service_id == service_id
     ).filter(
         FactBilling.service_id == service_id,
         FactBilling.bst_date >= year_start,
         FactBilling.bst_date <= year_end,
-        FactBilling.notification_type == SMS_TYPE
+        FactBilling.notification_type == SMS_TYPE,
+        AnnualBilling.financial_year_start == year,
     )