Add query_organisation_sms_usage_for_year to help fetch sms totals for org

This is functionally very similar to query_service_sms_usage_for_year,
except this query filters by organisation and returns for all live services
within that organisation.
To ensure that the cumulative free allowance counter resets properly for
each service, we use the `partition_by` flag to group up the window
function[^1]. This magically handles all the free allowances
independently for each service.

[^1]: https://www.postgresql.org/docs/current/tutorial-window.html

Co-authored-by: Leo Hemsted <leo.hemsted@digital.cabinet-office.gov.uk>

app/dao/fact_billing_dao.py

@@ -773,6 +773,63 @@ def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date):
     return query.all()
 
 
+def query_organisation_sms_usage_for_year(organisation_id, year):
+    """
+    See docstring for query_service_sms_usage_for_year()
+    """
+    year_start, year_end = get_financial_year_dates(year)
+    this_rows_chargeable_units = FactBilling.billable_units * FactBilling.rate_multiplier
+
+    # Subquery for the number of chargeable units in all rows preceding this one,
+    # which might be none if this is the first row (hence the "coalesce").
+    chargeable_units_used_so_far = func.coalesce(
+        func.sum(this_rows_chargeable_units).over(
+            # order is "ASC" by default
+            order_by=[FactBilling.bst_date],
+
+            # partition by service id
+            partition_by=FactBilling.service_id,
+
+            # first row to previous row
+            rows=(None, -1)
+        ).cast(Integer),
+        0
+    )
+
+    # Subquery for how much free allowance we have left before the current row,
+    # so we can work out the cost for this row after taking it into account.
+    remaining_free_allowance_before_this_row = func.greatest(
+        AnnualBilling.free_sms_fragment_limit - chargeable_units_used_so_far,
+        0
+    )
+
+    # Subquery for the number of chargeable_units that we will actually charge
+    # for, after taking any remaining free allowance into account.
+    charged_units = func.greatest(this_rows_chargeable_units - remaining_free_allowance_before_this_row, 0)
+
+    return db.session.query(
+        Service.id.label('service_id'),
+        FactBilling.bst_date,
+        this_rows_chargeable_units.label("chargeable_units"),
+        (charged_units * FactBilling.rate).label("cost"),
+        charged_units.label("charged_units"),
+    ).join(
+        AnnualBilling,
+        AnnualBilling.service_id == Service.id
+    ).outerjoin(
+        FactBilling,
+        and_(
+            Service.id == FactBilling.service_id,
+            FactBilling.bst_date >= year_start,
+            FactBilling.bst_date <= year_end,
+            FactBilling.notification_type == SMS_TYPE,
+        )
+    ).filter(
+        Service.organisation_id == organisation_id,
+        AnnualBilling.financial_year_start == year,
+    )
+
+
 def fetch_usage_year_for_organisation(organisation_id, year):
     year_start, year_end = get_financial_year_dates(year)
     today = convert_utc_to_bst(datetime.utcnow()).date()

tests/app/dao/test_fact_billing_dao.py

@@ -21,6 +21,7 @@ from app.dao.fact_billing_dao import (
     fetch_volumes_by_service,
     get_rate,
     get_rates_for_billing,
+    query_organisation_sms_usage_for_year
 )
 from app.dao.organisation_dao import dao_add_service_to_organisation
 from app.models import NOTIFICATION_STATUS_TYPES, FactBilling
@@ -932,6 +933,108 @@ def test_fetch_usage_year_for_organisation_only_returns_data_for_live_services(n
     assert results[str(live_service.id)]['emails_sent'] == 0
 
 
+@freeze_time('2022-04-27 13:30')
+def test_query_organisation_sms_usage_for_year_handles_multiple_services(notify_db_session):
+    today = datetime.utcnow().date()
+    yesterday = datetime.utcnow().date() - timedelta(days=1)
+    current_year = datetime.utcnow().year
+
+    org = create_organisation(name='Organisation 1')
+
+    service_1 = create_service(restricted=False, service_name="Service 1")
+    dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
+    sms_template_1 = create_template(service=service_1)
+    create_ft_billing(
+        bst_date=yesterday, template=sms_template_1, rate=1,
+        billable_unit=4, notifications_sent=4
+    )
+    create_ft_billing(
+        bst_date=today, template=sms_template_1, rate=1,
+        billable_unit=2, notifications_sent=2
+    )
+    create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=5, financial_year_start=current_year)
+
+    service_2 = create_service(restricted=False, service_name="Service 2")
+    dao_add_service_to_organisation(service=service_2, organisation_id=org.id)
+    sms_template_2 = create_template(service=service_2)
+    create_ft_billing(
+        bst_date=yesterday, template=sms_template_2, rate=1,
+        billable_unit=16, notifications_sent=16
+    )
+    create_ft_billing(
+        bst_date=today, template=sms_template_2, rate=1,
+        billable_unit=8, notifications_sent=8
+    )
+    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=current_year)
+
+    # ----------
+
+    result = query_organisation_sms_usage_for_year(org.id, 2022).all()
+
+    service_1_rows = [row for row in result if row.service_id == service_1.id]
+    service_2_rows = [row for row in result if row.service_id == service_2.id]
+
+    assert len(service_1_rows) == 2
+    assert len(service_2_rows) == 2
+
+    # service 1 has allowance of 5
+    # four fragments in total, all are used
+    assert service_1_rows[0]['bst_date'] == date(2022, 4, 26)
+    assert service_1_rows[0]['chargeable_units'] == 4
+    assert service_1_rows[0]['charged_units'] == 0
+    # two in total - one is free, one is charged
+    assert service_1_rows[1]['bst_date'] == date(2022, 4, 27)
+    assert service_1_rows[1]['chargeable_units'] == 2
+    assert service_1_rows[1]['charged_units'] == 1
+
+    # service 2 has allowance of 10
+    # sixteen fragments total, allowance is used and six are charged
+    assert service_2_rows[0]['bst_date'] == date(2022, 4, 26)
+    assert service_2_rows[0]['chargeable_units'] == 16
+    assert service_2_rows[0]['charged_units'] == 6
+    # eight fragments total, all are charged
+    assert service_2_rows[1]['bst_date'] == date(2022, 4, 27)
+    assert service_2_rows[1]['chargeable_units'] == 8
+    assert service_2_rows[1]['charged_units'] == 8
+
+    # assert total costs are accurate
+    assert float(sum(row.cost for row in service_1_rows)) == 1  # rows with 2 and 4, allowance of 5
+    assert float(sum(row.cost for row in service_2_rows)) == 14  # rows with 8 and 16, allowance of 10
+
+
+@freeze_time('2022-05-01 13:30')
+def test_query_organisation_sms_usage_for_year_handles_multiple_rates(notify_db_session):
+    old_rate_date = date(2022, 4, 29)
+    new_rate_date = date(2022, 5, 1)
+    current_year = datetime.utcnow().year
+
+    org = create_organisation(name='Organisation 1')
+
+    service_1 = create_service(restricted=False, service_name="Service 1")
+    dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
+    sms_template_1 = create_template(service=service_1)
+    create_ft_billing(
+        bst_date=old_rate_date, template=sms_template_1, rate=2,
+        billable_unit=4, notifications_sent=4
+    )
+    create_ft_billing(
+        bst_date=new_rate_date, template=sms_template_1, rate=3,
+        billable_unit=2, notifications_sent=2
+    )
+    create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=3, financial_year_start=current_year)
+
+    result = query_organisation_sms_usage_for_year(org.id, 2022).all()
+
+    # al lthe free allowance is used on the first day
+    assert result[0]['bst_date'] == date(2022, 4, 29)
+    assert result[0]['charged_units'] == 1
+    assert result[0]['cost'] == 2
+
+    assert result[1]['bst_date'] == date(2022, 5, 1)
+    assert result[1]['charged_units'] == 2
+    assert result[1]['cost'] == 6
+
+
 def test_fetch_daily_volumes_for_platform(
         notify_db_session, sample_template, sample_email_template, sample_letter_template
 ):