calculate free allowance used within ft_billing_subquery

previously we were looking at the chargeable units from within the
subquery, and then subtracting those from the free allowance to get the
free allowance remaining (sms_remainder).

this is fine for the org and service usage reports, which query for an
entire financial year.

however, the platform admin report is used for smaller periods,
generally monthly. this has the problem when we're querying, eg, may 1st
to 30th. Lets say a service has a free allowance of 10000 and sends 4000
messages in april and 5000 in may. we should be reporting their sms
remainder as 1000. However, when joining to the sms billing subquery we
want to filter out rows not in may, so that we can report on their usage
that month only. So we put in the bst_date filter - this means that the
"sms_billable_units", "chargeable_sms", and "sms_cost" only report on
that month. That's good. But remainder we were just looking at
chargeable sms and subtracting from the free allowance, ignoring however
much had been sent in april.

To solve this, move the free allowance remainder calc into the subquery
(which runs on the entire financial year, so has context about april's
usage as well). Essentially the chargeable_units_used_before_this_row,
plus this row.

app/dao/fact_billing_dao.py

@@ -34,84 +34,52 @@ from app.models import (
 from app.utils import get_london_midnight_in_utc
 
 
-def fetch_sms_free_allowance_remainder_until_date(end_date):
-    # ASSUMPTION: AnnualBilling has been populated for year.
-    billing_year = get_financial_year_for_datetime(end_date)
-    start_of_year = date(billing_year, 4, 1)
-
-    billable_units = func.coalesce(func.sum(FactBilling.billable_units * FactBilling.rate_multiplier), 0)
-
-    query = db.session.query(
-        AnnualBilling.service_id.label("service_id"),
-        AnnualBilling.free_sms_fragment_limit,
-        billable_units.label('billable_units'),
-        func.greatest((AnnualBilling.free_sms_fragment_limit - billable_units).cast(Integer), 0).label('sms_remainder')
-    ).outerjoin(
-        # if there are no ft_billing rows for a service we still want to return the annual billing so we can use the
-        # free_sms_fragment_limit)
-        FactBilling, and_(
-            AnnualBilling.service_id == FactBilling.service_id,
-            FactBilling.bst_date >= start_of_year,
-            FactBilling.bst_date < end_date,
-            FactBilling.notification_type == SMS_TYPE,
-        )
-    ).filter(
-        AnnualBilling.financial_year_start == billing_year,
-    ).group_by(
-        AnnualBilling.service_id,
-        AnnualBilling.free_sms_fragment_limit,
-    )
-    return query
-
-
 def fetch_sms_billing_for_all_services(start_date, end_date):
+    # ASSUMPTION: start and end date are in the same financial year
+    financial_year = get_financial_year_for_datetime(get_london_midnight_in_utc(start_date))
 
     # ASSUMPTION: AnnualBilling has been populated for year.
-    allowance_left_at_start_date_query = fetch_sms_free_allowance_remainder_until_date(start_date).subquery()
+    ft_billing_subquery = query_sms_usage_for_year_per_service(financial_year).subquery()
 
-    sms_billable_units = func.sum(FactBilling.billable_units * FactBilling.rate_multiplier)
+    sms_billable_units = func.sum(func.coalesce(ft_billing_subquery.c.chargeable_units, 0))
 
-    # subtract sms_billable_units units accrued since report's start date to get up-to-date
-    # allowance remainder
-    sms_allowance_left = func.greatest(allowance_left_at_start_date_query.c.sms_remainder - sms_billable_units, 0)
-
-    # billable units here are for period between start date and end date only, so to see
-    # how many are chargeable, we need to see how much free allowance was used up in the
-    # period up until report's start date and then do a subtraction
-    chargeable_sms = func.greatest(sms_billable_units - allowance_left_at_start_date_query.c.sms_remainder, 0)
-    sms_cost = chargeable_sms * FactBilling.rate
+    # get the lowest value allowance (which will be the last date within our filter range)
+    sms_allowance_left = func.greatest(
+        func.min(AnnualBilling.free_sms_fragment_limit - ft_billing_subquery.c.free_allowance_used_to_date),
+        0
+    )
 
+    chargeable_sms = func.sum(ft_billing_subquery.c.charged_units)
+    sms_cost = func.sum(ft_billing_subquery.c.cost)
     query = db.session.query(
         Organisation.name.label('organisation_name'),
         Organisation.id.label('organisation_id'),
         Service.name.label("service_name"),
         Service.id.label("service_id"),
-        allowance_left_at_start_date_query.c.free_sms_fragment_limit,
-        FactBilling.rate.label('sms_rate'),
-        sms_allowance_left.label("sms_remainder"),
-        sms_billable_units.label('sms_billable_units'),
-        chargeable_sms.label("chargeable_billable_sms"),
-        sms_cost.label('sms_cost'),
+        AnnualBilling.free_sms_fragment_limit,
+        func.coalesce(sms_allowance_left, 0).label("sms_remainder"),
+        func.coalesce(sms_billable_units, 0).label('sms_billable_units'),
+        func.coalesce(chargeable_sms, 0).label("chargeable_billable_sms"),
+        func.coalesce(sms_cost, 0).label('sms_cost'),
     ).select_from(
         Service
-    ).outerjoin(
-        allowance_left_at_start_date_query, Service.id == allowance_left_at_start_date_query.c.service_id
     ).outerjoin(
         Service.organisation
-    ).join(
-        FactBilling, FactBilling.service_id == Service.id,
+    ).outerjoin(
+        AnnualBilling,
+        and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == financial_year)
+    ).outerjoin(
+        ft_billing_subquery, Service.id == ft_billing_subquery.c.service_id
     ).filter(
-        FactBilling.bst_date >= start_date,
-        FactBilling.bst_date <= end_date,
-        FactBilling.notification_type == SMS_TYPE,
+        Service.restricted.is_(False),
+        ft_billing_subquery.c.bst_date >= start_date,
+        ft_billing_subquery.c.bst_date <= end_date,
     ).group_by(
         Organisation.name,
         Organisation.id,
         Service.id,
         Service.name,
-        allowance_left_at_start_date_query.c.free_sms_fragment_limit,
-        allowance_left_at_start_date_query.c.sms_remainder,
-        FactBilling.rate,
+        AnnualBilling.free_sms_fragment_limit
     ).order_by(
         Organisation.name,
         Service.name
@@ -762,6 +730,12 @@ def fetch_sms_billing_for_organisation(organisation_id, financial_year):
 
 
 def query_organisation_sms_usage_for_year(organisation_id, year):
+    return query_sms_usage_for_year_per_service(year).filter(
+        Service.organisation_id == organisation_id
+    )
+
+
+def query_sms_usage_for_year_per_service(year):
     """
     See docstring for query_service_sms_usage_for_year()
     """
@@ -795,12 +769,15 @@ def query_organisation_sms_usage_for_year(organisation_id, year):
     # for, after taking any remaining free allowance into account.
     charged_units = func.greatest(this_rows_chargeable_units - remaining_free_allowance_before_this_row, 0)
 
+    free_allowance_used_to_date = chargeable_units_used_before_this_row + this_rows_chargeable_units
+
     return db.session.query(
         Service.id.label('service_id'),
         FactBilling.bst_date,
         this_rows_chargeable_units.label("chargeable_units"),
         (charged_units * FactBilling.rate).label("cost"),
         charged_units.label("charged_units"),
+        free_allowance_used_to_date.label("free_allowance_used_to_date"),
     ).join(
         AnnualBilling,
         AnnualBilling.service_id == Service.id
@@ -813,7 +790,6 @@ def query_organisation_sms_usage_for_year(organisation_id, year):
             FactBilling.notification_type == SMS_TYPE,
         )
     ).filter(
-        Service.organisation_id == organisation_id,
         AnnualBilling.financial_year_start == year,
     )
 

app/platform_stats/rest.py

@@ -66,7 +66,6 @@ def validate_date_range_is_within_a_financial_year(start_date, end_date):
     return start_date, end_date
 
 
-@platform_stats_blueprint.route('usage-for-all-services')
 @platform_stats_blueprint.route('data-for-billing-report')
 def get_data_for_billing_report():
     start_date = request.args.get('start_date')

tests/app/dao/test_fact_billing_dao.py

@@ -16,12 +16,12 @@ from app.dao.fact_billing_dao import (
     fetch_letter_line_items_for_all_services,
     fetch_monthly_billing_for_year,
     fetch_sms_billing_for_all_services,
-    fetch_sms_free_allowance_remainder_until_date,
     fetch_usage_year_for_organisation,
     fetch_volumes_by_service,
     get_rate,
     get_rates_for_billing,
     query_organisation_sms_usage_for_year,
+    query_sms_usage_for_year_per_service,
 )
 from app.dao.organisation_dao import dao_add_service_to_organisation
 from app.models import NOTIFICATION_STATUS_TYPES, FactBilling
@@ -635,32 +635,6 @@ def test_delete_billing_data(notify_db_session):
     )
 
 
-def test_fetch_sms_free_allowance_remainder_until_date_with_two_services(notify_db_session):
-    service = create_service(service_name='has free allowance')
-    template = create_template(service=service)
-    org = create_organisation(name="Org for {}".format(service.name))
-    dao_add_service_to_organisation(service=service, organisation_id=org.id)
-    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2016)
-    create_ft_billing(template=template, bst_date=datetime(2016, 4, 20), billable_unit=2, rate=0.11)
-    create_ft_billing(template=template, bst_date=datetime(2016, 5, 20), billable_unit=3, rate=0.11)
-
-    service_2 = create_service(service_name='used free allowance')
-    template_2 = create_template(service=service_2)
-    org_2 = create_organisation(name="Org for {}".format(service_2.name))
-    dao_add_service_to_organisation(service=service_2, organisation_id=org_2.id)
-    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=20, financial_year_start=2016)
-    create_ft_billing(template=template_2, bst_date=datetime(2016, 4, 20), billable_unit=12, rate=0.11)
-    create_ft_billing(template=template_2, bst_date=datetime(2016, 4, 22), billable_unit=10, rate=0.11)
-    create_ft_billing(template=template_2, bst_date=datetime(2016, 5, 20), billable_unit=3, rate=0.11)
-
-    results = fetch_sms_free_allowance_remainder_until_date(datetime(2016, 5, 1)).all()
-    assert len(results) == 2
-    service_result = [row for row in results if row[0] == service.id]
-    assert service_result[0] == (service.id, 10, 2, 8)
-    service_2_result = [row for row in results if row[0] == service_2.id]
-    assert service_2_result[0] == (service_2.id, 20, 22, 0)
-
-
 def test_fetch_sms_billing_for_all_services_for_first_quarter(notify_db_session):
     # This test is useful because the inner query resultset is empty.
     service = create_service(service_name='a - has free allowance')
@@ -671,8 +645,15 @@ def test_fetch_sms_billing_for_all_services_for_first_quarter(notify_db_session)
     create_ft_billing(template=template, bst_date=datetime(2019, 4, 20), billable_unit=44, rate=0.11)
     results = fetch_sms_billing_for_all_services(datetime(2019, 4, 1), datetime(2019, 5, 30))
     assert len(results) == 1
-    assert results[0] == (org.name, org.id, service.name, service.id, 25000, Decimal('0.11'), 24956, 44, 0,
-                          Decimal('0'))
+    assert results[0].organisation_id == org.id
+    assert results[0].organisation_name == org.name
+    assert results[0].service_id == service.id
+    assert results[0].service_name == service.name
+    assert results[0].chargeable_billable_sms == 0
+    assert results[0].sms_billable_units == 44
+    assert results[0].free_sms_fragment_limit == 25000
+    assert results[0].sms_remainder == 24956
+    assert results[0].sms_cost == 0
 
 
 def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
@@ -701,6 +682,7 @@ def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
     create_ft_billing(template=template_3, bst_date=datetime(2019, 4, 20), billable_unit=5, rate=0.11)
     create_ft_billing(template=template_3, bst_date=datetime(2019, 5, 20), billable_unit=7, rate=0.11)
 
+    # this isn't included in results as it doesn't have any SMS rows
     service_4 = create_service(service_name='d - email only')
     email_template = create_template(service=service_4, template_type='email')
     org_4 = create_organisation(name="Org for {}".format(service_4.name))
@@ -717,21 +699,21 @@ def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
         # the requested report's start date.
         {
             "organisation_name": org.name, "organisation_id": org.id, "service_name": service_1.name,
-            "service_id": service_1.id, "free_sms_fragment_limit": 10, "sms_rate": Decimal('0.11'), "sms_remainder": 5,
+            "service_id": service_1.id, "free_sms_fragment_limit": 10, "sms_remainder": 5,
             "sms_billable_units": 3, "chargeable_billable_sms": 0, "sms_cost": Decimal('0.00')
         },
         # sms remainder is 0, because this service sent SMS worth 15 billable units, 12 of which were sent
         # before requested report's start date
         {
             "organisation_name": org_2.name, "organisation_id": org_2.id, "service_name": service_2.name,
-            "service_id": service_2.id, "free_sms_fragment_limit": 10, "sms_rate": Decimal('0.11'), "sms_remainder": 0,
+            "service_id": service_2.id, "free_sms_fragment_limit": 10, "sms_remainder": 0,
             "sms_billable_units": 3, "chargeable_billable_sms": 3, "sms_cost": Decimal('0.33')
         },
         # sms remainder is 0, because this service sent SMS worth 12 billable units, 5 of which were sent
         # before requested report's start date
         {
             "organisation_name": org_3.name, "organisation_id": org_3.id, "service_name": service_3.name,
-            "service_id": service_3.id, "free_sms_fragment_limit": 10, "sms_rate": Decimal('0.11'), "sms_remainder": 0,
+            "service_id": service_3.id, "free_sms_fragment_limit": 10, "sms_remainder": 0,
             "sms_billable_units": 7, "chargeable_billable_sms": 2, "sms_cost": Decimal('0.22')
         },
     ]
@@ -751,7 +733,7 @@ def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(noti
             "organisation_name": fixtures["org_1"].name, "organisation_id": fixtures["org_1"].id,
             "service_name": fixtures["service_1_sms_and_letter"].name,
             "service_id": fixtures["service_1_sms_and_letter"].id,
-            "free_sms_fragment_limit": 10, "sms_rate": Decimal('0.11'), "sms_remainder": 5,
+            "free_sms_fragment_limit": 10, "sms_remainder": 5,
             "sms_billable_units": 3, "chargeable_billable_sms": 0, "sms_cost": Decimal('0.00')
         },
         # sms remainder is 0, because this service sent SMS worth 15 billable units, 12 of which were sent
@@ -760,14 +742,14 @@ def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(noti
             "organisation_name": None, "organisation_id": None,
             "service_name": fixtures["service_with_sms_without_org"].name,
             "service_id": fixtures["service_with_sms_without_org"].id, "free_sms_fragment_limit": 10,
-            "sms_rate": Decimal('0.11'), "sms_remainder": 0,
+            "sms_remainder": 0,
             "sms_billable_units": 3, "chargeable_billable_sms": 3, "sms_cost": Decimal('0.33')
         },
         {
             "organisation_name": None, "organisation_id": None,
             "service_name": fixtures["service_with_sms_within_allowance"].name,
             "service_id": fixtures["service_with_sms_within_allowance"].id, "free_sms_fragment_limit": 10,
-            "sms_rate": Decimal('0.11'), "sms_remainder": 8,
+            "sms_remainder": 8,
             "sms_billable_units": 2, "chargeable_billable_sms": 0, "sms_cost": Decimal('0.00')
         },
     ]
@@ -1026,15 +1008,12 @@ def test_fetch_usage_year_for_organisation_only_returns_data_for_live_services(n
 
 
 @freeze_time('2022-04-27 13:30')
-def test_query_organisation_sms_usage_for_year_handles_multiple_services(notify_db_session):
+def test_query_sms_usage_for_year_per_service_handles_multiple_services(notify_db_session):
     today = datetime.utcnow().date()
     yesterday = datetime.utcnow().date() - timedelta(days=1)
     current_year = datetime.utcnow().year
 
-    org = create_organisation(name='Organisation 1')
-
     service_1 = create_service(restricted=False, service_name="Service 1")
-    dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
     sms_template_1 = create_template(service=service_1)
     create_ft_billing(
         bst_date=yesterday, template=sms_template_1, rate=1,
@@ -1047,7 +1026,6 @@ def test_query_organisation_sms_usage_for_year_handles_multiple_services(notify_
     create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=5, financial_year_start=current_year)
 
     service_2 = create_service(restricted=False, service_name="Service 2")
-    dao_add_service_to_organisation(service=service_2, organisation_id=org.id)
     sms_template_2 = create_template(service=service_2)
     create_ft_billing(
         bst_date=yesterday, template=sms_template_2, rate=1,
@@ -1061,7 +1039,7 @@ def test_query_organisation_sms_usage_for_year_handles_multiple_services(notify_
 
     # ----------
 
-    result = query_organisation_sms_usage_for_year(org.id, 2022).all()
+    result = query_sms_usage_for_year_per_service(2022).all()
 
     service_1_rows = [row for row in result if row.service_id == service_1.id]
     service_2_rows = [row for row in result if row.service_id == service_2.id]
@@ -1095,15 +1073,12 @@ def test_query_organisation_sms_usage_for_year_handles_multiple_services(notify_
 
 
 @freeze_time('2022-05-01 13:30')
-def test_query_organisation_sms_usage_for_year_handles_multiple_rates(notify_db_session):
+def test_query_sms_usage_for_year_per_service_handles_multiple_rates(notify_db_session):
     old_rate_date = date(2022, 4, 29)
     new_rate_date = date(2022, 5, 1)
     current_year = datetime.utcnow().year
 
-    org = create_organisation(name='Organisation 1')
-
     service_1 = create_service(restricted=False, service_name="Service 1")
-    dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
     sms_template_1 = create_template(service=service_1)
     create_ft_billing(
         bst_date=old_rate_date, template=sms_template_1, rate=2,
@@ -1115,7 +1090,7 @@ def test_query_organisation_sms_usage_for_year_handles_multiple_rates(notify_db_
     )
     create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=3, financial_year_start=current_year)
 
-    result = query_organisation_sms_usage_for_year(org.id, 2022).all()
+    result = query_sms_usage_for_year_per_service(2022).all()
 
     # al lthe free allowance is used on the first day
     assert result[0]['bst_date'] == date(2022, 4, 29)
@@ -1127,6 +1102,37 @@ def test_query_organisation_sms_usage_for_year_handles_multiple_rates(notify_db_
     assert result[1]['cost'] == 6
 
 
+@freeze_time('2022-05-01 13:30')
+def test_query_organisation_sms_usage_for_year_filters_on_organisation(notify_db_session):
+    bst_date = date(2022, 5, 1)
+    current_year = datetime.utcnow().year
+
+    service_1 = create_service(restricted=False, service_name="Service 1")
+    service_2 = create_service(restricted=False, service_name="Service 2")
+
+    org_1 = create_organisation(name="Org 1")
+    org_2 = create_organisation(name="Org 2")
+
+    dao_add_service_to_organisation(service=service_1, organisation_id=org_1.id)
+    dao_add_service_to_organisation(service=service_2, organisation_id=org_2.id)
+
+    sms_template_1 = create_template(service=service_1)
+    sms_template_2 = create_template(service=service_2)
+
+    create_ft_billing(bst_date=bst_date, template=sms_template_1, billable_unit=2)
+    create_ft_billing(bst_date=bst_date, template=sms_template_2, billable_unit=4)
+
+    create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=0, financial_year_start=current_year)
+    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=0, financial_year_start=current_year)
+
+    result = query_organisation_sms_usage_for_year(org_1.id, 2022).all()
+
+    assert len(result) == 1
+    assert result[0]['bst_date'] == date(2022, 5, 1)
+    assert result[0]['charged_units'] == 2
+    assert result[0]['service_id'] == service_1.id
+
+
 def test_fetch_daily_volumes_for_platform(
         notify_db_session, sample_template, sample_email_template, sample_letter_template
 ):