Merge pull request #3529 from alphagov/multiple_rates_sql_magic_for_org_view

Show correct sms cost in org view if sms rates change within financial year
This commit is contained in:
Pea Tyczynska
2022-05-19 12:09:42 +01:00
committed by GitHub
4 changed files with 305 additions and 53 deletions

View File

@@ -378,14 +378,14 @@ def query_service_sms_usage_for_year(service_id, year):
free allowance if it happens to run out when a rate changes.
"""
year_start, year_end = get_financial_year_dates(year)
chargeable_units = FactBilling.billable_units * FactBilling.rate_multiplier
this_rows_chargeable_units = FactBilling.billable_units * FactBilling.rate_multiplier
# Subquery for the number of chargeable units in all rows preceding this one,
# which might be none if this is the first row (hence the "coalesce"). For
# some reason the end result is a decimal despite all the input columns being
# integer - this seems to be a Sqlalchemy quirk (works in raw SQL).
cumulative_chargeable_units = func.coalesce(
func.sum(chargeable_units).over(
chargeable_units_used_before_this_row = func.coalesce(
func.sum(this_rows_chargeable_units).over(
# order is "ASC" by default
order_by=[FactBilling.bst_date],
# first row to previous row
@@ -396,22 +396,22 @@ def query_service_sms_usage_for_year(service_id, year):
# Subquery for how much free allowance we have left before the current row,
# so we can work out the cost for this row after taking it into account.
cumulative_free_remainder = func.greatest(
AnnualBilling.free_sms_fragment_limit - cumulative_chargeable_units,
remaining_free_allowance_before_this_row = func.greatest(
AnnualBilling.free_sms_fragment_limit - chargeable_units_used_before_this_row,
0
)
# Subquery for the number of chargeable_units that we will actually charge
# for, after taking any remaining free allowance into account.
charged_units = func.greatest(chargeable_units - cumulative_free_remainder, 0)
charged_units = func.greatest(this_rows_chargeable_units - remaining_free_allowance_before_this_row, 0)
free_allowance_used = func.least(cumulative_free_remainder, chargeable_units)
free_allowance_used = func.least(remaining_free_allowance_before_this_row, this_rows_chargeable_units)
return db.session.query(
FactBilling.bst_date,
FactBilling.postage, # should always be "none"
FactBilling.notifications_sent,
chargeable_units.label("chargeable_units"),
this_rows_chargeable_units.label("chargeable_units"),
FactBilling.rate,
FactBilling.notification_type,
(charged_units * FactBilling.rate).label("cost"),
@@ -720,52 +720,42 @@ def fetch_email_usage_for_organisation(organisation_id, start_date, end_date):
return query.all()
def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date):
def fetch_sms_billing_for_organisation(organisation_id, financial_year):
# ASSUMPTION: AnnualBilling has been populated for year.
allowance_left_at_start_date_query = fetch_sms_free_allowance_remainder_until_date(start_date).subquery()
ft_billing_subquery = query_organisation_sms_usage_for_year(organisation_id, financial_year).subquery()
sms_billable_units = func.coalesce(func.sum(FactBilling.billable_units * FactBilling.rate_multiplier), 0)
sms_billable_units = func.sum(func.coalesce(ft_billing_subquery.c.chargeable_units, 0))
# subtract sms_billable_units units accrued since report's start date to get up-to-date
# allowance remainder
sms_allowance_left = func.greatest(allowance_left_at_start_date_query.c.sms_remainder - sms_billable_units, 0)
sms_allowance_left = func.greatest(AnnualBilling.free_sms_fragment_limit - sms_billable_units, 0)
# billable units here are for period between start date and end date only, so to see
# how many are chargeable, we need to see how much free allowance was used up in the
# period up until report's start date and then do a subtraction
chargeable_sms = func.greatest(sms_billable_units - allowance_left_at_start_date_query.c.sms_remainder, 0)
sms_cost = chargeable_sms * FactBilling.rate
chargeable_sms = func.sum(ft_billing_subquery.c.charged_units)
sms_cost = func.sum(ft_billing_subquery.c.cost)
query = db.session.query(
Service.name.label("service_name"),
Service.id.label("service_id"),
func.coalesce(allowance_left_at_start_date_query.c.free_sms_fragment_limit, 0).label('free_sms_fragment_limit'),
func.coalesce(FactBilling.rate, 0).label('sms_rate'),
AnnualBilling.free_sms_fragment_limit,
func.coalesce(sms_allowance_left, 0).label("sms_remainder"),
func.coalesce(sms_billable_units, 0).label('sms_billable_units'),
func.coalesce(chargeable_sms, 0).label("chargeable_billable_sms"),
func.coalesce(sms_cost, 0).label('sms_cost'),
Service.active.label("active")
Service.active
).select_from(
Service
).outerjoin(
allowance_left_at_start_date_query, Service.id == allowance_left_at_start_date_query.c.service_id
AnnualBilling,
and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == financial_year)
).outerjoin(
FactBilling, and_(
Service.id == FactBilling.service_id,
FactBilling.bst_date >= start_date,
FactBilling.bst_date < end_date,
FactBilling.notification_type == SMS_TYPE,
)
ft_billing_subquery, Service.id == ft_billing_subquery.c.service_id
).filter(
Service.organisation_id == organisation_id,
Service.restricted.is_(False)
).group_by(
Service.id,
Service.name,
allowance_left_at_start_date_query.c.free_sms_fragment_limit,
allowance_left_at_start_date_query.c.sms_remainder,
FactBilling.rate,
AnnualBilling.free_sms_fragment_limit
).order_by(
Service.name
)
@@ -773,6 +763,63 @@ def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date):
return query.all()
def query_organisation_sms_usage_for_year(organisation_id, year):
"""
See docstring for query_service_sms_usage_for_year()
"""
year_start, year_end = get_financial_year_dates(year)
this_rows_chargeable_units = FactBilling.billable_units * FactBilling.rate_multiplier
# Subquery for the number of chargeable units in all rows preceding this one,
# which might be none if this is the first row (hence the "coalesce").
chargeable_units_used_before_this_row = func.coalesce(
func.sum(this_rows_chargeable_units).over(
# order is "ASC" by default
order_by=[FactBilling.bst_date],
# partition by service id
partition_by=FactBilling.service_id,
# first row to previous row
rows=(None, -1)
).cast(Integer),
0
)
# Subquery for how much free allowance we have left before the current row,
# so we can work out the cost for this row after taking it into account.
remaining_free_allowance_before_this_row = func.greatest(
AnnualBilling.free_sms_fragment_limit - chargeable_units_used_before_this_row,
0
)
# Subquery for the number of chargeable_units that we will actually charge
# for, after taking any remaining free allowance into account.
charged_units = func.greatest(this_rows_chargeable_units - remaining_free_allowance_before_this_row, 0)
return db.session.query(
Service.id.label('service_id'),
FactBilling.bst_date,
this_rows_chargeable_units.label("chargeable_units"),
(charged_units * FactBilling.rate).label("cost"),
charged_units.label("charged_units"),
).join(
AnnualBilling,
AnnualBilling.service_id == Service.id
).outerjoin(
FactBilling,
and_(
Service.id == FactBilling.service_id,
FactBilling.bst_date >= year_start,
FactBilling.bst_date <= year_end,
FactBilling.notification_type == SMS_TYPE,
)
).filter(
Service.organisation_id == organisation_id,
AnnualBilling.financial_year_start == year,
)
def fetch_usage_year_for_organisation(organisation_id, year):
year_start, year_end = get_financial_year_dates(year)
today = convert_utc_to_bst(datetime.utcnow()).date()
@@ -799,7 +846,7 @@ def fetch_usage_year_for_organisation(organisation_id, year):
'emails_sent': 0,
'active': service.active
}
sms_usages = fetch_sms_billing_for_organisation(organisation_id, year_start, year_end)
sms_usages = fetch_sms_billing_for_organisation(organisation_id, year)
letter_usages = fetch_letter_costs_for_organisation(organisation_id, year_start, year_end)
email_usages = fetch_email_usage_for_organisation(organisation_id, year_start, year_end)
for usage in sms_usages:

View File

@@ -21,6 +21,7 @@ from app.dao.fact_billing_dao import (
fetch_volumes_by_service,
get_rate,
get_rates_for_billing,
query_organisation_sms_usage_for_year,
)
from app.dao.organisation_dao import dao_add_service_to_organisation
from app.models import NOTIFICATION_STATUS_TYPES, FactBilling
@@ -844,6 +845,7 @@ def test_fetch_letter_line_items_for_all_service(notify_db_session):
def test_fetch_usage_year_for_organisation(notify_db_session):
fixtures = set_up_usage_data(datetime(2019, 5, 1))
service_with_emails_for_org = create_service(service_name='Service with emails for org')
create_annual_billing(service_with_emails_for_org.id, free_sms_fragment_limit=0, financial_year_start=2019)
dao_add_service_to_organisation(
service=service_with_emails_for_org,
organisation_id=fixtures["org_1"].id
@@ -905,6 +907,95 @@ def test_fetch_usage_year_for_organisation_populates_ft_billing_for_today(notify
assert FactBilling.query.count() == 1
@freeze_time('2022-05-01 13:30')
def test_fetch_usage_year_for_organisation_calculates_cost_from_multiple_rates(notify_db_session):
old_rate_date = date(2022, 4, 29)
new_rate_date = date(2022, 5, 1)
current_year = datetime.utcnow().year
org = create_organisation(name='Organisation 1')
service_1 = create_service(restricted=False, service_name="Service 1")
dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
sms_template_1 = create_template(service=service_1)
create_ft_billing(
bst_date=old_rate_date, template=sms_template_1, rate=2,
billable_unit=4, notifications_sent=4
)
create_ft_billing(
bst_date=new_rate_date, template=sms_template_1, rate=3,
billable_unit=2, notifications_sent=2
)
create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=3, financial_year_start=current_year)
results = fetch_usage_year_for_organisation(organisation_id=org.id, year=current_year)
assert len(results) == 1
assert results[str(service_1.id)]['free_sms_limit'] == 3
assert results[str(service_1.id)]['sms_remainder'] == 0
assert results[str(service_1.id)]['sms_billable_units'] == 6
assert results[str(service_1.id)]['chargeable_billable_sms'] == 3
assert results[str(service_1.id)]['sms_cost'] == 8.0
@freeze_time('2022-05-01 13:30')
def test_fetch_usage_year_for_organisation_when_no_usage(notify_db_session):
current_year = datetime.utcnow().year
org = create_organisation(name='Organisation 1')
service_1 = create_service(restricted=False, service_name="Service 1")
dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=3, financial_year_start=current_year)
results = fetch_usage_year_for_organisation(organisation_id=org.id, year=current_year)
assert len(results) == 1
assert results[str(service_1.id)]['free_sms_limit'] == 3
assert results[str(service_1.id)]['sms_remainder'] == 3
assert results[str(service_1.id)]['sms_billable_units'] == 0
assert results[str(service_1.id)]['chargeable_billable_sms'] == 0
assert results[str(service_1.id)]['sms_cost'] == 0.0
@freeze_time('2022-05-01 13:30')
def test_fetch_usage_year_for_organisation_only_queries_present_year(notify_db_session):
current_year = datetime.utcnow().year
last_year = current_year - 1
date_two_years_ago = date(2021, 3, 31)
date_in_last_financial_year = date(2022, 3, 31)
date_in_this_year = date.today()
org = create_organisation(name='Organisation 1')
service_1 = create_service(restricted=False, service_name="Service 1")
dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
sms_template_1 = create_template(service=service_1)
create_ft_billing(
bst_date=date_two_years_ago, template=sms_template_1, rate=1,
billable_unit=2, notifications_sent=2
)
create_ft_billing(
bst_date=date_in_last_financial_year, template=sms_template_1, rate=1,
billable_unit=4, notifications_sent=4
)
create_ft_billing(
bst_date=date_in_this_year, template=sms_template_1, rate=1,
billable_unit=8, notifications_sent=8
)
create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=4, financial_year_start=last_year - 1)
create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=0, financial_year_start=last_year)
create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=8, financial_year_start=current_year)
results = fetch_usage_year_for_organisation(organisation_id=org.id, year=last_year)
assert len(results) == 1
assert results[str(service_1.id)]['sms_billable_units'] == 4
assert results[str(service_1.id)]['chargeable_billable_sms'] == 4
assert results[str(service_1.id)]['sms_cost'] == 4.0
@freeze_time('2020-02-27 13:30')
def test_fetch_usage_year_for_organisation_only_returns_data_for_live_services(notify_db_session):
org = create_organisation(name='Organisation without live services')
@@ -924,6 +1015,8 @@ def test_fetch_usage_year_for_organisation_only_returns_data_for_live_services(n
notifications_sent=100)
create_ft_billing(bst_date=datetime.utcnow().date(), template=trial_letter_template, billable_unit=40, rate=0.30,
notifications_sent=20)
create_annual_billing(service_id=live_service.id, free_sms_fragment_limit=0, financial_year_start=2019)
create_annual_billing(service_id=trial_service.id, free_sms_fragment_limit=0, financial_year_start=2019)
results = fetch_usage_year_for_organisation(organisation_id=org.id, year=2019)
@@ -932,6 +1025,108 @@ def test_fetch_usage_year_for_organisation_only_returns_data_for_live_services(n
assert results[str(live_service.id)]['emails_sent'] == 0
@freeze_time('2022-04-27 13:30')
def test_query_organisation_sms_usage_for_year_handles_multiple_services(notify_db_session):
today = datetime.utcnow().date()
yesterday = datetime.utcnow().date() - timedelta(days=1)
current_year = datetime.utcnow().year
org = create_organisation(name='Organisation 1')
service_1 = create_service(restricted=False, service_name="Service 1")
dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
sms_template_1 = create_template(service=service_1)
create_ft_billing(
bst_date=yesterday, template=sms_template_1, rate=1,
billable_unit=4, notifications_sent=4
)
create_ft_billing(
bst_date=today, template=sms_template_1, rate=1,
billable_unit=2, notifications_sent=2
)
create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=5, financial_year_start=current_year)
service_2 = create_service(restricted=False, service_name="Service 2")
dao_add_service_to_organisation(service=service_2, organisation_id=org.id)
sms_template_2 = create_template(service=service_2)
create_ft_billing(
bst_date=yesterday, template=sms_template_2, rate=1,
billable_unit=16, notifications_sent=16
)
create_ft_billing(
bst_date=today, template=sms_template_2, rate=1,
billable_unit=8, notifications_sent=8
)
create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=current_year)
# ----------
result = query_organisation_sms_usage_for_year(org.id, 2022).all()
service_1_rows = [row for row in result if row.service_id == service_1.id]
service_2_rows = [row for row in result if row.service_id == service_2.id]
assert len(service_1_rows) == 2
assert len(service_2_rows) == 2
# service 1 has allowance of 5
# four fragments in total, all are used
assert service_1_rows[0]['bst_date'] == date(2022, 4, 26)
assert service_1_rows[0]['chargeable_units'] == 4
assert service_1_rows[0]['charged_units'] == 0
# two in total - one is free, one is charged
assert service_1_rows[1]['bst_date'] == date(2022, 4, 27)
assert service_1_rows[1]['chargeable_units'] == 2
assert service_1_rows[1]['charged_units'] == 1
# service 2 has allowance of 10
# sixteen fragments total, allowance is used and six are charged
assert service_2_rows[0]['bst_date'] == date(2022, 4, 26)
assert service_2_rows[0]['chargeable_units'] == 16
assert service_2_rows[0]['charged_units'] == 6
# eight fragments total, all are charged
assert service_2_rows[1]['bst_date'] == date(2022, 4, 27)
assert service_2_rows[1]['chargeable_units'] == 8
assert service_2_rows[1]['charged_units'] == 8
# assert total costs are accurate
assert float(sum(row.cost for row in service_1_rows)) == 1 # rows with 2 and 4, allowance of 5
assert float(sum(row.cost for row in service_2_rows)) == 14 # rows with 8 and 16, allowance of 10
@freeze_time('2022-05-01 13:30')
def test_query_organisation_sms_usage_for_year_handles_multiple_rates(notify_db_session):
old_rate_date = date(2022, 4, 29)
new_rate_date = date(2022, 5, 1)
current_year = datetime.utcnow().year
org = create_organisation(name='Organisation 1')
service_1 = create_service(restricted=False, service_name="Service 1")
dao_add_service_to_organisation(service=service_1, organisation_id=org.id)
sms_template_1 = create_template(service=service_1)
create_ft_billing(
bst_date=old_rate_date, template=sms_template_1, rate=2,
billable_unit=4, notifications_sent=4
)
create_ft_billing(
bst_date=new_rate_date, template=sms_template_1, rate=3,
billable_unit=2, notifications_sent=2
)
create_annual_billing(service_id=service_1.id, free_sms_fragment_limit=3, financial_year_start=current_year)
result = query_organisation_sms_usage_for_year(org.id, 2022).all()
# al lthe free allowance is used on the first day
assert result[0]['bst_date'] == date(2022, 4, 29)
assert result[0]['charged_units'] == 1
assert result[0]['cost'] == 2
assert result[1]['bst_date'] == date(2022, 5, 1)
assert result[1]['charged_units'] == 2
assert result[1]['cost'] == 6
def test_fetch_daily_volumes_for_platform(
notify_db_session, sample_template, sample_email_template, sample_letter_template
):
@@ -1055,7 +1250,8 @@ def test_fetch_volumes_by_service(notify_db_session):
results = fetch_volumes_by_service(start_date=datetime(2022, 2, 1), end_date=datetime(2022, 2, 28))
assert len(results) == 4
# since we are using a pre-set up fixture, we only care about some of the results
assert len(results) == 7
assert results[0].service_name == 'a - with sms and letter'
assert results[0].organisation_name == 'Org for a - with sms and letter'
assert results[0].free_allowance == 10
@@ -1076,22 +1272,22 @@ def test_fetch_volumes_by_service(notify_db_session):
assert results[1].letter_sheet_totals == 0
assert float(results[1].letter_cost) == 0
assert results[2].service_name == 'b - chargeable sms'
assert not results[2].organisation_name
assert results[2].free_allowance == 10
assert results[2].sms_notifications == 2
assert results[2].sms_chargeable_units == 3
assert results[2].email_totals == 0
assert results[2].letter_totals == 0
assert results[2].letter_sheet_totals == 0
assert float(results[2].letter_cost) == 0
assert results[4].service_name == 'b - chargeable sms'
assert not results[4].organisation_name
assert results[4].free_allowance == 10
assert results[4].sms_notifications == 2
assert results[4].sms_chargeable_units == 3
assert results[4].email_totals == 0
assert results[4].letter_totals == 0
assert results[4].letter_sheet_totals == 0
assert float(results[4].letter_cost) == 0
assert results[3].service_name == 'e - sms within allowance'
assert not results[3].organisation_name
assert results[3].free_allowance == 10
assert results[3].sms_notifications == 1
assert results[3].sms_chargeable_units == 2
assert results[3].email_totals == 0
assert results[3].letter_totals == 0
assert results[3].letter_sheet_totals == 0
assert float(results[3].letter_cost) == 0
assert results[6].service_name == 'e - sms within allowance'
assert not results[6].organisation_name
assert results[6].free_allowance == 10
assert results[6].sms_notifications == 1
assert results[6].sms_chargeable_units == 2
assert results[6].email_totals == 0
assert results[6].letter_totals == 0
assert results[6].letter_sheet_totals == 0
assert float(results[6].letter_cost) == 0

View File

@@ -998,6 +998,7 @@ def set_up_usage_data(start_date):
name='Org for {}'.format(service_with_emails.name),
)
dao_add_service_to_organisation(service=service_with_emails, organisation_id=org_2.id)
create_annual_billing(service_id=service_with_emails.id, free_sms_fragment_limit=0, financial_year_start=year)
create_ft_billing(bst_date=start_date, template=email_template, notifications_sent=10)
@@ -1012,6 +1013,7 @@ def set_up_usage_data(start_date):
billing_reference="org3 billing reference"
)
dao_add_service_to_organisation(service=service_with_letters, organisation_id=org_for_service_with_letters.id)
create_annual_billing(service_id=service_with_letters.id, free_sms_fragment_limit=0, financial_year_start=year)
create_ft_billing(bst_date=start_date, template=letter_template_3,
notifications_sent=2, billable_unit=3, rate=.50, postage='first')
@@ -1023,6 +1025,11 @@ def set_up_usage_data(start_date):
# service with letters, without an organisation:
service_with_letters_without_org = create_service(service_name='d - service without org')
letter_template_4 = create_template(service=service_with_letters_without_org, template_type='letter')
create_annual_billing(
service_id=service_with_letters_without_org.id,
free_sms_fragment_limit=0,
financial_year_start=year
)
create_ft_billing(bst_date=two_days_later, template=letter_template_4,
notifications_sent=7, billable_unit=4, rate=1.55, postage='rest-of-world')

View File

@@ -214,7 +214,9 @@ def test_volumes_by_service_report(
end_date='2022-03-01'
)
assert len(response) == 4
assert len(response) == 7
# since we are using a pre-set up fixture, we only care about some of the results
assert response[0] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0,
'letter_sheet_totals': 0, 'letter_totals': 0,
'organisation_id': str(fixture['org_1'].id),
@@ -228,12 +230,12 @@ def test_volumes_by_service_report(
'service_id': str(fixture['service_with_out_ft_billing_this_year'].id),
'service_name': fixture['service_with_out_ft_billing_this_year'].name,
'sms_chargeable_units': 0, 'sms_notifications': 0}
assert response[2] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
assert response[4] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
'letter_totals': 0, 'organisation_id': '', 'organisation_name': '',
'service_id': str(fixture['service_with_sms_without_org'].id),
'service_name': fixture['service_with_sms_without_org'].name,
'sms_chargeable_units': 0, 'sms_notifications': 0}
assert response[3] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
assert response[6] == {'email_totals': 0, 'free_allowance': 10, 'letter_cost': 0.0, 'letter_sheet_totals': 0,
'letter_totals': 0, 'organisation_id': '', 'organisation_name': '',
'service_id': str(fixture['service_with_sms_within_allowance'].id),
'service_name': fixture['service_with_sms_within_allowance'].name,