From 236d111c2299ad5ea4dfa411689dd8d3bc39739c Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Tue, 6 Aug 2019 13:29:59 +0100 Subject: [PATCH] Update queries for data model change (removal of organisation_to_service) New query to get letter breakdown. --- app/dao/date_util.py | 7 + app/dao/fact_billing_dao.py | 154 ++++++++++++++++++++- tests/app/dao/test_ft_billing_dao.py | 199 ++++++++++++++++++++++++++- tests/app/db.py | 3 +- 4 files changed, 358 insertions(+), 5 deletions(-) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 7a77f9562..77a46c69e 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -61,3 +61,10 @@ def get_current_financial_year_start_year(): if now < start_date: financial_year_start = financial_year_start - 1 return financial_year_start + + +def which_financial_year(start_date): + if start_date <= get_april_fools(int(start_date.strftime('%Y'))): + return int(start_date.strftime('%Y')) - 1 + else: + return int(start_date.strftime('%Y')) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index b27ac7ea1..587f2e792 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -6,7 +6,12 @@ from sqlalchemy.dialects.postgresql import insert from sqlalchemy import func, case, desc, Date, Integer from app import db -from app.dao.date_util import get_financial_year +from app.dao.date_util import ( + get_april_fools as financial_year_start, + get_financial_year, + which_financial_year +) + from app.models import ( FactBilling, Notification, @@ -19,11 +24,156 @@ from app.models import ( NOTIFICATION_STATUS_TYPES_BILLABLE, NotificationHistory, EMAIL_TYPE, - NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS + NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS, + AnnualBilling, + Organisation, + organisation_to_service ) from app.utils import get_london_midnight_in_utc +def fetch_sms_free_allowance_remainder(start_date): + # ASSUMPTION: AnnualBilling has been populated for year. + billing_year = which_financial_year(start_date) + start_of_year = convert_utc_to_bst(financial_year_start(billing_year)) + query = db.session.query( + FactBilling.service_id.label("service_id"), + AnnualBilling.free_sms_fragment_limit, + func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label('billable_units'), + func.greatest((AnnualBilling.free_sms_fragment_limit - + func.sum(FactBilling.billable_units * FactBilling.rate_multiplier) + ).cast(Integer), 0).label('sms_remainder') + ).filter( + FactBilling.service_id == Service.id, + FactBilling.service_id == AnnualBilling.service_id, + FactBilling.bst_date >= start_of_year, + FactBilling.bst_date < start_date, + FactBilling.notification_type == SMS_TYPE, + AnnualBilling.financial_year_start == billing_year, + ).group_by( + FactBilling.service_id, + AnnualBilling.free_sms_fragment_limit, + ) + return query + + +def fetch_sms_billing_for_all_services(start_date, end_date): + + # ASSUMPTION: AnnualBilling has been populated for year. + billing_year = which_financial_year(start_date) + free_allowance_remainder = fetch_sms_free_allowance_remainder(start_date).subquery() + sms_billable_units = func.sum(FactBilling.billable_units * FactBilling.rate_multiplier) + sms_remainder = func.coalesce(free_allowance_remainder.c.sms_remainder, 0) + chargeable_sms = case([(sms_remainder == 0, sms_billable_units), + (sms_billable_units - sms_remainder <= 0, 0), + (sms_billable_units - sms_remainder > 0, + sms_billable_units - sms_remainder)], else_=0) + sms_cost = case([(sms_remainder == 0, sms_billable_units * FactBilling.rate), + (sms_billable_units - sms_remainder <= 0, 0), + (sms_billable_units - sms_remainder > 0, (sms_billable_units - sms_remainder) * FactBilling.rate) + ], else_=0) + + query = db.session.query( + Organisation.name.label('organisation_name'), + Organisation.id.label('organisation_id'), + FactBilling.service_id.label("service_id"), + Service.name.label("service_name"), + AnnualBilling.free_sms_fragment_limit, + FactBilling.rate.label('sms_rate'), + sms_remainder.label("sms_remainder"), + sms_billable_units.label('sms_billable_units'), + chargeable_sms.label("chargeable_billable_sms"), + sms_cost.label('sms_cost'), + ).join( + Service.annual_billing, + ).outerjoin( + free_allowance_remainder, Service.id == free_allowance_remainder.c.service_id + ).outerjoin( + Organisation, Service.organisation_id == Organisation.id + ).filter( + FactBilling.service_id == Service.id, + FactBilling.bst_date >= start_date, + FactBilling.bst_date <= end_date, + FactBilling.notification_type == SMS_TYPE, + AnnualBilling.financial_year_start == billing_year, + ).group_by( + Organisation.name, + Organisation.id, + FactBilling.service_id, + Service.name, + AnnualBilling.free_sms_fragment_limit, + free_allowance_remainder.c.sms_remainder, + FactBilling.rate, + ).order_by( + Organisation.name, + Service.name + ) + + return query.all() + + +def fetch_letter_costs_for_all_services(start_date, end_date): + query = db.session.query( + Organisation.name.label("organisation_name"), + Organisation.id.label("organisation_id"), + FactBilling.service_id.label("service_id"), + Service.name.label("service_name"), + func.sum(FactBilling.notifications_sent * FactBilling.rate).label("letter_cost") + ).outerjoin( + Organisation, Service.organisation_id == Organisation.id + ).filter( + FactBilling.service_id == Service.id, + FactBilling.bst_date >= start_date, + FactBilling.bst_date <= end_date, + FactBilling.notification_type == LETTER_TYPE, + ).group_by( + Organisation.name, + Organisation.id, + FactBilling.service_id, + Service.name, + ).order_by( + Organisation.name, + Service.name + ) + + return query.all() + + +def fetch_letter_line_items_for_all_services(start_date, end_date): + query = db.session.query( + Organisation.name.label("organisation_name"), + Organisation.id.label("organisation_id"), + FactBilling.service_id.label("service_id"), + Service.name.label("service_name"), + FactBilling.billable_units.label('sheet_count'), + FactBilling.rate.label("letter_rate"), + FactBilling.postage('postage'), + func.sum(FactBilling.notifications_sent).label("letters_sent"), + ).outerjoin( + Organisation, Service.organisation_id == Organisation.id + ).filter( + FactBilling.service_id == Service.id, + FactBilling.bst_date >= start_date, + FactBilling.bst_date <= end_date, + FactBilling.notification_type == LETTER_TYPE, + ).group_by( + Organisation.name, + Organisation.id, + FactBilling.service_id, + Service.name, + FactBilling.billable_units, + FactBilling.rate, + FactBilling.postage + ).order_by( + Organisation.name, + Service.name, + FactBilling.billable_units, + FactBilling.postage + ) + + return query.all() + + def fetch_billing_totals_for_year(service_id, year): year_start_date, year_end_date = get_financial_year(year) """ diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index e5bd45a7b..b9b53a1d8 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -16,7 +16,10 @@ from app.dao.fact_billing_dao import ( fetch_monthly_billing_for_year, get_rate, get_rates_for_billing, -) + fetch_sms_free_allowance_remainder, + fetch_sms_billing_for_all_services, + fetch_letter_costs_for_all_services, fetch_letter_line_items_for_all_services) +from app.dao.organisation_dao import dao_add_service_to_organisation from app.models import ( FactBilling, Notification, @@ -29,7 +32,9 @@ from tests.app.db import ( create_notification, create_rate, create_letter_rate, - create_notification_history + create_notification_history, + create_annual_billing, + create_organisation ) @@ -478,3 +483,193 @@ def test_delete_billing_data(notify_db_session): assert sorted(x.billable_units for x in current_rows) == sorted( [other_day.billable_units, other_service.billable_units] ) + + +def test_fetch_sms_free_allowance_remainder_with_two_services(notify_db_session): + service = create_service(service_name='has free allowance') + template = create_template(service=service) + org = create_organisation(name="Org for {}".format(service.name)) + dao_add_service_to_organisation(service=service, organisation_id=org.id) + create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2016) + create_ft_billing(service=service, template=template, + bst_date=datetime(2016, 4, 20), notification_type='sms', billable_unit=2, rate=0.11) + create_ft_billing(service=service, template=template, bst_date=datetime(2016, 5, 20), notification_type='sms', + billable_unit=3, rate=0.11) + + service_2 = create_service(service_name='used free allowance') + template_2 = create_template(service=service_2) + org_2 = create_organisation(name="Org for {}".format(service_2.name)) + dao_add_service_to_organisation(service=service_2, organisation_id=org_2.id) + create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=20, financial_year_start=2016) + create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2016, 4, 20), notification_type='sms', + billable_unit=12, rate=0.11) + create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2016, 4, 22), notification_type='sms', + billable_unit=10, rate=0.11) + create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2016, 5, 20), notification_type='sms', + billable_unit=3, rate=0.11) + results = fetch_sms_free_allowance_remainder(datetime(2016, 5, 1)).all() + assert len(results) == 2 + service_result = [row for row in results if row[0] == service.id] + assert service_result[0] == (service.id, 10, 2, 8) + service_2_result = [row for row in results if row[0] == service_2.id] + assert service_2_result[0] == (service_2.id, 20, 22, 0) + + +def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session): + service = create_service(service_name='a - has free allowance') + template = create_template(service=service) + org = create_organisation(name="Org for {}".format(service.name)) + dao_add_service_to_organisation(service=service, organisation_id=org.id) + create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2019) + create_ft_billing(service=service, template=template, + bst_date=datetime(2019, 4, 20), notification_type='sms', billable_unit=2, rate=0.11) + create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 20), notification_type='sms', + billable_unit=2, rate=0.11) + create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 22), notification_type='sms', + billable_unit=1, rate=0.11) + + service_2 = create_service(service_name='b - used free allowance') + template_2 = create_template(service=service_2) + org_2 = create_organisation(name="Org for {}".format(service_2.name)) + dao_add_service_to_organisation(service=service_2, organisation_id=org_2.id) + create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=2019) + create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 4, 20), notification_type='sms', + billable_unit=12, rate=0.11) + create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 5, 20), notification_type='sms', + billable_unit=3, rate=0.11) + + service_3 = create_service(service_name='c - partial allowance') + template_3 = create_template(service=service_3) + org_3 = create_organisation(name="Org for {}".format(service_3.name)) + dao_add_service_to_organisation(service=service_3, organisation_id=org_3.id) + create_annual_billing(service_id=service_3.id, free_sms_fragment_limit=10, financial_year_start=2019) + create_ft_billing(service=service_3, template=template_3, bst_date=datetime(2019, 4, 20), notification_type='sms', + billable_unit=5, rate=0.11) + create_ft_billing(service=service_3, template=template_3, bst_date=datetime(2019, 5, 20), notification_type='sms', + billable_unit=7, rate=0.11) + + service_4 = create_service(service_name='d - email only') + email_template = create_template(service=service_4, template_type='email') + org_4 = create_organisation(name="Org for {}".format(service_4.name)) + dao_add_service_to_organisation(service=service_4, organisation_id=org_4.id) + create_annual_billing(service_id=service_4.id, free_sms_fragment_limit=10, financial_year_start=2019) + create_ft_billing(service=service_4, template=email_template, bst_date=datetime(2019, 5, 22), notifications_sent=5, + notification_type='email', billable_unit=0, rate=0) + + results = fetch_sms_billing_for_all_services(datetime(2019, 5, 1), datetime(2019, 5, 31)) + assert len(results) == 3 + + assert results[0].organisation_id == org.id + assert results[0].service_id == service.id + assert results[0].sms_billable_units == 3 + assert results[0].sms_remainder == 8 + assert results[0].chargeable_billable_sms == 0 + assert results[0].sms_cost == Decimal('0') + + assert results[1].organisation_id == org_2.id + assert results[1].service_id == service_2.id + assert results[1].sms_billable_units == 3 + assert results[1].sms_remainder == 0 + assert results[1].chargeable_billable_sms == 3 + assert results[1].sms_cost == Decimal('0.33') + + assert results[2].organisation_id == org_3.id + assert results[2].service_id == service_3.id + assert results[2].sms_billable_units == 7 + assert results[2].sms_remainder == 5 + assert results[2].chargeable_billable_sms == 2 + assert results[2].sms_cost == Decimal('0.22') + + +def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(notify_db_session): + service = create_service(service_name='a - has free allowance') + template = create_template(service=service) + org = create_organisation(name="Org for {}".format(service.name)) + service.organisation = org + create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2019) + create_ft_billing(service=service, template=template, + bst_date=datetime(2019, 4, 20), notification_type='sms', billable_unit=2, rate=0.11) + create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 20), notification_type='sms', + billable_unit=2, rate=0.11) + create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 22), notification_type='sms', + billable_unit=1, rate=0.11) + + service_2 = create_service(service_name='b - used free allowance') + template_2 = create_template(service=service_2) + + create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=2019) + create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 4, 20), notification_type='sms', + billable_unit=12, rate=0.11) + create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 5, 20), notification_type='sms', + billable_unit=3, rate=0.11) + results = fetch_sms_billing_for_all_services(datetime(2019, 5, 1), datetime(2019, 5, 31)) + assert len(results) == 2 + + assert results[0].organisation_name == org.name + assert results[0].service_id == service.id + assert results[0].sms_billable_units == 3 + assert results[0].sms_remainder == 8 + assert results[0].chargeable_billable_sms == 0 + assert results[0].sms_cost == Decimal('0') + + assert not results[1].organisation_name + assert results[1].service_id == service_2.id + assert results[1].sms_billable_units == 3 + assert results[1].sms_remainder == 0 + assert results[1].chargeable_billable_sms == 3 + assert results[1].sms_cost == Decimal('0.33') + + +def test_fetch_letter_costs_for_all_services(notify_db_session): + org, org_3, service, service_3 = set_up_letter_data() + + results = fetch_letter_costs_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30)) + + assert len(results) == 2 + assert results[0].organisation_id == org.id + assert results[0].organisation_name == org.name + assert results[0].service_id == service.id + assert results[0].service_name == service.name + assert results[0].letter_cost == Decimal('3.40') + + assert results[1].organisation_id == org_3.id + assert results[1].organisation_name == org_3.name + assert results[1].service_id == service_3.id + assert results[1].service_name == service_3.name + assert results[1].letter_cost == Decimal('6.20') + + +def test_fetch_letter_line_items_for_all_service(notify_db_session): + set_up_letter_data() + results = fetch_letter_line_items_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30)) + print(results) + assert len(results) == 4 + + +def set_up_letter_data(): + service = create_service(service_name='a - first service') + letter_template = create_template(service=service, template_type='letter') + org = create_organisation(name="Org for {}".format(service.name)) + dao_add_service_to_organisation(service=service, organisation_id=org.id) + service_2 = create_service(service_name='b - second service') + sms_template = create_template(service=service, template_type='sms') + dao_add_service_to_organisation(service=service_2, organisation_id=org.id) + service_3 = create_service(service_name='c - third service') + template_3 = create_template(service=service_3) + org_3 = create_organisation(name="Org for {}".format(service_3.name)) + dao_add_service_to_organisation(service=service_3, organisation_id=org_3.id) + create_ft_billing(bst_date=datetime(2019, 8, 12), service=service_2, notification_type='sms', + template=sms_template) + create_ft_billing(bst_date=datetime(2019, 8, 15), service=service, notification_type='letter', + template=letter_template, + notifications_sent=2, billable_unit=1, rate=.35, postage='first') + create_ft_billing(bst_date=datetime(2019, 8, 20), service=service, notification_type='letter', + template=letter_template, + notifications_sent=6, billable_unit=2, rate=.45, postage='second') + create_ft_billing(bst_date=datetime(2019, 9, 1), service=service_3, notification_type='letter', + template=template_3, + notifications_sent=2, billable_unit=3, rate=.50, postage='first') + create_ft_billing(bst_date=datetime(2019, 9, 20), service=service_3, notification_type='letter', + template=template_3, + notifications_sent=8, billable_unit=5, rate=.65, postage='second') + return org, org_3, service, service_3 diff --git a/tests/app/db.py b/tests/app/db.py index 9a6250769..641e4f320 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -108,7 +108,8 @@ def create_service( check_if_service_exists=False, go_live_user=None, go_live_at=None, - crown=True + crown=True, + organisation=None ): if check_if_service_exists: service = Service.query.filter_by(name=service_name).first()