Update queries for data model change (removal of organisation_to_service)

New query to get letter breakdown.

app/dao/date_util.py

@@ -61,3 +61,10 @@ def get_current_financial_year_start_year():
     if now < start_date:
         financial_year_start = financial_year_start - 1
     return financial_year_start
+
+
+def which_financial_year(start_date):
+    if start_date <= get_april_fools(int(start_date.strftime('%Y'))):
+        return int(start_date.strftime('%Y')) - 1
+    else:
+        return int(start_date.strftime('%Y'))

app/dao/fact_billing_dao.py

@@ -6,7 +6,12 @@ from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy import func, case, desc, Date, Integer
 
 from app import db
-from app.dao.date_util import get_financial_year
+from app.dao.date_util import (
+    get_april_fools as financial_year_start,
+    get_financial_year,
+    which_financial_year
+)
+
 from app.models import (
     FactBilling,
     Notification,
@@ -19,11 +24,156 @@ from app.models import (
     NOTIFICATION_STATUS_TYPES_BILLABLE,
     NotificationHistory,
     EMAIL_TYPE,
-    NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS
+    NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS,
+    AnnualBilling,
+    Organisation,
+    organisation_to_service
 )
 from app.utils import get_london_midnight_in_utc
 
 
+def fetch_sms_free_allowance_remainder(start_date):
+    # ASSUMPTION: AnnualBilling has been populated for year.
+    billing_year = which_financial_year(start_date)
+    start_of_year = convert_utc_to_bst(financial_year_start(billing_year))
+    query = db.session.query(
+        FactBilling.service_id.label("service_id"),
+        AnnualBilling.free_sms_fragment_limit,
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label('billable_units'),
+        func.greatest((AnnualBilling.free_sms_fragment_limit -
+                       func.sum(FactBilling.billable_units * FactBilling.rate_multiplier)
+                       ).cast(Integer), 0).label('sms_remainder')
+    ).filter(
+        FactBilling.service_id == Service.id,
+        FactBilling.service_id == AnnualBilling.service_id,
+        FactBilling.bst_date >= start_of_year,
+        FactBilling.bst_date < start_date,
+        FactBilling.notification_type == SMS_TYPE,
+        AnnualBilling.financial_year_start == billing_year,
+    ).group_by(
+        FactBilling.service_id,
+        AnnualBilling.free_sms_fragment_limit,
+    )
+    return query
+
+
+def fetch_sms_billing_for_all_services(start_date, end_date):
+
+    # ASSUMPTION: AnnualBilling has been populated for year.
+    billing_year = which_financial_year(start_date)
+    free_allowance_remainder = fetch_sms_free_allowance_remainder(start_date).subquery()
+    sms_billable_units = func.sum(FactBilling.billable_units * FactBilling.rate_multiplier)
+    sms_remainder = func.coalesce(free_allowance_remainder.c.sms_remainder, 0)
+    chargeable_sms = case([(sms_remainder == 0, sms_billable_units),
+                           (sms_billable_units - sms_remainder <= 0, 0),
+                           (sms_billable_units - sms_remainder > 0,
+                            sms_billable_units - sms_remainder)], else_=0)
+    sms_cost = case([(sms_remainder == 0, sms_billable_units * FactBilling.rate),
+                     (sms_billable_units - sms_remainder <= 0, 0),
+                     (sms_billable_units - sms_remainder > 0, (sms_billable_units - sms_remainder) * FactBilling.rate)
+                     ], else_=0)
+
+    query = db.session.query(
+        Organisation.name.label('organisation_name'),
+        Organisation.id.label('organisation_id'),
+        FactBilling.service_id.label("service_id"),
+        Service.name.label("service_name"),
+        AnnualBilling.free_sms_fragment_limit,
+        FactBilling.rate.label('sms_rate'),
+        sms_remainder.label("sms_remainder"),
+        sms_billable_units.label('sms_billable_units'),
+        chargeable_sms.label("chargeable_billable_sms"),
+        sms_cost.label('sms_cost'),
+    ).join(
+        Service.annual_billing,
+    ).outerjoin(
+        free_allowance_remainder, Service.id == free_allowance_remainder.c.service_id
+    ).outerjoin(
+        Organisation, Service.organisation_id == Organisation.id
+    ).filter(
+        FactBilling.service_id == Service.id,
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == SMS_TYPE,
+        AnnualBilling.financial_year_start == billing_year,
+    ).group_by(
+        Organisation.name,
+        Organisation.id,
+        FactBilling.service_id,
+        Service.name,
+        AnnualBilling.free_sms_fragment_limit,
+        free_allowance_remainder.c.sms_remainder,
+        FactBilling.rate,
+    ).order_by(
+        Organisation.name,
+        Service.name
+    )
+
+    return query.all()
+
+
+def fetch_letter_costs_for_all_services(start_date, end_date):
+    query = db.session.query(
+        Organisation.name.label("organisation_name"),
+        Organisation.id.label("organisation_id"),
+        FactBilling.service_id.label("service_id"),
+        Service.name.label("service_name"),
+        func.sum(FactBilling.notifications_sent * FactBilling.rate).label("letter_cost")
+    ).outerjoin(
+        Organisation, Service.organisation_id == Organisation.id
+    ).filter(
+        FactBilling.service_id == Service.id,
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == LETTER_TYPE,
+    ).group_by(
+        Organisation.name,
+        Organisation.id,
+        FactBilling.service_id,
+        Service.name,
+    ).order_by(
+        Organisation.name,
+        Service.name
+    )
+
+    return query.all()
+
+
+def fetch_letter_line_items_for_all_services(start_date, end_date):
+    query = db.session.query(
+        Organisation.name.label("organisation_name"),
+        Organisation.id.label("organisation_id"),
+        FactBilling.service_id.label("service_id"),
+        Service.name.label("service_name"),
+        FactBilling.billable_units.label('sheet_count'),
+        FactBilling.rate.label("letter_rate"),
+        FactBilling.postage('postage'),
+        func.sum(FactBilling.notifications_sent).label("letters_sent"),
+    ).outerjoin(
+        Organisation, Service.organisation_id == Organisation.id
+    ).filter(
+        FactBilling.service_id == Service.id,
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == LETTER_TYPE,
+    ).group_by(
+        Organisation.name,
+        Organisation.id,
+        FactBilling.service_id,
+        Service.name,
+        FactBilling.billable_units,
+        FactBilling.rate,
+        FactBilling.postage
+    ).order_by(
+        Organisation.name,
+        Service.name,
+        FactBilling.billable_units,
+        FactBilling.postage
+    )
+
+    return query.all()
+
+
 def fetch_billing_totals_for_year(service_id, year):
     year_start_date, year_end_date = get_financial_year(year)
     """

tests/app/dao/test_ft_billing_dao.py

@@ -16,7 +16,10 @@ from app.dao.fact_billing_dao import (
     fetch_monthly_billing_for_year,
     get_rate,
     get_rates_for_billing,
-)
+    fetch_sms_free_allowance_remainder,
+    fetch_sms_billing_for_all_services,
+    fetch_letter_costs_for_all_services, fetch_letter_line_items_for_all_services)
+from app.dao.organisation_dao import dao_add_service_to_organisation
 from app.models import (
     FactBilling,
     Notification,
@@ -29,7 +32,9 @@ from tests.app.db import (
     create_notification,
     create_rate,
     create_letter_rate,
-    create_notification_history
+    create_notification_history,
+    create_annual_billing,
+    create_organisation
 )
 
 
@@ -478,3 +483,193 @@ def test_delete_billing_data(notify_db_session):
     assert sorted(x.billable_units for x in current_rows) == sorted(
         [other_day.billable_units, other_service.billable_units]
     )
+
+
+def test_fetch_sms_free_allowance_remainder_with_two_services(notify_db_session):
+    service = create_service(service_name='has free allowance')
+    template = create_template(service=service)
+    org = create_organisation(name="Org for {}".format(service.name))
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2016)
+    create_ft_billing(service=service, template=template,
+                      bst_date=datetime(2016, 4, 20), notification_type='sms', billable_unit=2, rate=0.11)
+    create_ft_billing(service=service, template=template, bst_date=datetime(2016, 5, 20), notification_type='sms',
+                      billable_unit=3, rate=0.11)
+
+    service_2 = create_service(service_name='used free allowance')
+    template_2 = create_template(service=service_2)
+    org_2 = create_organisation(name="Org for {}".format(service_2.name))
+    dao_add_service_to_organisation(service=service_2, organisation_id=org_2.id)
+    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=20, financial_year_start=2016)
+    create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2016, 4, 20), notification_type='sms',
+                      billable_unit=12, rate=0.11)
+    create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2016, 4, 22), notification_type='sms',
+                      billable_unit=10, rate=0.11)
+    create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2016, 5, 20), notification_type='sms',
+                      billable_unit=3, rate=0.11)
+    results = fetch_sms_free_allowance_remainder(datetime(2016, 5, 1)).all()
+    assert len(results) == 2
+    service_result = [row for row in results if row[0] == service.id]
+    assert service_result[0] == (service.id, 10, 2, 8)
+    service_2_result = [row for row in results if row[0] == service_2.id]
+    assert service_2_result[0] == (service_2.id, 20, 22, 0)
+
+
+def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
+    service = create_service(service_name='a - has free allowance')
+    template = create_template(service=service)
+    org = create_organisation(name="Org for {}".format(service.name))
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2019)
+    create_ft_billing(service=service, template=template,
+                      bst_date=datetime(2019, 4, 20), notification_type='sms', billable_unit=2, rate=0.11)
+    create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 20), notification_type='sms',
+                      billable_unit=2, rate=0.11)
+    create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 22), notification_type='sms',
+                      billable_unit=1, rate=0.11)
+
+    service_2 = create_service(service_name='b - used free allowance')
+    template_2 = create_template(service=service_2)
+    org_2 = create_organisation(name="Org for {}".format(service_2.name))
+    dao_add_service_to_organisation(service=service_2, organisation_id=org_2.id)
+    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=2019)
+    create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 4, 20), notification_type='sms',
+                      billable_unit=12, rate=0.11)
+    create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 5, 20), notification_type='sms',
+                      billable_unit=3, rate=0.11)
+    
+    service_3 = create_service(service_name='c - partial allowance')
+    template_3 = create_template(service=service_3)
+    org_3 = create_organisation(name="Org for {}".format(service_3.name))
+    dao_add_service_to_organisation(service=service_3, organisation_id=org_3.id)
+    create_annual_billing(service_id=service_3.id, free_sms_fragment_limit=10, financial_year_start=2019)
+    create_ft_billing(service=service_3, template=template_3, bst_date=datetime(2019, 4, 20), notification_type='sms',
+                      billable_unit=5, rate=0.11)
+    create_ft_billing(service=service_3, template=template_3, bst_date=datetime(2019, 5, 20), notification_type='sms',
+                      billable_unit=7, rate=0.11)
+
+    service_4 = create_service(service_name='d - email only')
+    email_template = create_template(service=service_4, template_type='email')
+    org_4 = create_organisation(name="Org for {}".format(service_4.name))
+    dao_add_service_to_organisation(service=service_4, organisation_id=org_4.id)
+    create_annual_billing(service_id=service_4.id, free_sms_fragment_limit=10, financial_year_start=2019)
+    create_ft_billing(service=service_4, template=email_template, bst_date=datetime(2019, 5, 22), notifications_sent=5,
+                      notification_type='email', billable_unit=0, rate=0)
+
+    results = fetch_sms_billing_for_all_services(datetime(2019, 5, 1), datetime(2019, 5, 31))
+    assert len(results) == 3
+
+    assert results[0].organisation_id == org.id
+    assert results[0].service_id == service.id
+    assert results[0].sms_billable_units == 3
+    assert results[0].sms_remainder == 8
+    assert results[0].chargeable_billable_sms == 0
+    assert results[0].sms_cost == Decimal('0')
+
+    assert results[1].organisation_id == org_2.id
+    assert results[1].service_id == service_2.id
+    assert results[1].sms_billable_units == 3
+    assert results[1].sms_remainder == 0
+    assert results[1].chargeable_billable_sms == 3
+    assert results[1].sms_cost == Decimal('0.33')
+
+    assert results[2].organisation_id == org_3.id
+    assert results[2].service_id == service_3.id
+    assert results[2].sms_billable_units == 7
+    assert results[2].sms_remainder == 5
+    assert results[2].chargeable_billable_sms == 2
+    assert results[2].sms_cost == Decimal('0.22')
+
+
+def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(notify_db_session):
+    service = create_service(service_name='a - has free allowance')
+    template = create_template(service=service)
+    org = create_organisation(name="Org for {}".format(service.name))
+    service.organisation = org
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2019)
+    create_ft_billing(service=service, template=template,
+                      bst_date=datetime(2019, 4, 20), notification_type='sms', billable_unit=2, rate=0.11)
+    create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 20), notification_type='sms',
+                      billable_unit=2, rate=0.11)
+    create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 22), notification_type='sms',
+                      billable_unit=1, rate=0.11)
+
+    service_2 = create_service(service_name='b - used free allowance')
+    template_2 = create_template(service=service_2)
+
+    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=2019)
+    create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 4, 20), notification_type='sms',
+                      billable_unit=12, rate=0.11)
+    create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 5, 20), notification_type='sms',
+                      billable_unit=3, rate=0.11)
+    results = fetch_sms_billing_for_all_services(datetime(2019, 5, 1), datetime(2019, 5, 31))
+    assert len(results) == 2
+
+    assert results[0].organisation_name == org.name
+    assert results[0].service_id == service.id
+    assert results[0].sms_billable_units == 3
+    assert results[0].sms_remainder == 8
+    assert results[0].chargeable_billable_sms == 0
+    assert results[0].sms_cost == Decimal('0')
+
+    assert not results[1].organisation_name
+    assert results[1].service_id == service_2.id
+    assert results[1].sms_billable_units == 3
+    assert results[1].sms_remainder == 0
+    assert results[1].chargeable_billable_sms == 3
+    assert results[1].sms_cost == Decimal('0.33')
+
+
+def test_fetch_letter_costs_for_all_services(notify_db_session):
+    org, org_3, service, service_3 = set_up_letter_data()
+
+    results = fetch_letter_costs_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30))
+
+    assert len(results) == 2
+    assert results[0].organisation_id == org.id
+    assert results[0].organisation_name == org.name
+    assert results[0].service_id == service.id
+    assert results[0].service_name == service.name
+    assert results[0].letter_cost == Decimal('3.40')
+
+    assert results[1].organisation_id == org_3.id
+    assert results[1].organisation_name == org_3.name
+    assert results[1].service_id == service_3.id
+    assert results[1].service_name == service_3.name
+    assert results[1].letter_cost == Decimal('6.20')
+
+
+def test_fetch_letter_line_items_for_all_service(notify_db_session):
+    set_up_letter_data()
+    results = fetch_letter_line_items_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30))
+    print(results)
+    assert len(results) == 4
+
+
+def set_up_letter_data():
+    service = create_service(service_name='a - first service')
+    letter_template = create_template(service=service, template_type='letter')
+    org = create_organisation(name="Org for {}".format(service.name))
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+    service_2 = create_service(service_name='b - second service')
+    sms_template = create_template(service=service, template_type='sms')
+    dao_add_service_to_organisation(service=service_2, organisation_id=org.id)
+    service_3 = create_service(service_name='c - third service')
+    template_3 = create_template(service=service_3)
+    org_3 = create_organisation(name="Org for {}".format(service_3.name))
+    dao_add_service_to_organisation(service=service_3, organisation_id=org_3.id)
+    create_ft_billing(bst_date=datetime(2019, 8, 12), service=service_2, notification_type='sms',
+                      template=sms_template)
+    create_ft_billing(bst_date=datetime(2019, 8, 15), service=service, notification_type='letter',
+                      template=letter_template,
+                      notifications_sent=2, billable_unit=1, rate=.35, postage='first')
+    create_ft_billing(bst_date=datetime(2019, 8, 20), service=service, notification_type='letter',
+                      template=letter_template,
+                      notifications_sent=6, billable_unit=2, rate=.45, postage='second')
+    create_ft_billing(bst_date=datetime(2019, 9, 1), service=service_3, notification_type='letter',
+                      template=template_3,
+                      notifications_sent=2, billable_unit=3, rate=.50, postage='first')
+    create_ft_billing(bst_date=datetime(2019, 9, 20), service=service_3, notification_type='letter',
+                      template=template_3,
+                      notifications_sent=8, billable_unit=5, rate=.65, postage='second')
+    return org, org_3, service, service_3

tests/app/db.py

@@ -108,7 +108,8 @@ def create_service(
         check_if_service_exists=False,
         go_live_user=None,
         go_live_at=None,
-        crown=True
+        crown=True,
+        organisation=None
 ):
     if check_if_service_exists:
         service = Service.query.filter_by(name=service_name).first()