From eb083e30edb0b78dee0daff9f5fa271c8c2f0f79 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 16 May 2018 12:21:59 +0100 Subject: [PATCH] - Only rebuild current month for monthly_billing if today is in the current year. - Change the usage queries to a union so that billing_units is correct for all notification types. Removing the business logic from the schema. - Added tests for different fragment counts, rates and sheet counts. --- app/billing/billing_schemas.py | 10 +- app/dao/fact_billing_dao.py | 76 ++++++++++--- app/dao/monthly_billing_dao.py | 6 +- tests/app/billing/test_billing.py | 147 +++++++++++++++++++++++++- tests/app/dao/test_ft_billing_dao.py | 5 - tests/app/dao/test_monthly_billing.py | 10 ++ 6 files changed, 224 insertions(+), 30 deletions(-) diff --git a/app/billing/billing_schemas.py b/app/billing/billing_schemas.py index 007a1cd49..fb41901d8 100644 --- a/app/billing/billing_schemas.py +++ b/app/billing/billing_schemas.py @@ -16,12 +16,12 @@ create_or_update_free_sms_fragment_limit_schema = { def serialize_ft_billing_remove_emails(data): results = [] billed_notifications = [x for x in data if x.notification_type != 'email'] - for notifications in billed_notifications: + for notification in billed_notifications: json_result = { - "month": (datetime.strftime(notifications.month, "%B")), - "notification_type": notifications.notification_type, - "billing_units": int(notifications.billable_units), - "rate": float(notifications.rate), + "month": (datetime.strftime(notification.month, "%B")), + "notification_type": notification.notification_type, + "billing_units": notification.billable_units, + "rate": float(notification.rate), } results.append(json_result) return results diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index 18387a25e..9a3a30148 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta, time +from flask import current_app from sqlalchemy.dialects.postgresql import insert from sqlalchemy import func, case, desc, Date @@ -16,31 +17,55 @@ from app.models import ( SMS_TYPE, Rate, LetterRate, - NotificationHistory + NotificationHistory, + EMAIL_TYPE ) from app.utils import convert_utc_to_bst, convert_bst_to_utc def fetch_billing_totals_for_year(service_id, year): year_start_date, year_end_date = get_financial_year(year) - - yearly_data = db.session.query( + """ + Billing for email: only record the total number of emails. + Billing for letters: The billing units is used to fetch the correct rate for the sheet count of the letter. + Total cost is notifications_sent * rate. + Rate multiplier does not apply to email or letters. + """ + email_and_letters = db.session.query( + func.sum(FactBilling.notifications_sent).label("notifications_sent"), + func.sum(FactBilling.notifications_sent).label("billable_units"), + FactBilling.rate.label('rate'), + FactBilling.notification_type.label('notification_type') + ).filter( + FactBilling.service_id == service_id, + FactBilling.bst_date >= year_start_date, + FactBilling.bst_date <= year_end_date, + FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE]) + ).group_by( + FactBilling.rate, + FactBilling.notification_type + ) + """ + Billing for SMS using the billing_units * rate_multiplier. Billing unit of SMS is the fragment count of a message + """ + sms = db.session.query( func.sum(FactBilling.notifications_sent).label("notifications_sent"), func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"), - FactBilling.service_id, FactBilling.rate, FactBilling.notification_type ).filter( FactBilling.service_id == service_id, FactBilling.bst_date >= year_start_date, - FactBilling.bst_date <= year_end_date + FactBilling.bst_date <= year_end_date, + FactBilling.notification_type == SMS_TYPE ).group_by( - FactBilling.service_id, FactBilling.rate, FactBilling.notification_type - ).order_by( - FactBilling.service_id, - FactBilling.notification_type + ) + + yearly_data = email_and_letters.union_all(sms).order_by( + 'notification_type', + 'rate' ).all() return yearly_data @@ -58,26 +83,44 @@ def fetch_monthly_billing_for_year(service_id, year): for d in data: update_fact_billing(data=d, process_day=day) - yearly_data = db.session.query( + email_and_letters = db.session.query( + func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"), + func.sum(FactBilling.notifications_sent).label("notifications_sent"), + func.sum(FactBilling.notifications_sent).label("billable_units"), + FactBilling.rate.label('rate'), + FactBilling.notification_type.label('notification_type') + ).filter( + FactBilling.service_id == service_id, + FactBilling.bst_date >= year_start_date, + FactBilling.bst_date <= year_end_date, + FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE]) + ).group_by( + 'month', + FactBilling.rate, + FactBilling.notification_type + ) + + sms = db.session.query( func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"), func.sum(FactBilling.notifications_sent).label("notifications_sent"), func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"), - FactBilling.service_id, FactBilling.rate, FactBilling.notification_type ).filter( FactBilling.service_id == service_id, FactBilling.bst_date >= year_start_date, - FactBilling.bst_date <= year_end_date + FactBilling.bst_date <= year_end_date, + FactBilling.notification_type == SMS_TYPE ).group_by( 'month', - FactBilling.service_id, FactBilling.rate, FactBilling.notification_type - ).order_by( - FactBilling.service_id, + ) + + yearly_data = email_and_letters.union_all(sms).order_by( 'month', - FactBilling.notification_type + 'notification_type', + 'rate' ).all() return yearly_data @@ -88,6 +131,7 @@ def fetch_billing_data_for_day(process_day, service_id=None): end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) # use notification_history if process day is older than 7 days # this is useful if we need to rebuild the ft_billing table for a date older than 7 days ago. + current_app.logger.info("Populate ft_billing for {} to {}".format(start_date, end_date)) table = Notification if start_date < datetime.utcnow() - timedelta(days=7): table = NotificationHistory diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py index 715cd164b..502df8fa5 100644 --- a/app/dao/monthly_billing_dao.py +++ b/app/dao/monthly_billing_dao.py @@ -116,11 +116,11 @@ def get_monthly_billing_by_notification_type(service_id, billing_month, notifica @statsd(namespace="dao") def get_billing_data_for_financial_year(service_id, year, notification_types=[SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]): - # Update totals to the latest so we include data for today now = convert_utc_to_bst(datetime.utcnow()) - create_or_update_monthly_billing(service_id=service_id, billing_month=now) - start_date, end_date = get_financial_year(year) + if start_date <= now <= end_date: + # Update totals to the latest so we include data for today + create_or_update_monthly_billing(service_id=service_id, billing_month=now) results = get_yearly_billing_data_for_date_range( service_id, start_date, end_date, notification_types diff --git a/tests/app/billing/test_billing.py b/tests/app/billing/test_billing.py index 43b49dfa2..f931edbde 100644 --- a/tests/app/billing/test_billing.py +++ b/tests/app/billing/test_billing.py @@ -522,6 +522,7 @@ def set_up_yearly_data(): service=service, template=email_template, notification_type='email', + billable_unit=0, rate=0) create_ft_billing(bst_date='2016-{}-{}'.format(mon, d), service=service, @@ -559,7 +560,7 @@ def set_up_yearly_data(): return service -def test_get_yearly_billing_usage_summary_from_ft_billing_comapre_to_monthyl_billing( +def test_get_yearly_billing_usage_summary_from_ft_billing_compare_to_monthly_billing( client, notify_db_session ): service = set_up_yearly_data() @@ -621,3 +622,147 @@ def test_get_yearly_billing_usage_summary_from_ft_billing(client, notify_db_sess assert json_response[2]['billing_units'] == 825 assert json_response[2]['rate'] == 0.0162 assert json_response[2]['letter_total'] == 0 + + +def test_get_yearly_usage_by_monthly_from_ft_billing_all_cases(client, notify_db_session): + service = set_up_data_for_all_cases() + response = client.get('service/{}/billing/ft-monthly-usage?year=2018'.format(service.id), + headers=[('Content-Type', 'application/json'), create_authorization_header()]) + + assert response.status_code == 200 + json_response = json.loads(response.get_data(as_text=True)) + assert len(json_response) == 5 + assert json_response[0]['month'] == 'May' + assert json_response[0]['notification_type'] == 'letter' + assert json_response[0]['rate'] == 0.33 + assert json_response[0]['billing_units'] == 1 + + assert json_response[1]['month'] == 'May' + assert json_response[1]['notification_type'] == 'letter' + assert json_response[1]['rate'] == 0.36 + assert json_response[1]['billing_units'] == 1 + + assert json_response[2]['month'] == 'May' + assert json_response[2]['notification_type'] == 'letter' + assert json_response[2]['rate'] == 0.39 + assert json_response[2]['billing_units'] == 1 + + assert json_response[3]['month'] == 'May' + assert json_response[3]['notification_type'] == 'sms' + assert json_response[3]['rate'] == 0.0150 + assert json_response[3]['billing_units'] == 4 + + assert json_response[4]['month'] == 'May' + assert json_response[4]['notification_type'] == 'sms' + assert json_response[4]['rate'] == 0.162 + assert json_response[4]['billing_units'] == 5 + + +def test_get_yearly_billing_usage_summary_from_ft_billing_all_cases(client, notify_db_session): + service = set_up_data_for_all_cases() + response = client.get('/service/{}/billing/ft-yearly-usage-summary?year=2018'.format(service.id), + headers=[create_authorization_header()]) + assert response.status_code == 200 + json_response = json.loads(response.get_data(as_text=True)) + + assert len(json_response) == 6 + assert json_response[0]["notification_type"] == 'email' + assert json_response[0]["billing_units"] == 1 + assert json_response[0]["rate"] == 0 + assert json_response[0]["letter_total"] == 0 + + assert json_response[1]["notification_type"] == 'letter' + assert json_response[1]["billing_units"] == 1 + assert json_response[1]["rate"] == 0.33 + assert json_response[1]["letter_total"] == 0.33 + + assert json_response[2]["notification_type"] == 'letter' + assert json_response[2]["billing_units"] == 1 + assert json_response[2]["rate"] == 0.36 + assert json_response[2]["letter_total"] == 0.36 + + assert json_response[3]["notification_type"] == 'letter' + assert json_response[3]["billing_units"] == 1 + assert json_response[3]["rate"] == 0.39 + assert json_response[3]["letter_total"] == 0.39 + + assert json_response[4]["notification_type"] == 'sms' + assert json_response[4]["billing_units"] == 4 + assert json_response[4]["rate"] == 0.0150 + assert json_response[4]["letter_total"] == 0 + + assert json_response[5]["notification_type"] == 'sms' + assert json_response[5]["billing_units"] == 5 + assert json_response[5]["rate"] == 0.162 + assert json_response[5]["letter_total"] == 0 + + +def set_up_data_for_all_cases(): + service = create_service() + sms_template = create_template(service=service, template_type="sms") + email_template = create_template(service=service, template_type="email") + letter_template = create_template(service=service, template_type="letter") + create_ft_billing(bst_date='2018-05-16', + notification_type='sms', + template=sms_template, + service=service, + rate_multiplier=1, + international=False, + rate=0.162, + billable_unit=1, + notifications_sent=1) + create_ft_billing(bst_date='2018-05-17', + notification_type='sms', + template=sms_template, + service=service, + rate_multiplier=2, + international=False, + rate=0.162, + billable_unit=2, + notifications_sent=1) + create_ft_billing(bst_date='2018-05-16', + notification_type='sms', + template=sms_template, + service=service, + rate_multiplier=2, + international=False, + rate=0.0150, + billable_unit=2, + notifications_sent=1) + create_ft_billing(bst_date='2018-05-16', + notification_type='email', + template=email_template, + service=service, + rate_multiplier=1, + international=False, + rate=0, + billable_unit=0, + notifications_sent=1) + create_ft_billing(bst_date='2018-05-16', + notification_type='letter', + template=letter_template, + service=service, + rate_multiplier=1, + international=False, + rate=0.33, + billable_unit=1, + notifications_sent=1) + create_ft_billing(bst_date='2018-05-17', + notification_type='letter', + template=letter_template, + service=service, + rate_multiplier=1, + international=False, + rate=0.36, + billable_unit=2, + notifications_sent=1) + create_ft_billing(bst_date='2018-05-18', + notification_type='letter', + template=letter_template, + service=service, + rate_multiplier=1, + international=False, + rate=0.39, + billable_unit=3, + notifications_sent=1) + return service diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index 09deb9266..7d64a43a0 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -266,14 +266,12 @@ def test_fetch_monthly_billing_for_year(notify_db_session): assert str(results[0].month) == "2018-06-01" assert results[0].notifications_sent == 30 assert results[0].billable_units == Decimal('60') - assert results[0].service_id == service.id assert results[0].rate == Decimal('0.162') assert results[0].notification_type == 'sms' assert str(results[1].month) == "2018-07-01" assert results[1].notifications_sent == 31 assert results[1].billable_units == Decimal('31') - assert results[1].service_id == service.id assert results[1].rate == Decimal('0.158') assert results[1].notification_type == 'sms' @@ -330,19 +328,16 @@ def test_fetch_billing_totals_for_year(notify_db_session): assert len(results) == 3 assert results[0].notification_type == 'email' - assert results[0].service_id == service.id assert results[0].notifications_sent == 365 assert results[0].billable_units == 365 assert results[0].rate == Decimal('0') assert results[1].notification_type == 'letter' - assert results[1].service_id == service.id assert results[1].notifications_sent == 365 assert results[1].billable_units == 365 assert results[1].rate == Decimal('0.33') assert results[2].notification_type == 'sms' - assert results[2].service_id == service.id assert results[2].notifications_sent == 365 assert results[2].billable_units == 365 assert results[2].rate == Decimal('0.162') diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py index e09814708..e41057501 100644 --- a/tests/app/dao/test_monthly_billing.py +++ b/tests/app/dao/test_monthly_billing.py @@ -511,3 +511,13 @@ def test_get_yearly_billing_data_for_year_includes_current_day_totals(sample_tem ) assert billing_data[0].monthly_totals[0]['billing_units'] == 3 + + +@freeze_time("2017-06-16 13:00:00") +def test_get_billing_data_for_financial_year_updated_monthly_billing_if_today_is_in_current_year( + sample_service, + mocker +): + mock = mocker.patch("app.dao.monthly_billing_dao.create_or_update_monthly_billing") + get_billing_data_for_financial_year(sample_service.id, 2016) + mock.assert_not_called()