From cb7fff6100c13ba291958b9ad0f14c1c26493c13 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Thu, 22 Aug 2019 17:48:24 +0100 Subject: [PATCH] Add endpoint to return structured data --- app/billing/rest.py | 16 ---- app/platform_stats/rest.py | 75 ++++++++++++++- tests/app/dao/test_ft_billing_dao.py | 132 +++----------------------- tests/app/db.py | 67 ++++++++++++- tests/app/platform_stats/test_rest.py | 58 ++++++++++- 5 files changed, 211 insertions(+), 137 deletions(-) diff --git a/app/billing/rest.py b/app/billing/rest.py index bfe62e4e8..13f34bf37 100644 --- a/app/billing/rest.py +++ b/app/billing/rest.py @@ -14,9 +14,6 @@ from app.dao.annual_billing_dao import ( from app.dao.date_util import get_current_financial_year_start_year from app.dao.fact_billing_dao import ( fetch_monthly_billing_for_year, fetch_billing_totals_for_year, - fetch_sms_billing_for_all_services, - fetch_letter_costs_for_all_services, - fetch_letter_line_items_for_all_services ) from app.errors import InvalidRequest @@ -33,19 +30,6 @@ billing_blueprint = Blueprint( register_errors(billing_blueprint) -@billing_blueprint.route('usage-for-all-services') -def get_usage_for_all_services(): - start_date = request.args.get('start_date') - end_date = request.args.get('end_date') - - sms_totals = fetch_sms_billing_for_all_services(start_date, end_date) - letter_totals = fetch_letter_costs_for_all_services(start_date, end_date) - letter_breakdown = fetch_letter_line_items_for_all_services(start_date, end_date) - - - - - @billing_blueprint.route('/ft-monthly-usage') @billing_blueprint.route('/monthly-usage') def get_yearly_usage_by_monthly_from_ft_billing(service_id): diff --git a/app/platform_stats/rest.py b/app/platform_stats/rest.py index efe936f84..243f98979 100644 --- a/app/platform_stats/rest.py +++ b/app/platform_stats/rest.py @@ -2,8 +2,13 @@ from datetime import datetime from flask import Blueprint, jsonify, request +from app.dao.date_util import get_financial_year +from app.dao.fact_billing_dao import ( + fetch_sms_billing_for_all_services, fetch_letter_costs_for_all_services, + fetch_letter_line_items_for_all_services +) from app.dao.fact_notification_status_dao import fetch_notification_status_totals_for_all_services -from app.errors import register_errors +from app.errors import register_errors, InvalidRequest from app.platform_stats.platform_stats_schema import platform_stats_request from app.service.statistics import format_admin_stats from app.schema_validation import validate @@ -27,3 +32,71 @@ def get_platform_stats(): stats = format_admin_stats(data) return jsonify(stats) + + +def validate_date_range_is_within_a_financial_year(start_date, end_date): + try: + start_date = datetime.strptime(start_date, "%Y-%m-%d").date() + end_date = datetime.strptime(end_date, "%Y-%m-%d").date() + except ValueError: + raise InvalidRequest(message="Input must be a date in the format: YYYY-MM-DD", status_code=400) + if end_date < start_date: + raise InvalidRequest(message="Start date must be before end date", status_code=400) + if 4 <= int(start_date.strftime("%m")) <= 12: + year_start, year_end = get_financial_year(year=int(start_date.strftime("%Y"))) + else: + year_start, year_end = get_financial_year(year=int(start_date.strftime("%Y")) - 1) + year_start = year_start.date() + year_end = year_end.date() + if year_start <= start_date <= year_end and year_start <= end_date <= year_end: + return True + else: + raise InvalidRequest(message="Date must be in a single financial year.", status_code=400) + + +@platform_stats_blueprint.route('usage-for-all-services') +def get_usage_for_all_services(): + start_date = request.args.get('start_date') + end_date = request.args.get('end_date') + + validate_date_range_is_within_a_financial_year(start_date, end_date) + start_date = datetime.strptime(start_date, "%Y-%m-%d") + end_date = datetime.strptime(end_date, "%Y-%m-%d") + + sms_costs = fetch_sms_billing_for_all_services(start_date, end_date) + letter_costs = fetch_letter_costs_for_all_services(start_date, end_date) + letter_breakdown = fetch_letter_line_items_for_all_services(start_date, end_date) + + lb_by_service = [(lb.service_id, "{} {} class letters at {}p".format(lb.letters_sent, lb.postage, lb.letter_rate)) + for lb in letter_breakdown] + combined = {} + for s in sms_costs: + entry = { + "Organisation_id": str(s.organisation_id) if s.organisation_id else "", + "Organisation_name": s.organisation_name or "", + "service_id": str(s.service_id), + "service_name": s.service_name, + "sms_cost": str(s.sms_cost), + "letter_cost": 0, + "letter_breakdown": "" + } + combined[str(s.service_id)] = entry + + for l in letter_costs: + if l.service_id in combined: + combined[str(l.service_id)].update({'letter_cost': l.letter_cost}) + else: + letter_entry = { + "Organisation_id": str(l.organisation_id) if l.organisation_id else "", + "Organisation_name": l.organisation_name or "", + "service_id": str(l.service_id), + "service_name": l.service_name, + "sms_cost": 0, + "letter_cost": str(l.letter_cost), + "letter_breakdown": "" + } + combined[str(l.service_id)] = letter_entry + for service_id, breakdown in lb_by_service: + combined[str(service_id)]['letter_breakdown'] += (breakdown + '\n') + + return jsonify(list(combined.values())) diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index 995c2ccde..e21b2e094 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -2,6 +2,9 @@ from calendar import monthrange from decimal import Decimal from datetime import datetime, timedelta, date +from itertools import groupby +from operator import itemgetter + from freezegun import freeze_time import pytest @@ -34,7 +37,8 @@ from tests.app.db import ( create_letter_rate, create_notification_history, create_annual_billing, - create_organisation + create_organisation, + set_up_usage_data ) @@ -567,46 +571,18 @@ def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session): def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(notify_db_session): - service = create_service(service_name='a - has free allowance') - template = create_template(service=service) - org = create_organisation(name="Org for {}".format(service.name)) - service.organisation = org - create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2019) - create_ft_billing(service=service, template=template, - bst_date=datetime(2019, 4, 20), notification_type='sms', billable_unit=2, rate=0.11) - create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 20), notification_type='sms', - billable_unit=2, rate=0.11) - create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 22), notification_type='sms', - billable_unit=1, rate=0.11) - - service_2 = create_service(service_name='b - used free allowance') - template_2 = create_template(service=service_2) - - create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=2019) - create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 4, 20), notification_type='sms', - billable_unit=12, rate=0.11) - create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 5, 20), notification_type='sms', - billable_unit=3, rate=0.11) + org, org_2, service, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 5, 1)) results = fetch_sms_billing_for_all_services(datetime(2019, 5, 1), datetime(2019, 5, 31)) + assert len(results) == 2 - - assert results[0].organisation_name == org.name - assert results[0].service_id == service.id - assert results[0].sms_billable_units == 3 - assert results[0].sms_remainder == 8 - assert results[0].chargeable_billable_sms == 0 - assert results[0].sms_cost == Decimal('0') - - assert not results[1].organisation_name - assert results[1].service_id == service_2.id - assert results[1].sms_billable_units == 3 - assert results[1].sms_remainder == 0 - assert results[1].chargeable_billable_sms == 3 - assert results[1].sms_cost == Decimal('0.33') + # organisation_name, organisation_id, service_name, service_id, free_sms_fragment_limit, + # sms_rate, sms_remainder, sms_billable_units, chargeable_billable_units, sms_cost + assert results[0] == (org.name, org.id, service.name, service.id, 10, Decimal('0.11'), 8, 3, 0, Decimal('0')) + assert results[1] == (None, None, service_sms_only.name, service_sms_only.id, 10, Decimal('0.11'), 0, 3, 3, Decimal('0.33')) def test_fetch_letter_costs_for_all_services(notify_db_session): - org, org_2, service, service_2, service_3 = set_up_letter_data() + org, org_2, service, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 6, 1)) results = fetch_letter_costs_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30)) @@ -617,91 +593,13 @@ def test_fetch_letter_costs_for_all_services(notify_db_session): def test_fetch_letter_line_items_for_all_service(notify_db_session): - org_1, org_2, service_1, service_2, service_3 = set_up_letter_data() + org_1, org_2, service_1, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 6, 1)) + results = fetch_letter_line_items_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30)) + assert len(results) == 5 assert results[0] == (org_1.name, org_1.id, service_1.name, service_1.id, 2, Decimal('0.45'), 'second', 6) assert results[1] == (org_1.name, org_1.id, service_1.name, service_1.id, 1, Decimal("0.35"), 'first', 2) assert results[2] == (org_2.name, org_2.id, service_2.name, service_2.id, 5, Decimal("0.65"), 'second', 20) assert results[3] == (org_2.name, org_2.id, service_2.name, service_2.id, 3, Decimal("0.50"), 'first', 2) assert results[4] == (None, None, service_3.name, service_3.id, 4, Decimal("0.55"), 'second', 15) - - -def test_all_three(notify_db_session): - set_up_letter_data() - sms_costs = fetch_sms_billing_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30)) - letter_costs = fetch_letter_costs_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30)) - letter_breakdown = fetch_letter_line_items_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30)) - - combined = [] - for s in sms_costs: - entry = { - "service_id": s.service_id, - "service_name": s.service_name, - "sms_cost": s.sms_cost, - "letter_cost": 0 - } - combined.append(entry) - print("sms", combined) - print("letters", letter_costs) - for l in letter_costs: - row = [x for x in combined if x['service_id'] == l.service_id] - print("row: ", row) - if row: - row[0].update({'letter_cost': l.letter_cost}) - print("row: ", row) - else: - letter_entry = { - "service_id": l.service_id, - "service_name": l.service_name, - "sms_cost": 0, - "letter_cost": l.letter_cost - } - combined.append(letter_entry) - print("final", combined) - assert 1==0 - - -def set_up_letter_data(): - service = create_service(service_name='a - first service') - letter_template = create_template(service=service, template_type='letter') - sms_template_1 = create_template(service=service, template_type='sms') - create_annual_billing(service_id=service.id, free_sms_fragment_limit=1, financial_year_start=2019) - org = create_organisation(name="Org for {}".format(service.name)) - dao_add_service_to_organisation(service=service, organisation_id=org.id) - service_2 = create_service(service_name='b - second service') - sms_template = create_template(service=service_2, template_type='sms') - create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=2019) - dao_add_service_to_organisation(service=service_2, organisation_id=org.id) - service_3 = create_service(service_name='c - third service') - template_3 = create_template(service=service_3) - org_3 = create_organisation(name="Org for {}".format(service_3.name)) - dao_add_service_to_organisation(service=service_3, organisation_id=org_3.id) - service_4 = create_service(service_name='d - service without org') - template_4 = create_template(service=service_4, template_type='letter') - create_ft_billing(bst_date=datetime(2019, 8, 12), service=service, notification_type='sms', - template=sms_template_1) - create_ft_billing(bst_date=datetime(2019, 8, 12), service=service_2, notification_type='sms', - template=sms_template) - create_ft_billing(bst_date=datetime(2019, 9, 12), service=service_2, notification_type='sms', - template=sms_template, billable_unit=20) - create_ft_billing(bst_date=datetime(2019, 8, 15), service=service, notification_type='letter', - template=letter_template, - notifications_sent=2, billable_unit=1, rate=.35, postage='first') - create_ft_billing(bst_date=datetime(2019, 8, 20), service=service, notification_type='letter', - template=letter_template, - notifications_sent=6, billable_unit=2, rate=.45, postage='second') - create_ft_billing(bst_date=datetime(2019, 9, 1), service=service_3, notification_type='letter', - template=template_3, - notifications_sent=2, billable_unit=3, rate=.50, postage='first') - create_ft_billing(bst_date=datetime(2019, 9, 20), service=service_3, notification_type='letter', - template=template_3, - notifications_sent=8, billable_unit=5, rate=.65, postage='second') - create_ft_billing(bst_date=datetime(2019, 9, 21), service=service_3, notification_type='letter', - template=template_3, - notifications_sent=12, billable_unit=5, rate=.65, postage='second') - create_ft_billing(bst_date=datetime(2019, 9, 12), service=service_4, notification_type='letter', - template=template_4, - notifications_sent=15, billable_unit=4, rate=.55, postage='second') - - return org, org_3, service, service_3, service_4 diff --git a/tests/app/db.py b/tests/app/db.py index 641e4f320..b06d856ef 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -1,6 +1,6 @@ import random import uuid -from datetime import datetime, date +from datetime import datetime, date, timedelta from app import db from app.dao.email_branding_dao import dao_create_email_branding @@ -12,7 +12,7 @@ from app.dao.notifications_dao import ( dao_create_notification, dao_created_scheduled_notification ) -from app.dao.organisation_dao import dao_create_organisation +from app.dao.organisation_dao import dao_create_organisation, dao_add_service_to_organisation from app.dao.permissions_dao import permission_dao from app.dao.service_callback_api_dao import save_service_callback_api from app.dao.service_data_retention_dao import insert_service_data_retention @@ -873,3 +873,66 @@ def create_letter_branding(name='HM Government', filename='hm-government'): db.session.add(test_domain_branding) db.session.commit() return test_domain_branding + + +def set_up_usage_data(start_date): + year = int(start_date.strftime('%Y')) + one_week_earlier = start_date - timedelta(days=7) + two_days_later = start_date + timedelta(days=2) + one_week_later = start_date + timedelta(days=7) + one_month_later = start_date + timedelta(days=31) + + service = create_service(service_name='a - with sms and letter') + letter_template = create_template(service=service, template_type='letter') + sms_template_1 = create_template(service=service, template_type='sms') + create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=year) + org = create_organisation(name="Org for {}".format(service.name)) + dao_add_service_to_organisation(service=service, organisation_id=org.id) + + service_3 = create_service(service_name='c - letters only') + template_3 = create_template(service=service_3) + org_3 = create_organisation(name="Org for {}".format(service_3.name)) + dao_add_service_to_organisation(service=service_3, organisation_id=org_3.id) + + service_4 = create_service(service_name='d - service without org') + template_4 = create_template(service=service_4, template_type='letter') + + service_sms_only = create_service(service_name='b - chargeable sms') + sms_template = create_template(service=service_sms_only, template_type='sms') + create_annual_billing(service_id=service_sms_only.id, free_sms_fragment_limit=10, financial_year_start=year) + + create_ft_billing(bst_date=one_week_earlier, service=service, notification_type='sms', + template=sms_template_1, billable_unit=2, rate=0.11) + create_ft_billing(bst_date=start_date, service=service, notification_type='sms', + template=sms_template_1, billable_unit=2, rate=0.11) + create_ft_billing(bst_date=two_days_later, service=service, notification_type='sms', + template=sms_template_1, billable_unit=1, rate=0.11) + create_ft_billing(bst_date=one_week_later, service=service, notification_type='letter', + template=letter_template, + notifications_sent=2, billable_unit=1, rate=.35, postage='first') + create_ft_billing(bst_date=one_month_later, service=service, notification_type='letter', + template=letter_template, + notifications_sent=6, billable_unit=2, rate=.45, postage='second') + + create_ft_billing(bst_date=one_week_earlier, service=service_sms_only, notification_type='sms', + template=sms_template, rate=0.11, billable_unit=12) + create_ft_billing(bst_date=two_days_later, service=service_sms_only, notification_type='sms', + template=sms_template, rate=0.11) + create_ft_billing(bst_date=one_week_later, service=service_sms_only, notification_type='sms', + template=sms_template, billable_unit=2, rate=0.11) + + create_ft_billing(bst_date=start_date, service=service_3, notification_type='letter', + template=template_3, + notifications_sent=2, billable_unit=3, rate=.50, postage='first') + create_ft_billing(bst_date=one_week_later, service=service_3, notification_type='letter', + template=template_3, + notifications_sent=8, billable_unit=5, rate=.65, postage='second') + create_ft_billing(bst_date=one_month_later, service=service_3, notification_type='letter', + template=template_3, + notifications_sent=12, billable_unit=5, rate=.65, postage='second') + + create_ft_billing(bst_date=two_days_later, service=service_4, notification_type='letter', + template=template_4, + notifications_sent=15, billable_unit=4, rate=.55, postage='second') + + return org, org_3, service, service_3, service_4, service_sms_only diff --git a/tests/app/platform_stats/test_rest.py b/tests/app/platform_stats/test_rest.py index 91474171e..421559045 100644 --- a/tests/app/platform_stats/test_rest.py +++ b/tests/app/platform_stats/test_rest.py @@ -1,9 +1,15 @@ from datetime import date, datetime +import pytest from freezegun import freeze_time +from app.errors import InvalidRequest from app.models import SMS_TYPE, EMAIL_TYPE -from tests.app.db import create_service, create_template, create_ft_notification_status, create_notification +from app.platform_stats.rest import validate_date_range_is_within_a_financial_year +from tests.app.db import ( + create_service, create_template, create_ft_notification_status, create_notification, + set_up_usage_data +) @freeze_time('2018-06-01') @@ -72,3 +78,53 @@ def test_get_platform_stats_with_real_query(admin_request, notify_db_session): 'total': 11, 'test-key': 1 } } + + +@pytest.mark.parametrize('start_date, end_date', + [('2019-04-01', '2019-06-30'), + ('2019-08-01', '2019-09-30'), + ('2019-01-01', '2019-03-31'), + ('2019-12-01', '2020-02-28')]) +def test_validate_date_range_is_within_a_financial_year_returns_true(start_date, end_date): + assert validate_date_range_is_within_a_financial_year(start_date, end_date) + + +@pytest.mark.parametrize('start_date, end_date', + [('2019-04-01', '2020-06-30'), + ('2019-01-01', '2019-04-30'), + ('2019-12-01', '2020-04-30'), + ('2019-03-31', '2019-04-01')]) +def test_validate_date_range_is_within_a_financial_year_returns_false(start_date, end_date): + with pytest.raises(expected_exception=InvalidRequest) as e: + validate_date_range_is_within_a_financial_year(start_date, end_date) + assert e.message == 'Date must be in a single financial year.' + assert e.code == 400 + + +def test_validate_date_is_within_a_financial_year_raises_validation_error(): + start_date = '2019-08-01' + end_date = '2019-06-01' + + with pytest.raises(expected_exception=InvalidRequest) as e: + validate_date_range_is_within_a_financial_year(start_date, end_date) + assert e.message == 'Start date must be before end date' + assert e.code == 400 + + +@pytest.mark.parametrize('start_date, end_date', + [('22-01-2019', '2019-08-01'), + ('2019-07-01', 'not-date')]) +def test_validate_date_is_within_a_financial_year_when_input_is_not_a_date(start_date, end_date): + with pytest.raises(expected_exception=InvalidRequest) as e: + assert validate_date_range_is_within_a_financial_year(start_date, end_date) + assert e.message == 'Input must be a date in the format: YYYY-MM-DD' + assert e.code == 400 + + +def test_get_usage_for_all_services(notify_db_session, admin_request): + org, org_2, service, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 5, 1)) + response = admin_request.get("platform_stats.get_usage_for_all_services", + start_date='2019-05-01', + end_date='2019-06-30') + print(response) + assert 1 == 0