Add endpoint to return structured data

app/billing/rest.py

@@ -14,9 +14,6 @@ from app.dao.annual_billing_dao import (
 from app.dao.date_util import get_current_financial_year_start_year
 from app.dao.fact_billing_dao import (
     fetch_monthly_billing_for_year, fetch_billing_totals_for_year,
-    fetch_sms_billing_for_all_services,
-    fetch_letter_costs_for_all_services,
-    fetch_letter_line_items_for_all_services
 )
 
 from app.errors import InvalidRequest
@@ -33,19 +30,6 @@ billing_blueprint = Blueprint(
 register_errors(billing_blueprint)
 
 
-@billing_blueprint.route('usage-for-all-services')
-def get_usage_for_all_services():
-    start_date = request.args.get('start_date')
-    end_date = request.args.get('end_date')
-
-    sms_totals = fetch_sms_billing_for_all_services(start_date, end_date)
-    letter_totals = fetch_letter_costs_for_all_services(start_date, end_date)
-    letter_breakdown = fetch_letter_line_items_for_all_services(start_date, end_date)
-
-
-
-
-
 @billing_blueprint.route('/ft-monthly-usage')
 @billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_monthly_from_ft_billing(service_id):

app/platform_stats/rest.py

@@ -2,8 +2,13 @@ from datetime import datetime
 
 from flask import Blueprint, jsonify, request
 
+from app.dao.date_util import get_financial_year
+from app.dao.fact_billing_dao import (
+    fetch_sms_billing_for_all_services, fetch_letter_costs_for_all_services,
+    fetch_letter_line_items_for_all_services
+)
 from app.dao.fact_notification_status_dao import fetch_notification_status_totals_for_all_services
-from app.errors import register_errors
+from app.errors import register_errors, InvalidRequest
 from app.platform_stats.platform_stats_schema import platform_stats_request
 from app.service.statistics import format_admin_stats
 from app.schema_validation import validate
@@ -27,3 +32,71 @@ def get_platform_stats():
     stats = format_admin_stats(data)
 
     return jsonify(stats)
+
+
+def validate_date_range_is_within_a_financial_year(start_date, end_date):
+    try:
+        start_date = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end_date = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise InvalidRequest(message="Input must be a date in the format: YYYY-MM-DD", status_code=400)
+    if end_date < start_date:
+        raise InvalidRequest(message="Start date must be before end date", status_code=400)
+    if 4 <= int(start_date.strftime("%m")) <= 12:
+        year_start, year_end = get_financial_year(year=int(start_date.strftime("%Y")))
+    else:
+        year_start, year_end = get_financial_year(year=int(start_date.strftime("%Y")) - 1)
+    year_start = year_start.date()
+    year_end = year_end.date()
+    if year_start <= start_date <= year_end and year_start <= end_date <= year_end:
+        return True
+    else:
+        raise InvalidRequest(message="Date must be in a single financial year.", status_code=400)
+
+
+@platform_stats_blueprint.route('usage-for-all-services')
+def get_usage_for_all_services():
+    start_date = request.args.get('start_date')
+    end_date = request.args.get('end_date')
+
+    validate_date_range_is_within_a_financial_year(start_date, end_date)
+    start_date = datetime.strptime(start_date, "%Y-%m-%d")
+    end_date = datetime.strptime(end_date, "%Y-%m-%d")
+
+    sms_costs = fetch_sms_billing_for_all_services(start_date, end_date)
+    letter_costs = fetch_letter_costs_for_all_services(start_date, end_date)
+    letter_breakdown = fetch_letter_line_items_for_all_services(start_date, end_date)
+
+    lb_by_service = [(lb.service_id, "{} {} class letters at {}p".format(lb.letters_sent, lb.postage, lb.letter_rate))
+                     for lb in letter_breakdown]
+    combined = {}
+    for s in sms_costs:
+        entry = {
+            "Organisation_id": str(s.organisation_id) if s.organisation_id else "",
+            "Organisation_name": s.organisation_name or "",
+            "service_id": str(s.service_id),
+            "service_name": s.service_name,
+            "sms_cost": str(s.sms_cost),
+            "letter_cost": 0,
+            "letter_breakdown": ""
+        }
+        combined[str(s.service_id)] = entry
+
+    for l in letter_costs:
+        if l.service_id in combined:
+            combined[str(l.service_id)].update({'letter_cost': l.letter_cost})
+        else:
+            letter_entry = {
+                "Organisation_id": str(l.organisation_id) if l.organisation_id else "",
+                "Organisation_name": l.organisation_name or "",
+                "service_id": str(l.service_id),
+                "service_name": l.service_name,
+                "sms_cost": 0,
+                "letter_cost": str(l.letter_cost),
+                "letter_breakdown": ""
+            }
+            combined[str(l.service_id)] = letter_entry
+    for service_id, breakdown in lb_by_service:
+        combined[str(service_id)]['letter_breakdown'] += (breakdown + '\n')
+
+    return jsonify(list(combined.values()))

tests/app/dao/test_ft_billing_dao.py

@@ -2,6 +2,9 @@ from calendar import monthrange
 from decimal import Decimal
 
 from datetime import datetime, timedelta, date
+from itertools import groupby
+from operator import itemgetter
+
 from freezegun import freeze_time
 
 import pytest
@@ -34,7 +37,8 @@ from tests.app.db import (
     create_letter_rate,
     create_notification_history,
     create_annual_billing,
-    create_organisation
+    create_organisation,
+    set_up_usage_data
 )
 
 
@@ -567,46 +571,18 @@ def test_fetch_sms_billing_for_all_services_with_remainder(notify_db_session):
 
 
 def test_fetch_sms_billing_for_all_services_without_an_organisation_appears(notify_db_session):
-    service = create_service(service_name='a - has free allowance')
+    org, org_2, service, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 5, 1))
-    template = create_template(service=service)
-    org = create_organisation(name="Org for {}".format(service.name))
-    service.organisation = org
-    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2019)
-    create_ft_billing(service=service, template=template,
-                      bst_date=datetime(2019, 4, 20), notification_type='sms', billable_unit=2, rate=0.11)
-    create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 20), notification_type='sms',
-                      billable_unit=2, rate=0.11)
-    create_ft_billing(service=service, template=template, bst_date=datetime(2019, 5, 22), notification_type='sms',
-                      billable_unit=1, rate=0.11)
-
-    service_2 = create_service(service_name='b - used free allowance')
-    template_2 = create_template(service=service_2)
-
-    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=2019)
-    create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 4, 20), notification_type='sms',
-                      billable_unit=12, rate=0.11)
-    create_ft_billing(service=service_2, template=template_2, bst_date=datetime(2019, 5, 20), notification_type='sms',
-                      billable_unit=3, rate=0.11)
     results = fetch_sms_billing_for_all_services(datetime(2019, 5, 1), datetime(2019, 5, 31))
+
     assert len(results) == 2
-
+    # organisation_name, organisation_id, service_name, service_id, free_sms_fragment_limit,
-    assert results[0].organisation_name == org.name
+    # sms_rate, sms_remainder, sms_billable_units, chargeable_billable_units, sms_cost
-    assert results[0].service_id == service.id
+    assert results[0] == (org.name, org.id, service.name, service.id, 10, Decimal('0.11'), 8, 3, 0, Decimal('0'))
-    assert results[0].sms_billable_units == 3
+    assert results[1] == (None, None, service_sms_only.name, service_sms_only.id, 10, Decimal('0.11'), 0, 3, 3, Decimal('0.33'))
-    assert results[0].sms_remainder == 8
-    assert results[0].chargeable_billable_sms == 0
-    assert results[0].sms_cost == Decimal('0')
-
-    assert not results[1].organisation_name
-    assert results[1].service_id == service_2.id
-    assert results[1].sms_billable_units == 3
-    assert results[1].sms_remainder == 0
-    assert results[1].chargeable_billable_sms == 3
-    assert results[1].sms_cost == Decimal('0.33')
 
 
 def test_fetch_letter_costs_for_all_services(notify_db_session):
-    org, org_2, service, service_2, service_3 = set_up_letter_data()
+    org, org_2, service, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 6, 1))
 
     results = fetch_letter_costs_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30))
 
@@ -617,91 +593,13 @@ def test_fetch_letter_costs_for_all_services(notify_db_session):
 
 
 def test_fetch_letter_line_items_for_all_service(notify_db_session):
-    org_1, org_2, service_1, service_2, service_3 = set_up_letter_data()
+    org_1, org_2, service_1, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 6, 1))
+
     results = fetch_letter_line_items_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30))
+
     assert len(results) == 5
     assert results[0] == (org_1.name, org_1.id, service_1.name, service_1.id, 2, Decimal('0.45'), 'second', 6)
     assert results[1] == (org_1.name, org_1.id, service_1.name, service_1.id, 1, Decimal("0.35"), 'first', 2)
     assert results[2] == (org_2.name, org_2.id, service_2.name, service_2.id, 5, Decimal("0.65"), 'second', 20)
     assert results[3] == (org_2.name, org_2.id, service_2.name, service_2.id, 3, Decimal("0.50"), 'first', 2)
     assert results[4] == (None, None, service_3.name, service_3.id, 4, Decimal("0.55"), 'second', 15)
-
-
-def test_all_three(notify_db_session):
-    set_up_letter_data()
-    sms_costs = fetch_sms_billing_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30))
-    letter_costs = fetch_letter_costs_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30))
-    letter_breakdown = fetch_letter_line_items_for_all_services(datetime(2019, 6, 1), datetime(2019, 9, 30))
-
-    combined = []
-    for s in sms_costs:
-        entry = {
-            "service_id": s.service_id,
-            "service_name": s.service_name,
-            "sms_cost": s.sms_cost,
-            "letter_cost": 0
-        }
-        combined.append(entry)
-    print("sms", combined)
-    print("letters", letter_costs)
-    for l in letter_costs:
-        row = [x for x in combined if x['service_id'] == l.service_id]
-        print("row: ", row)
-        if row:
-            row[0].update({'letter_cost': l.letter_cost})
-            print("row: ", row)
-        else:
-            letter_entry = {
-                "service_id": l.service_id,
-                "service_name": l.service_name,
-                "sms_cost": 0,
-                "letter_cost": l.letter_cost
-            }
-            combined.append(letter_entry)
-    print("final", combined)
-    assert 1==0
-
-
-def set_up_letter_data():
-    service = create_service(service_name='a - first service')
-    letter_template = create_template(service=service, template_type='letter')
-    sms_template_1 = create_template(service=service, template_type='sms')
-    create_annual_billing(service_id=service.id, free_sms_fragment_limit=1, financial_year_start=2019)
-    org = create_organisation(name="Org for {}".format(service.name))
-    dao_add_service_to_organisation(service=service, organisation_id=org.id)
-    service_2 = create_service(service_name='b - second service')
-    sms_template = create_template(service=service_2, template_type='sms')
-    create_annual_billing(service_id=service_2.id, free_sms_fragment_limit=10, financial_year_start=2019)
-    dao_add_service_to_organisation(service=service_2, organisation_id=org.id)
-    service_3 = create_service(service_name='c - third service')
-    template_3 = create_template(service=service_3)
-    org_3 = create_organisation(name="Org for {}".format(service_3.name))
-    dao_add_service_to_organisation(service=service_3, organisation_id=org_3.id)
-    service_4 = create_service(service_name='d - service without org')
-    template_4 = create_template(service=service_4, template_type='letter')
-    create_ft_billing(bst_date=datetime(2019, 8, 12), service=service, notification_type='sms',
-                      template=sms_template_1)
-    create_ft_billing(bst_date=datetime(2019, 8, 12), service=service_2, notification_type='sms',
-                      template=sms_template)
-    create_ft_billing(bst_date=datetime(2019, 9, 12), service=service_2, notification_type='sms',
-                      template=sms_template, billable_unit=20)
-    create_ft_billing(bst_date=datetime(2019, 8, 15), service=service, notification_type='letter',
-                      template=letter_template,
-                      notifications_sent=2, billable_unit=1, rate=.35, postage='first')
-    create_ft_billing(bst_date=datetime(2019, 8, 20), service=service, notification_type='letter',
-                      template=letter_template,
-                      notifications_sent=6, billable_unit=2, rate=.45, postage='second')
-    create_ft_billing(bst_date=datetime(2019, 9, 1), service=service_3, notification_type='letter',
-                      template=template_3,
-                      notifications_sent=2, billable_unit=3, rate=.50, postage='first')
-    create_ft_billing(bst_date=datetime(2019, 9, 20), service=service_3, notification_type='letter',
-                      template=template_3,
-                      notifications_sent=8, billable_unit=5, rate=.65, postage='second')
-    create_ft_billing(bst_date=datetime(2019, 9, 21), service=service_3, notification_type='letter',
-                      template=template_3,
-                      notifications_sent=12, billable_unit=5, rate=.65, postage='second')
-    create_ft_billing(bst_date=datetime(2019, 9, 12), service=service_4, notification_type='letter',
-                      template=template_4,
-                      notifications_sent=15, billable_unit=4, rate=.55, postage='second')
-
-    return org, org_3, service, service_3, service_4

tests/app/db.py

@@ -1,6 +1,6 @@
 import random
 import uuid
-from datetime import datetime, date
+from datetime import datetime, date, timedelta
 
 from app import db
 from app.dao.email_branding_dao import dao_create_email_branding
@@ -12,7 +12,7 @@ from app.dao.notifications_dao import (
     dao_create_notification,
     dao_created_scheduled_notification
 )
-from app.dao.organisation_dao import dao_create_organisation
+from app.dao.organisation_dao import dao_create_organisation, dao_add_service_to_organisation
 from app.dao.permissions_dao import permission_dao
 from app.dao.service_callback_api_dao import save_service_callback_api
 from app.dao.service_data_retention_dao import insert_service_data_retention
@@ -873,3 +873,66 @@ def create_letter_branding(name='HM Government', filename='hm-government'):
     db.session.add(test_domain_branding)
     db.session.commit()
     return test_domain_branding
+
+
+def set_up_usage_data(start_date):
+    year = int(start_date.strftime('%Y'))
+    one_week_earlier = start_date - timedelta(days=7)
+    two_days_later = start_date + timedelta(days=2)
+    one_week_later = start_date + timedelta(days=7)
+    one_month_later = start_date + timedelta(days=31)
+
+    service = create_service(service_name='a - with sms and letter')
+    letter_template = create_template(service=service, template_type='letter')
+    sms_template_1 = create_template(service=service, template_type='sms')
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=year)
+    org = create_organisation(name="Org for {}".format(service.name))
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+
+    service_3 = create_service(service_name='c - letters only')
+    template_3 = create_template(service=service_3)
+    org_3 = create_organisation(name="Org for {}".format(service_3.name))
+    dao_add_service_to_organisation(service=service_3, organisation_id=org_3.id)
+
+    service_4 = create_service(service_name='d - service without org')
+    template_4 = create_template(service=service_4, template_type='letter')
+
+    service_sms_only = create_service(service_name='b - chargeable sms')
+    sms_template = create_template(service=service_sms_only, template_type='sms')
+    create_annual_billing(service_id=service_sms_only.id, free_sms_fragment_limit=10, financial_year_start=year)
+
+    create_ft_billing(bst_date=one_week_earlier, service=service, notification_type='sms',
+                      template=sms_template_1, billable_unit=2, rate=0.11)
+    create_ft_billing(bst_date=start_date, service=service, notification_type='sms',
+                      template=sms_template_1, billable_unit=2, rate=0.11)
+    create_ft_billing(bst_date=two_days_later, service=service, notification_type='sms',
+                      template=sms_template_1, billable_unit=1, rate=0.11)
+    create_ft_billing(bst_date=one_week_later, service=service, notification_type='letter',
+                      template=letter_template,
+                      notifications_sent=2, billable_unit=1, rate=.35, postage='first')
+    create_ft_billing(bst_date=one_month_later, service=service, notification_type='letter',
+                      template=letter_template,
+                      notifications_sent=6, billable_unit=2, rate=.45, postage='second')
+
+    create_ft_billing(bst_date=one_week_earlier, service=service_sms_only, notification_type='sms',
+                      template=sms_template, rate=0.11, billable_unit=12)
+    create_ft_billing(bst_date=two_days_later, service=service_sms_only, notification_type='sms',
+                      template=sms_template, rate=0.11)
+    create_ft_billing(bst_date=one_week_later, service=service_sms_only, notification_type='sms',
+                      template=sms_template, billable_unit=2, rate=0.11)
+
+    create_ft_billing(bst_date=start_date, service=service_3, notification_type='letter',
+                      template=template_3,
+                      notifications_sent=2, billable_unit=3, rate=.50, postage='first')
+    create_ft_billing(bst_date=one_week_later, service=service_3, notification_type='letter',
+                      template=template_3,
+                      notifications_sent=8, billable_unit=5, rate=.65, postage='second')
+    create_ft_billing(bst_date=one_month_later, service=service_3, notification_type='letter',
+                      template=template_3,
+                      notifications_sent=12, billable_unit=5, rate=.65, postage='second')
+
+    create_ft_billing(bst_date=two_days_later, service=service_4, notification_type='letter',
+                      template=template_4,
+                      notifications_sent=15, billable_unit=4, rate=.55, postage='second')
+
+    return org, org_3, service, service_3, service_4, service_sms_only

tests/app/platform_stats/test_rest.py

@@ -1,9 +1,15 @@
 from datetime import date, datetime
 
+import pytest
 from freezegun import freeze_time
 
+from app.errors import InvalidRequest
 from app.models import SMS_TYPE, EMAIL_TYPE
-from tests.app.db import create_service, create_template, create_ft_notification_status, create_notification
+from app.platform_stats.rest import validate_date_range_is_within_a_financial_year
+from tests.app.db import (
+    create_service, create_template, create_ft_notification_status, create_notification,
+    set_up_usage_data
+)
 
 
 @freeze_time('2018-06-01')
@@ -72,3 +78,53 @@ def test_get_platform_stats_with_real_query(admin_request, notify_db_session):
             'total': 11, 'test-key': 1
         }
     }
+
+
+@pytest.mark.parametrize('start_date, end_date',
+                         [('2019-04-01', '2019-06-30'),
+                          ('2019-08-01', '2019-09-30'),
+                          ('2019-01-01', '2019-03-31'),
+                          ('2019-12-01', '2020-02-28')])
+def test_validate_date_range_is_within_a_financial_year_returns_true(start_date, end_date):
+    assert validate_date_range_is_within_a_financial_year(start_date, end_date)
+
+
+@pytest.mark.parametrize('start_date, end_date',
+                         [('2019-04-01', '2020-06-30'),
+                          ('2019-01-01', '2019-04-30'),
+                          ('2019-12-01', '2020-04-30'),
+                          ('2019-03-31', '2019-04-01')])
+def test_validate_date_range_is_within_a_financial_year_returns_false(start_date, end_date):
+    with pytest.raises(expected_exception=InvalidRequest) as e:
+        validate_date_range_is_within_a_financial_year(start_date, end_date)
+        assert e.message == 'Date must be in a single financial year.'
+        assert e.code == 400
+
+
+def test_validate_date_is_within_a_financial_year_raises_validation_error():
+    start_date = '2019-08-01'
+    end_date = '2019-06-01'
+
+    with pytest.raises(expected_exception=InvalidRequest) as e:
+        validate_date_range_is_within_a_financial_year(start_date, end_date)
+        assert e.message == 'Start date must be before end date'
+        assert e.code == 400
+
+
+@pytest.mark.parametrize('start_date, end_date',
+                         [('22-01-2019', '2019-08-01'),
+                          ('2019-07-01', 'not-date')])
+def test_validate_date_is_within_a_financial_year_when_input_is_not_a_date(start_date, end_date):
+    with pytest.raises(expected_exception=InvalidRequest) as e:
+        assert validate_date_range_is_within_a_financial_year(start_date, end_date)
+        assert e.message == 'Input must be a date in the format: YYYY-MM-DD'
+        assert e.code == 400
+
+
+def test_get_usage_for_all_services(notify_db_session, admin_request):
+    org, org_2, service, service_2, service_3, service_sms_only = set_up_usage_data(datetime(2019, 5, 1))
+    response = admin_request.get("platform_stats.get_usage_for_all_services",
+                                 start_date='2019-05-01',
+                                 end_date='2019-06-30')
+    print(response)
+    assert 1 == 0