New endpoint to get monthly billing usage from the ft_billing table.

New command to compare the results of monthly billing to ft_billing.

app/billing/rest.py

@@ -37,10 +37,9 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
         year = int(request.args.get('year'))
     except TypeError:
         return jsonify(result='error', message='No valid year provided'), 400
-
     results = fetch_monthly_billing_for_year(service_id=service_id, year=year)
     data = serialize_ft_billing(results)
-    return jsonify(monthly_usage=data)
+    return jsonify(data)
 
 
 @billing_blueprint.route('/monthly-usage')
@@ -49,13 +48,13 @@ def get_yearly_usage_by_month(service_id):
         year = int(request.args.get('year'))
         results = []
         for month in get_months_for_financial_year(year):
-            billing_for_month = get_monthly_billing_by_notification_type(service_id, month, SMS_TYPE)
-            if billing_for_month:
-                results.append(_transform_billing_for_month_sms(billing_for_month))
             letter_billing_for_month = get_monthly_billing_by_notification_type(service_id, month, LETTER_TYPE)
             if letter_billing_for_month:
                 results.extend(_transform_billing_for_month_letters(letter_billing_for_month))
-        return json.dumps(results)
+            billing_for_month = get_monthly_billing_by_notification_type(service_id, month, SMS_TYPE)
+            if billing_for_month:
+                results.append(_transform_billing_for_month_sms(billing_for_month))
+        return jsonify(results)
 
     except TypeError:
         return jsonify(result='error', message='No valid year provided'), 400
@@ -209,16 +208,13 @@ def update_free_sms_fragment_limit_data(service_id, free_sms_fragment_limit, fin
 
 def serialize_ft_billing(data):
     results = []
-    for d in data:
+    no_emails = [x for x in data if x.notification_type != 'email']
+    for d in no_emails:
         j = {
             "month": (datetime.strftime(d.month, "%B")),
-            "service_id": str(d.service_id),
+            "notification_type": d.notification_type,
-            "notifications_type": d.notification_type,
+            "billing_units": int(d.billable_units),
-            "notifications_sent": int(d.notifications_sent),
-            "billable_units": int(d.billable_units),
             "rate": float(d.rate),
-            "rate_multiplier": int(d.rate_multiplier),
-            "international": d.international,
         }
         results.append(j)
     return results

app/commands.py

@@ -7,16 +7,18 @@ from decimal import Decimal
 import click
 import flask
 from click_datetime import Datetime as click_dt
-from flask import current_app
+from flask import current_app, json
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy import func
 from notifications_utils.statsd_decorators import statsd
 
 from app import db, DATETIME_FORMAT, encryption, redis_store
+from app.billing.rest import get_yearly_usage_by_month, get_yearly_usage_by_monthly_from_ft_billing
 from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
 from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.celery.letters_pdf_tasks import create_letters_pdf
 from app.config import QueueNames
+from app.dao.date_util import get_financial_year
 from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing
 from app.dao.monthly_billing_dao import (
     create_or_update_monthly_billing,
@@ -562,3 +564,39 @@ def rebuild_ft_billing_for_month_and_service(service_id, day):
     transit_data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
     for data in transit_data:
         update_fact_billing(data, day)
+
+
+@notify_command(name='compare-ft-billing-to-monthly-billing')
+@click.option('-y', '--year', required=True)
+@click.option('-s', '--service_id', required=False, type=click.UUID)
+def compare_ft_billing_to_monthly_billing(year, service_id=None):
+    """
+    This command checks the results of monthly_billing to ft_billing for the given year.
+    If service id is not included all services are compared for the given year.
+    """
+
+    def compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response):
+        # Remove the rows with 0 billing_units and rate, ft_billing doesn't populate those rows.
+        mo_json = json.loads(monthly_billing_response.get_data(as_text=True))
+        rm_zero_rows = [x for x in mo_json if x['billing_units'] != 0 and x['rate'] != 0]
+        assert rm_zero_rows == json.loads(ft_billing_response.get_data(as_text=True))
+
+    if not service_id:
+        start_date, end_date = get_financial_year(year=int(year))
+        services = get_service_ids_that_need_billing_populated(start_date, end_date)
+        for service_id in services:
+            with current_app.test_request_context(
+                    path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
+                monthly_billing_response = get_yearly_usage_by_month(service_id)
+            with current_app.test_request_context(
+                    path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
+                ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
+            compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)
+    else:
+        with current_app.test_request_context(
+                path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
+            monthly_billing_response = get_yearly_usage_by_month(service_id)
+        with current_app.test_request_context(
+                path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
+            ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
+        compare_ft_billing_to_monthly_billing(ft_billing_response, monthly_billing_response)

app/dao/fact_billing_dao.py

@@ -23,9 +23,9 @@ from app.utils import convert_utc_to_bst, convert_bst_to_utc
 def fetch_monthly_billing_for_year(service_id, year):
     year_start_date, year_end_date = get_financial_year(year)
     utcnow = datetime.utcnow()
-    today = convert_utc_to_bst(utcnow).date()
+    today = convert_utc_to_bst(utcnow)
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
-    if year_end_date.date() >= today:
+    if year_end_date >= today:
         yesterday = today - timedelta(days=1)
         for day in [yesterday, today]:
             data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
@@ -35,11 +35,9 @@ def fetch_monthly_billing_for_year(service_id, year):
     yearly_data = db.session.query(
         func.date_trunc('month', FactBilling.bst_date).label("month"),
         func.sum(FactBilling.notifications_sent).label("notifications_sent"),
-        func.sum(FactBilling.billable_units).label("billable_units"),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
         FactBilling.service_id,
         FactBilling.rate,
-        FactBilling.rate_multiplier,
-        FactBilling.international,
         FactBilling.notification_type
     ).filter(
         FactBilling.service_id == service_id,
@@ -49,8 +47,6 @@ def fetch_monthly_billing_for_year(service_id, year):
         'month',
         FactBilling.service_id,
         FactBilling.rate,
-        FactBilling.rate_multiplier,
-        FactBilling.international,
         FactBilling.notification_type
     ).order_by(
         FactBilling.service_id,
@@ -125,7 +121,7 @@ def update_fact_billing(data, process_day):
     inserted_records = 0
     updated_records = 0
     non_letter_rates, letter_rates = get_rates_for_billing()
-
+    print("process_day: {} {}".format(type(process_day), process_day))
     update_count = FactBilling.query.filter(
         FactBilling.bst_date == datetime.date(process_day),
         FactBilling.template_id == data.template_id,

app/dao/monthly_billing_dao.py

@@ -100,7 +100,8 @@ def get_yearly_billing_data_for_date_range(
         MonthlyBilling.end_date <= end_date,
         MonthlyBilling.notification_type.in_(notification_types)
     ).order_by(
-        MonthlyBilling.notification_type
+        MonthlyBilling.start_date,
+        MonthlyBilling.notification_type,
     ).all()
 
     return results

tests/app/billing/test_billing.py

@@ -8,8 +8,8 @@ from app.dao.monthly_billing_dao import (
     create_or_update_monthly_billing,
     get_monthly_billing_by_notification_type,
 )
-from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE
+from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, FactBilling
-from app.dao.date_util import get_current_financial_year_start_year
+from app.dao.date_util import get_current_financial_year_start_year, get_month_start_and_end_date_in_utc
 from app.dao.annual_billing_dao import dao_get_free_sms_fragment_limit_for_year
 from tests.app.db import (
     create_notification,
@@ -143,30 +143,29 @@ def test_get_yearly_usage_by_month_returns_correctly(client, sample_template):
 
     resp_json = json.loads(response.get_data(as_text=True))
     _assert_dict_equals(resp_json[0], {
+        'billing_units': 0,
+        'month': 'May',
+        'notification_type': LETTER_TYPE,
+        'rate': 0
+    })
+    _assert_dict_equals(resp_json[1], {
         'billing_units': 2,
         'month': 'May',
         'notification_type': SMS_TYPE,
         'rate': 0.12
     })
-    _assert_dict_equals(resp_json[1], {
+    _assert_dict_equals(resp_json[2], {
         'billing_units': 0,
-        'month': 'May',
+        'month': 'June',
         'notification_type': LETTER_TYPE,
         'rate': 0
     })
-
+    _assert_dict_equals(resp_json[3], {
-    _assert_dict_equals(resp_json[2], {
         'billing_units': 6,
         'month': 'June',
         'notification_type': SMS_TYPE,
         'rate': 0.12
     })
-    _assert_dict_equals(resp_json[3], {
-        'billing_units': 0,
-        'month': 'June',
-        'notification_type': LETTER_TYPE,
-        'rate': 0
-    })
 
 
 def test_transform_billing_for_month_returns_empty_if_no_monthly_totals(sample_service):
@@ -413,6 +412,25 @@ def test_update_free_sms_fragment_limit_data(client, sample_service):
     assert annual_billing.free_sms_fragment_limit == 9999
 
 
+def test_get_yearly_usage_by_monthly_from_ft_billing_populates_deltas(client, notify_db_session):
+    service = create_service()
+    sms_template = create_template(service=service, template_type="sms")
+    create_rate(start_date=datetime.utcnow() - timedelta(days=1), value=0.158, notification_type='sms')
+
+    create_notification(template=sms_template, status='delivered')
+
+    assert FactBilling.query.count() == 0
+
+    response = client.get('service/{}/billing/ft-monthly-usage?year=2018'.format(service.id),
+                          headers=[('Content-Type', 'application/json'), create_authorization_header()])
+
+    assert response.status_code == 200
+    assert len(json.loads(response.get_data(as_text=True))) == 1
+    fact_billing = FactBilling.query.all()
+    assert len(fact_billing) == 1
+    assert fact_billing[0].notification_type == 'sms'
+
+
 def test_get_yearly_usage_by_monthly_from_ft_billing(client, notify_db_session):
     service = create_service()
     sms_template = create_template(service=service, template_type="sms")
@@ -427,29 +445,113 @@ def test_get_yearly_usage_by_monthly_from_ft_billing(client, notify_db_session):
                               service=service,
                               template=sms_template,
                               notification_type='sms',
+                              billable_unit=1,
                               rate=0.162)
             create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
                               service=service,
                               template=email_template,
                               notification_type='email',
                               rate=0)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              service=service,
+                              template=letter_template,
+                              notification_type='letter',
+                              billable_unit=1,
+                              rate=0.33)
+
+    response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id),
+                          headers=[('Content-Type', 'application/json'), create_authorization_header()])
+
+    json_resp = json.loads(response.get_data(as_text=True))
+    ft_letters = [x for x in json_resp if x['notification_type'] == 'letter']
+    ft_sms = [x for x in json_resp if x['notification_type'] == 'sms']
+    ft_email = [x for x in json_resp if x['notification_type'] == 'email']
+    keys = [x.keys() for x in ft_sms][0]
+    expected_sms_april = {"month": "April",
+                          "notification_type": "sms",
+                          "billing_units": 30,
+                          "rate": 0.162
+                          }
+    expected_letter_april = {"month": "April",
+                             "notification_type": "letter",
+                             "billing_units": 30,
+                             "rate": 0.33
+                             }
+
+    for k in keys:
+        assert ft_sms[0][k] == expected_sms_april[k]
+        assert ft_letters[0][k] == expected_letter_april[k]
+    assert len(ft_email) == 0
+
+
+def test_compare_ft_billing_to_monthly_billing(client, notify_db_session):
+    service = create_service()
+    sms_template = create_template(service=service, template_type="sms")
+    email_template = create_template(service=service, template_type="email")
+    letter_template = create_template(service=service, template_type="letter")
+    for month in range(4, 9):
+        mon = str(month).zfill(2)
+        days_in_month = {1: 32, 2: 30, 3: 32, 4: 31, 5: 32, 6: 31, 7: 32, 8: 32, 9: 31, 10: 32, 11: 31, 12: 32}
+        for day in range(1, days_in_month[month]):
+            d = str(day).zfill(2)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              service=service,
+                              template=sms_template,
+                              notification_type='sms',
+                              rate=0.0162)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              service=service,
+                              template=sms_template,
+                              notification_type='sms',
+                              rate_multiplier=2,
+                              rate=0.0162)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              service=service,
+                              template=email_template,
+                              notification_type='email',
+                              rate=0)
             create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
                               service=service,
                               template=letter_template,
                               notification_type='letter',
                               rate=0.33)
 
-    response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id),
+        start_date, end_date = get_month_start_and_end_date_in_utc(datetime(2016, int(mon), 1))
-                          headers=[('Content-Type', 'application/json'), create_authorization_header()])
+        create_monthly_billing_entry(service=service, start_date=start_date,
+                                     end_date=end_date,
+                                     notification_type='sms',
+                                     monthly_totals=[
+                                         {"rate": 0.0162, "international": False,
+                                          "rate_multiplier": 1, "billing_units": int(d),
+                                          "total_cost": 0.0162 * int(d)},
+                                         {"rate": 0.0162, "international": False,
+                                          "rate_multiplier": 2, "billing_units": int(d),
+                                          "total_cost": 0.0162 * int(d)}]
+                                     )
+        create_monthly_billing_entry(service=service, start_date=start_date,
+                                     end_date=end_date,
+                                     notification_type='email',
+                                     monthly_totals=[
+                                         {"rate": 0, "international": False,
+                                          "rate_multiplier": 1, "billing_units": int(d),
+                                          "total_cost": 0}]
+                                     )
+        create_monthly_billing_entry(service=service, start_date=start_date,
+                                     end_date=end_date,
+                                     notification_type='letter',
+                                     monthly_totals=[
+                                         {"rate": 0.33, "international": False,
+                                          "rate_multiplier": 1, "billing_units": int(d),
+                                          "total_cost": 0.33 * int(d)}]
+                                     )
 
-    json_resp = json.loads(response.get_data(as_text=True))
+    monthly_billing_response = client.get('/service/{}/billing/monthly-usage?year=2016'.format(service.id),
+                                          headers=[create_authorization_header()])
 
-    assert json_resp["monthly_usage"][0] == {"month": "April",
+    ft_billing_response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id),
-                                             "service_id": str(service.id),
+                                     headers=[('Content-Type', 'application/json'), create_authorization_header()])
-                                             "notifications_type": 'email',
+
-                                             "notifications_sent": 30,
+    monthly_billing_json_resp = json.loads(monthly_billing_response.get_data(as_text=True))
-                                             "billable_units": 30,
+    ft_billing_json_resp = json.loads(ft_billing_response.get_data(as_text=True))
-                                             "rate": 0.0,
+
-                                             "rate_multiplier": 1,
+    assert monthly_billing_json_resp == ft_billing_json_resp
-                                             "international": False,
-                                             }

tests/app/dao/test_ft_billing_dao.py

@@ -208,6 +208,7 @@ def test_fetch_monthly_billing_for_year(notify_db_session):
                           service=service,
                           template=template,
                           notification_type='sms',
+                          rate_multiplier=2,
                           rate=0.162)
     for i in range(1, 32):
         create_ft_billing(bst_date='2018-07-{}'.format(i),
@@ -221,11 +222,9 @@ def test_fetch_monthly_billing_for_year(notify_db_session):
     assert len(results) == 2
     assert str(results[0].month) == "2018-06-01 00:00:00+01:00"
     assert results[0].notifications_sent == 30
-    assert results[0].billable_units == Decimal('30')
+    assert results[0].billable_units == Decimal('60')
     assert results[0].service_id == service.id
     assert results[0].rate == Decimal('0.162')
-    assert results[0].rate_multiplier == Decimal('1')
-    assert results[0].international is False
     assert results[0].notification_type == 'sms'
 
     assert str(results[1].month) == "2018-07-01 00:00:00+01:00"
@@ -233,8 +232,6 @@ def test_fetch_monthly_billing_for_year(notify_db_session):
     assert results[1].billable_units == Decimal('31')
     assert results[1].service_id == service.id
     assert results[1].rate == Decimal('0.158')
-    assert results[1].rate_multiplier == Decimal('1')
-    assert results[1].international is False
     assert results[1].notification_type == 'sms'