Merge pull request #1867 from alphagov/use-ft-billing-for-usage

Update ft_billing table to use integer data types

app/billing/billing_schemas.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+
 create_or_update_free_sms_fragment_limit_schema = {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST annual billing schema",
@@ -24,3 +25,17 @@ def serialize_ft_billing_remove_emails(data):
         }
         results.append(json_result)
     return results
+
+
+def serialize_ft_billing_yearly_totals(data):
+    yearly_totals = []
+    for total in data:
+        json_result = {
+            "notification_type": total.notification_type,
+            "billing_units": total.billable_units,
+            "rate": float(total.rate),
+            "letter_total": float(total.billable_units * total.rate) if total.notification_type == 'letter' else 0
+        }
+        yearly_totals.append(json_result)
+
+    return yearly_totals

app/billing/rest.py

@@ -5,7 +5,8 @@ from flask import Blueprint, jsonify, request
 
 from app.billing.billing_schemas import (
     create_or_update_free_sms_fragment_limit_schema,
-    serialize_ft_billing_remove_emails
+    serialize_ft_billing_remove_emails,
+    serialize_ft_billing_yearly_totals,
 )
 from app.dao.annual_billing_dao import (
     dao_get_free_sms_fragment_limit_for_year,
@@ -15,7 +16,7 @@ from app.dao.annual_billing_dao import (
 )
 from app.dao.date_util import get_current_financial_year_start_year
 from app.dao.date_util import get_months_for_financial_year
-from app.dao.fact_billing_dao import fetch_monthly_billing_for_year
+from app.dao.fact_billing_dao import fetch_monthly_billing_for_year, fetch_billing_totals_for_year
 from app.dao.monthly_billing_dao import (
     get_billing_data_for_financial_year,
     get_monthly_billing_by_notification_type
@@ -47,6 +48,18 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     return jsonify(data)
 
 
+@billing_blueprint.route('/ft-yearly-usage-summary')
+def get_yearly_billing_usage_summary_from_ft_billing(service_id):
+    try:
+        year = int(request.args.get('year'))
+    except TypeError:
+        return jsonify(result='error', message='No valid year provided'), 400
+
+    billing_data = fetch_billing_totals_for_year(service_id, year)
+    data = serialize_ft_billing_yearly_totals(billing_data)
+    return jsonify(data)
+
+
 @billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_month(service_id):
     try:
@@ -70,7 +83,7 @@ def get_yearly_billing_usage_summary(service_id):
     try:
         year = int(request.args.get('year'))
         billing_data = get_billing_data_for_financial_year(service_id, year)
-        notification_types = [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]
+        notification_types = [EMAIL_TYPE, LETTER_TYPE, SMS_TYPE]
         response = [
             _get_total_billable_units_and_rate_for_notification_type(billing_data, notification_type)
             for notification_type in notification_types
@@ -102,8 +115,8 @@ def _get_total_billable_units_and_rate_for_notification_type(billing_data, noti_
     return {
         "notification_type": noti_type,
         "billing_units": total_sent,
-        "rate": rate,
+        "rate": float(rate),
-        "letter_total": letter_total
+        "letter_total": round(float(letter_total), 3)
     }
 
 
@@ -132,7 +145,7 @@ def _transform_billing_for_month_letters(billing_for_month):
             "month": month_name,
             "billing_units": (total['billing_units'] * total['rate_multiplier']),
             "notification_type": billing_for_month.notification_type,
-            "rate": total['rate']
+            "rate": float(total['rate'])
         }
         x.append(y)
     if len(billing_for_month.monthly_totals) == 0:

app/celery/reporting_tasks.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta
 
+from flask import current_app
 from notifications_utils.statsd_decorators import statsd
 
 from app import notify_celery
@@ -24,3 +25,6 @@ def create_nightly_billing(day_start=None):
 
         for data in transit_data:
             update_fact_billing(data, process_day)
+
+        current_app.logger.info(
+            "create-nightly-billing task complete. {} rows updated for day: {}".format(len(transit_data), process_day))

app/commands.py

@@ -551,19 +551,26 @@ def populate_redis_template_usage(service_id, day):
         )
 
 
-@notify_command(name='rebuild-ft-billing-for-month-and-service')
+@notify_command(name='rebuild-ft-billing-for-day')
-@click.option('-s', '--service_id', required=True, type=click.UUID)
+@click.option('-s', '--service_id', required=False, type=click.UUID)
 @click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True,
               type=click_dt(format='%Y-%m-%d'))
-def rebuild_ft_billing_for_month_and_service(service_id, day):
+def rebuild_ft_billing_for_day(service_id, day):
     """
     Rebuild the data in ft_billing for the given service_id and date
     """
-    # confirm the service exists
+    def rebuild_ft_data(process_day, service):
-    dao_fetch_service_by_id(service_id)
+        transit_data = fetch_billing_data_for_day(process_day=process_day, service_id=service)
-    transit_data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
+        for data in transit_data:
-    for data in transit_data:
+            update_fact_billing(data, process_day)
-        update_fact_billing(data, day)
+    if service_id:
+        # confirm the service exists
+        dao_fetch_service_by_id(service_id)
+        rebuild_ft_data(day, service_id)
+    else:
+        services = get_service_ids_that_need_billing_populated(day, day)
+        for service_id in services:
+            rebuild_ft_data(day, service_id)
 
 
 @notify_command(name='compare-ft-billing-to-monthly-billing')

app/dao/fact_billing_dao.py

@@ -1,6 +1,5 @@
 from datetime import datetime, timedelta, time
 
-from flask import current_app
 from sqlalchemy import func, case, desc, Date
 
 from app import db
@@ -20,6 +19,31 @@ from app.models import (
 from app.utils import convert_utc_to_bst, convert_bst_to_utc
 
 
+def fetch_billing_totals_for_year(service_id, year):
+    year_start_date, year_end_date = get_financial_year(year)
+
+    yearly_data = db.session.query(
+        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
+        FactBilling.service_id,
+        FactBilling.rate,
+        FactBilling.notification_type
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date
+    ).group_by(
+        FactBilling.service_id,
+        FactBilling.rate,
+        FactBilling.notification_type
+    ).order_by(
+        FactBilling.service_id,
+        FactBilling.notification_type
+    ).all()
+
+    return yearly_data
+
+
 def fetch_monthly_billing_for_year(service_id, year):
     year_start_date, year_end_date = get_financial_year(year)
     utcnow = datetime.utcnow()
@@ -97,6 +121,7 @@ def fetch_billing_data_for_day(process_day, service_id=None):
     )
     if service_id:
         transit_data = transit_data.filter(Notification.service_id == service_id)
+
     return transit_data.all()
 
 
@@ -121,7 +146,6 @@ def update_fact_billing(data, process_day):
     inserted_records = 0
     updated_records = 0
     non_letter_rates, letter_rates = get_rates_for_billing()
-    print("process_day: {} {}".format(type(process_day), process_day))
     update_count = FactBilling.query.filter(
         FactBilling.bst_date == datetime.date(process_day),
         FactBilling.template_id == data.template_id,
@@ -147,8 +171,6 @@ def update_fact_billing(data, process_day):
         inserted_records += 1
     updated_records += update_count
     db.session.commit()
-    current_app.logger.info('ft_billing for {}: {} rows updated, {} rows inserted'
-                            .format(process_day, updated_records, inserted_records))
 
 
 def create_billing_record(data, rate, process_day):

app/models.py

@@ -1784,10 +1784,10 @@ class FactBilling(db.Model):
     service_id = db.Column(UUID(as_uuid=True), nullable=False, index=True)
     notification_type = db.Column(db.Text, nullable=False, primary_key=True)
     provider = db.Column(db.Text, nullable=True, primary_key=True)
-    rate_multiplier = db.Column(db.Numeric(), nullable=True, primary_key=True)
+    rate_multiplier = db.Column(db.Integer(), nullable=True, primary_key=True)
     international = db.Column(db.Boolean, nullable=False, primary_key=False)
     rate = db.Column(db.Numeric(), nullable=True)
-    billable_units = db.Column(db.Numeric(), nullable=True)
+    billable_units = db.Column(db.Integer(), nullable=True)
     notifications_sent = db.Column(db.Integer(), nullable=True)
 
 

migrations/versions/0189_ft_billing_data_type.py

@@ -0,0 +1,32 @@
+"""
+
+Revision ID: 0189_ft_billing_data_type
+Revises: 0188_add_ft_notification_status
+Create Date: 2018-05-10 14:57:52.589773
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = '0189_ft_billing_data_type'
+down_revision = '0188_add_ft_notification_status'
+
+
+def upgrade():
+    op.alter_column('ft_billing', 'billable_units',
+                    existing_type=sa.NUMERIC(),
+                    type_=sa.Integer(),
+                    existing_nullable=True)
+    op.alter_column('ft_billing', 'rate_multiplier',
+                    existing_type=sa.NUMERIC(),
+                    type_=sa.Integer())
+
+
+def downgrade():
+    op.alter_column('ft_billing', 'rate_multiplier',
+                    existing_type=sa.Integer(),
+                    type_=sa.NUMERIC())
+    op.alter_column('ft_billing', 'billable_units',
+                    existing_type=sa.Integer(),
+                    type_=sa.NUMERIC(),
+                    existing_nullable=True)

tests/app/billing/test_billing.py

@@ -67,24 +67,23 @@ def test_get_yearly_billing_summary_returns_correct_breakdown(client, sample_tem
     assert len(resp_json) == 3
 
     _assert_dict_equals(resp_json[0], {
-        'notification_type': SMS_TYPE,
-        'billing_units': 8,
-        'rate': 0.12,
-        'letter_total': 0
-    })
-
-    _assert_dict_equals(resp_json[1], {
         'notification_type': EMAIL_TYPE,
         'billing_units': 0,
         'rate': 0,
         'letter_total': 0
     })
-    _assert_dict_equals(resp_json[2], {
+    _assert_dict_equals(resp_json[1], {
         'notification_type': LETTER_TYPE,
         'billing_units': 2,
         'rate': 0,
         'letter_total': 0.72
     })
+    _assert_dict_equals(resp_json[2], {
+        'notification_type': SMS_TYPE,
+        'billing_units': 8,
+        'rate': 0.12,
+        'letter_total': 0
+    })
 
 
 def test_get_yearly_billing_usage_breakdown_returns_400_if_missing_year(client, sample_service):
@@ -485,6 +484,21 @@ def test_get_yearly_usage_by_monthly_from_ft_billing(client, notify_db_session):
 
 
 def test_compare_ft_billing_to_monthly_billing(client, notify_db_session):
+    service = set_up_yearly_data()
+
+    monthly_billing_response = client.get('/service/{}/billing/monthly-usage?year=2016'.format(service.id),
+                                          headers=[create_authorization_header()])
+
+    ft_billing_response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id),
+                                     headers=[('Content-Type', 'application/json'), create_authorization_header()])
+
+    monthly_billing_json_resp = json.loads(monthly_billing_response.get_data(as_text=True))
+    ft_billing_json_resp = json.loads(ft_billing_response.get_data(as_text=True))
+
+    assert monthly_billing_json_resp == ft_billing_json_resp
+
+
+def set_up_yearly_data():
     service = create_service()
     sms_template = create_template(service=service, template_type="sms")
     email_template = create_template(service=service, template_type="email")
@@ -542,14 +556,68 @@ def test_compare_ft_billing_to_monthly_billing(client, notify_db_session):
                                           "rate_multiplier": 1, "billing_units": int(d),
                                           "total_cost": 0.33 * int(d)}]
                                      )
+    return service
 
-    monthly_billing_response = client.get('/service/{}/billing/monthly-usage?year=2016'.format(service.id),
+
+def test_get_yearly_billing_usage_summary_from_ft_billing_comapre_to_monthyl_billing(
+        client, notify_db_session
+):
+    service = set_up_yearly_data()
+    monthly_billing_response = client.get('/service/{}/billing/yearly-usage-summary?year=2016'.format(service.id),
                                           headers=[create_authorization_header()])
 
-    ft_billing_response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id),
+    ft_billing_response = client.get('service/{}/billing/ft-yearly-usage-summary?year=2016'.format(service.id),
                                      headers=[('Content-Type', 'application/json'), create_authorization_header()])
 
     monthly_billing_json_resp = json.loads(monthly_billing_response.get_data(as_text=True))
     ft_billing_json_resp = json.loads(ft_billing_response.get_data(as_text=True))
 
-    assert monthly_billing_json_resp == ft_billing_json_resp
+    assert len(monthly_billing_json_resp) == 3
+    assert len(ft_billing_json_resp) == 3
+    for i in range(0, 3):
+        assert sorted(monthly_billing_json_resp[i]) == sorted(ft_billing_json_resp[i])
+
+
+def test_get_yearly_billing_usage_summary_from_ft_billing_returns_400_if_missing_year(client, sample_service):
+    response = client.get(
+        '/service/{}/billing/ft-yearly-usage-summary'.format(sample_service.id),
+        headers=[create_authorization_header()]
+    )
+    assert response.status_code == 400
+    assert json.loads(response.get_data(as_text=True)) == {
+        'message': 'No valid year provided', 'result': 'error'
+    }
+
+
+def test_get_yearly_billing_usage_summary_from_ft_billing_returns_empty_list_if_no_billing_data(
+        client, sample_service
+):
+    response = client.get(
+        '/service/{}/billing/ft-yearly-usage-summary?year=2016'.format(sample_service.id),
+        headers=[create_authorization_header()]
+    )
+    assert response.status_code == 200
+    assert json.loads(response.get_data(as_text=True)) == []
+
+
+def test_get_yearly_billing_usage_summary_from_ft_billing(client, notify_db_session):
+    service = set_up_yearly_data()
+
+    response = client.get('/service/{}/billing/ft-yearly-usage-summary?year=2016'.format(service.id),
+                          headers=[create_authorization_header()]
+                          )
+    assert response.status_code == 200
+    json_response = json.loads(response.get_data(as_text=True))
+    assert len(json_response) == 3
+    assert json_response[0]['notification_type'] == 'email'
+    assert json_response[0]['billing_units'] == 275
+    assert json_response[0]['rate'] == 0
+    assert json_response[0]['letter_total'] == 0
+    assert json_response[1]['notification_type'] == 'letter'
+    assert json_response[1]['billing_units'] == 275
+    assert json_response[1]['rate'] == 0.33
+    assert json_response[1]['letter_total'] == 90.75
+    assert json_response[2]['notification_type'] == 'sms'
+    assert json_response[2]['billing_units'] == 825
+    assert json_response[2]['rate'] == 0.0162
+    assert json_response[2]['letter_total'] == 0

tests/app/dao/test_ft_billing_dao.py

@@ -7,7 +7,8 @@ from freezegun import freeze_time
 from app import db
 from app.dao.fact_billing_dao import (
     fetch_monthly_billing_for_year, fetch_billing_data_for_day, get_rates_for_billing,
-    get_rate
+    get_rate,
+    fetch_billing_totals_for_year,
 )
 from app.models import FactBilling
 from app.utils import convert_utc_to_bst
@@ -21,6 +22,34 @@ from tests.app.db import (
 )
 
 
+def set_up_yearly_data():
+    service = create_service()
+    sms_template = create_template(service=service, template_type="sms")
+    email_template = create_template(service=service, template_type="email")
+    letter_template = create_template(service=service, template_type="letter")
+    for year in (2016, 2017):
+        for month in range(1, 13):
+            mon = str(month).zfill(2)
+            for day in range(1, monthrange(year, month)[1] + 1):
+                d = str(day).zfill(2)
+                create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d),
+                                  service=service,
+                                  template=sms_template,
+                                  notification_type='sms',
+                                  rate=0.162)
+                create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d),
+                                  service=service,
+                                  template=email_template,
+                                  notification_type='email',
+                                  rate=0)
+                create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d),
+                                  service=service,
+                                  template=letter_template,
+                                  notification_type='letter',
+                                  rate=0.33)
+    return service
+
+
 def test_fetch_billing_data_for_today_includes_data_with_the_right_status(notify_db_session):
     service = create_service()
     template = create_template(service=service, template_type="email")
@@ -256,31 +285,7 @@ def test_fetch_monthly_billing_for_year_adds_data_for_today(notify_db_session):
 
 
 def test_fetch_monthly_billing_for_year_return_financial_year(notify_db_session):
-    service = create_service()
+    service = set_up_yearly_data()
-    sms_template = create_template(service=service, template_type="sms")
-    email_template = create_template(service=service, template_type="email")
-    letter_template = create_template(service=service, template_type="letter")
-
-    for year in (2016, 2017):
-        for month in range(1, 13):
-            mon = str(month).zfill(2)
-            for day in range(1, monthrange(year, month)[1] + 1):
-                d = str(day).zfill(2)
-                create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d),
-                                  service=service,
-                                  template=sms_template,
-                                  notification_type='sms',
-                                  rate=0.162)
-                create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d),
-                                  service=service,
-                                  template=email_template,
-                                  notification_type='email',
-                                  rate=0)
-                create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d),
-                                  service=service,
-                                  template=letter_template,
-                                  notification_type='letter',
-                                  rate=0.33)
 
     results = fetch_monthly_billing_for_year(service.id, 2016)
     # returns 3 rows, per month, returns financial year april to end of march
@@ -304,3 +309,27 @@ def test_fetch_monthly_billing_for_year_return_financial_year(notify_db_session)
     assert results[2].rate == Decimal('0.162')
     assert str(results[3].month) == "2016-05-01"
     assert str(results[35].month) == "2017-03-01"
+
+
+def test_fetch_billing_totals_for_year(notify_db_session):
+    service = set_up_yearly_data()
+    results = fetch_billing_totals_for_year(service_id=service.id, year=2016)
+
+    assert len(results) == 3
+    assert results[0].notification_type == 'email'
+    assert results[0].service_id == service.id
+    assert results[0].notifications_sent == 365
+    assert results[0].billable_units == 365
+    assert results[0].rate == Decimal('0')
+
+    assert results[1].notification_type == 'letter'
+    assert results[1].service_id == service.id
+    assert results[1].notifications_sent == 365
+    assert results[1].billable_units == 365
+    assert results[1].rate == Decimal('0.33')
+
+    assert results[2].notification_type == 'sms'
+    assert results[2].service_id == service.id
+    assert results[2].notifications_sent == 365
+    assert results[2].billable_units == 365
+    assert results[2].rate == Decimal('0.162')