Merge pull request #1863 from alphagov/use-ft-billing-for-usage

Use ft billing for usage

app/billing/billing_schemas.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 create_or_update_free_sms_fragment_limit_schema = {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST annual billing schema",
@@ -8,3 +10,17 @@ create_or_update_free_sms_fragment_limit_schema = {
     },
     "required": ["free_sms_fragment_limit"]
 }
+
+
+def serialize_ft_billing_remove_emails(data):
+    results = []
+    billed_notifications = [x for x in data if x.notification_type != 'email']
+    for notifications in billed_notifications:
+        json_result = {
+            "month": (datetime.strftime(notifications.month, "%B")),
+            "notification_type": notifications.notification_type,
+            "billing_units": int(notifications.billable_units),
+            "rate": float(notifications.rate),
+        }
+        results.append(json_result)
+    return results

app/billing/rest.py

@@ -1,24 +1,30 @@
-from datetime import datetime
 import json
+from datetime import datetime
 
 from flask import Blueprint, jsonify, request
 
+from app.billing.billing_schemas import (
+    create_or_update_free_sms_fragment_limit_schema,
+    serialize_ft_billing_remove_emails
+)
+from app.dao.annual_billing_dao import (
+    dao_get_free_sms_fragment_limit_for_year,
+    dao_get_all_free_sms_fragment_limit,
+    dao_create_or_update_annual_billing_for_year,
+    dao_update_annual_billing_for_future_years
+)
+from app.dao.date_util import get_current_financial_year_start_year
+from app.dao.date_util import get_months_for_financial_year
+from app.dao.fact_billing_dao import fetch_monthly_billing_for_year
 from app.dao.monthly_billing_dao import (
     get_billing_data_for_financial_year,
     get_monthly_billing_by_notification_type
 )
-from app.dao.date_util import get_months_for_financial_year
+from app.errors import InvalidRequest
 from app.errors import register_errors
 from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE
-from app.utils import convert_utc_to_bst
-from app.dao.annual_billing_dao import (dao_get_free_sms_fragment_limit_for_year,
-                                        dao_get_all_free_sms_fragment_limit,
-                                        dao_create_or_update_annual_billing_for_year,
-                                        dao_update_annual_billing_for_future_years)
-from app.billing.billing_schemas import create_or_update_free_sms_fragment_limit_schema
-from app.errors import InvalidRequest
 from app.schema_validation import validate
-from app.dao.date_util import get_current_financial_year_start_year
+from app.utils import convert_utc_to_bst
 
 billing_blueprint = Blueprint(
     'billing',
@@ -30,19 +36,30 @@ billing_blueprint = Blueprint(
 register_errors(billing_blueprint)
 
 
+@billing_blueprint.route('/ft-monthly-usage')
+def get_yearly_usage_by_monthly_from_ft_billing(service_id):
+    try:
+        year = int(request.args.get('year'))
+    except TypeError:
+        return jsonify(result='error', message='No valid year provided'), 400
+    results = fetch_monthly_billing_for_year(service_id=service_id, year=year)
+    data = serialize_ft_billing_remove_emails(results)
+    return jsonify(data)
+
+
 @billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_month(service_id):
     try:
         year = int(request.args.get('year'))
         results = []
         for month in get_months_for_financial_year(year):
-            billing_for_month = get_monthly_billing_by_notification_type(service_id, month, SMS_TYPE)
-            if billing_for_month:
-                results.append(_transform_billing_for_month_sms(billing_for_month))
             letter_billing_for_month = get_monthly_billing_by_notification_type(service_id, month, LETTER_TYPE)
             if letter_billing_for_month:
                 results.extend(_transform_billing_for_month_letters(letter_billing_for_month))
-        return json.dumps(results)
+            billing_for_month = get_monthly_billing_by_notification_type(service_id, month, SMS_TYPE)
+            if billing_for_month:
+                results.append(_transform_billing_for_month_sms(billing_for_month))
+        return jsonify(results)
 
     except TypeError:
         return jsonify(result='error', message='No valid year provided'), 400

app/commands.py

@@ -7,16 +7,18 @@ from decimal import Decimal
 import click
 import flask
 from click_datetime import Datetime as click_dt
-from flask import current_app
+from flask import current_app, json
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy import func
 from notifications_utils.statsd_decorators import statsd
 
 from app import db, DATETIME_FORMAT, encryption, redis_store
+from app.billing.rest import get_yearly_usage_by_month, get_yearly_usage_by_monthly_from_ft_billing
 from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
 from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.celery.letters_pdf_tasks import create_letters_pdf
 from app.config import QueueNames
+from app.dao.date_util import get_financial_year
 from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing
 from app.dao.monthly_billing_dao import (
     create_or_update_monthly_billing,
@@ -562,3 +564,42 @@ def rebuild_ft_billing_for_month_and_service(service_id, day):
     transit_data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
     for data in transit_data:
         update_fact_billing(data, day)
+
+
+@notify_command(name='compare-ft-billing-to-monthly-billing')
+@click.option('-y', '--year', required=True)
+@click.option('-s', '--service_id', required=False, type=click.UUID)
+def compare_ft_billing_to_monthly_billing(year, service_id=None):
+    """
+    This command checks the results of monthly_billing to ft_billing for the given year.
+    If service id is not included all services are compared for the given year.
+    """
+
+    def compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response):
+        # Remove the rows with 0 billing_units and rate, ft_billing doesn't populate those rows.
+        mo_json = json.loads(monthly_billing_response.get_data(as_text=True))
+        rm_zero_rows = [x for x in mo_json if x['billing_units'] != 0 and x['rate'] != 0]
+        try:
+            assert rm_zero_rows == json.loads(ft_billing_response.get_data(as_text=True))
+        except AssertionError:
+            print("Comparison failed for service: {} and year: {}".format(service_id, year))
+
+    if not service_id:
+        start_date, end_date = get_financial_year(year=int(year))
+        services = get_service_ids_that_need_billing_populated(start_date, end_date)
+        for service_id in services:
+            with current_app.test_request_context(
+                    path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
+                monthly_billing_response = get_yearly_usage_by_month(service_id)
+            with current_app.test_request_context(
+                    path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
+                ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
+            compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)
+    else:
+        with current_app.test_request_context(
+                path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
+            monthly_billing_response = get_yearly_usage_by_month(service_id)
+        with current_app.test_request_context(
+                path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
+            ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
+        compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)

app/dao/fact_billing_dao.py

@@ -1,7 +1,7 @@
 from datetime import datetime, timedelta, time
 
 from flask import current_app
-from sqlalchemy import func, case, desc, extract
+from sqlalchemy import func, case, desc, Date
 
 from app import db
 from app.dao.date_util import get_financial_year
@@ -20,12 +20,12 @@ from app.models import (
 from app.utils import convert_utc_to_bst, convert_bst_to_utc
 
 
-def fetch_montly_billing_for_year(service_id, year):
+def fetch_monthly_billing_for_year(service_id, year):
     year_start_date, year_end_date = get_financial_year(year)
     utcnow = datetime.utcnow()
-    today = convert_utc_to_bst(utcnow).date()
+    today = convert_utc_to_bst(utcnow)
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
-    if year_end_date.date() >= today:
+    if year_end_date >= today:
         yesterday = today - timedelta(days=1)
         for day in [yesterday, today]:
             data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
@@ -33,23 +33,25 @@ def fetch_montly_billing_for_year(service_id, year):
                 update_fact_billing(data=d, process_day=day)
 
     yearly_data = db.session.query(
-        extract('month', FactBilling.bst_date).label("Month"),
+        func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
         func.sum(FactBilling.notifications_sent).label("notifications_sent"),
-        func.sum(FactBilling.billable_units).label("billable_units"),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
         FactBilling.service_id,
         FactBilling.rate,
-        FactBilling.rate_multiplier,
-        FactBilling.international
+        FactBilling.notification_type
     ).filter(
         FactBilling.service_id == service_id,
         FactBilling.bst_date >= year_start_date,
         FactBilling.bst_date <= year_end_date
     ).group_by(
-        'Month',
+        'month',
         FactBilling.service_id,
         FactBilling.rate,
-        FactBilling.rate_multiplier,
-        FactBilling.international
+        FactBilling.notification_type
+    ).order_by(
+        FactBilling.service_id,
+        'month',
+        FactBilling.notification_type
     ).all()
 
     return yearly_data
@@ -119,7 +121,7 @@ def update_fact_billing(data, process_day):
     inserted_records = 0
     updated_records = 0
     non_letter_rates, letter_rates = get_rates_for_billing()
-
+    print("process_day: {} {}".format(type(process_day), process_day))
     update_count = FactBilling.query.filter(
         FactBilling.bst_date == datetime.date(process_day),
         FactBilling.template_id == data.template_id,

app/dao/monthly_billing_dao.py

@@ -100,7 +100,8 @@ def get_yearly_billing_data_for_date_range(
         MonthlyBilling.end_date <= end_date,
         MonthlyBilling.notification_type.in_(notification_types)
     ).order_by(
-        MonthlyBilling.notification_type
+        MonthlyBilling.start_date,
+        MonthlyBilling.notification_type,
     ).all()
 
     return results