Merge branch 'use-ft-billing-for-usage' of github.com:alphagov/notifications-api

app/billing/billing_schemas.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+
 create_or_update_free_sms_fragment_limit_schema = {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST annual billing schema",
@@ -24,3 +25,17 @@ def serialize_ft_billing_remove_emails(data):
         }
         results.append(json_result)
     return results
+
+
+def serialize_ft_billing_yearly_totals(data):
+    yearly_totals = []
+    for total in data:
+        json_result = {
+            "notification_type": total.notification_type,
+            "billing_units": total.billable_units,
+            "rate": float(total.rate),
+            "letter_total": float(total.billable_units * total.rate) if total.notification_type == 'letter' else 0
+        }
+        yearly_totals.append(json_result)
+
+    return yearly_totals

app/billing/rest.py

@@ -5,7 +5,8 @@ from flask import Blueprint, jsonify, request
 
 from app.billing.billing_schemas import (
     create_or_update_free_sms_fragment_limit_schema,
-    serialize_ft_billing_remove_emails
+    serialize_ft_billing_remove_emails,
+    serialize_ft_billing_yearly_totals,
 )
 from app.dao.annual_billing_dao import (
     dao_get_free_sms_fragment_limit_for_year,
@@ -15,7 +16,7 @@ from app.dao.annual_billing_dao import (
 )
 from app.dao.date_util import get_current_financial_year_start_year
 from app.dao.date_util import get_months_for_financial_year
-from app.dao.fact_billing_dao import fetch_monthly_billing_for_year
+from app.dao.fact_billing_dao import fetch_monthly_billing_for_year, fetch_billing_totals_for_year
 from app.dao.monthly_billing_dao import (
     get_billing_data_for_financial_year,
     get_monthly_billing_by_notification_type
@@ -47,6 +48,18 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     return jsonify(data)
 
 
+@billing_blueprint.route('/ft-yearly-usage-summary')
+def get_yearly_billing_usage_summary_from_ft_billing(service_id):
+    try:
+        year = int(request.args.get('year'))
+    except TypeError:
+        return jsonify(result='error', message='No valid year provided'), 400
+
+    billing_data = fetch_billing_totals_for_year(service_id, year)
+    data = serialize_ft_billing_yearly_totals(billing_data)
+    return jsonify(data)
+
+
 @billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_month(service_id):
     try:
@@ -70,7 +83,7 @@ def get_yearly_billing_usage_summary(service_id):
     try:
         year = int(request.args.get('year'))
         billing_data = get_billing_data_for_financial_year(service_id, year)
-        notification_types = [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]
+        notification_types = [EMAIL_TYPE, LETTER_TYPE, SMS_TYPE]
         response = [
             _get_total_billable_units_and_rate_for_notification_type(billing_data, notification_type)
             for notification_type in notification_types
@@ -102,8 +115,8 @@ def _get_total_billable_units_and_rate_for_notification_type(billing_data, noti_
     return {
         "notification_type": noti_type,
         "billing_units": total_sent,
-        "rate": rate,
-        "letter_total": letter_total
+        "rate": float(rate),
+        "letter_total": round(float(letter_total), 3)
     }
 
 
@@ -132,7 +145,7 @@ def _transform_billing_for_month_letters(billing_for_month):
             "month": month_name,
             "billing_units": (total['billing_units'] * total['rate_multiplier']),
             "notification_type": billing_for_month.notification_type,
-            "rate": total['rate']
+            "rate": float(total['rate'])
         }
         x.append(y)
     if len(billing_for_month.monthly_totals) == 0:

app/celery/reporting_tasks.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta
 
+from flask import current_app
 from notifications_utils.statsd_decorators import statsd
 
 from app import notify_celery
@@ -24,3 +25,6 @@ def create_nightly_billing(day_start=None):
 
         for data in transit_data:
             update_fact_billing(data, process_day)
+
+        current_app.logger.info(
+            "create-nightly-billing task complete. {} rows updated for day: {}".format(len(transit_data), process_day))

app/commands.py

@@ -551,19 +551,26 @@ def populate_redis_template_usage(service_id, day):
         )
 
 
-@notify_command(name='rebuild-ft-billing-for-month-and-service')
-@click.option('-s', '--service_id', required=True, type=click.UUID)
+@notify_command(name='rebuild-ft-billing-for-day')
+@click.option('-s', '--service_id', required=False, type=click.UUID)
 @click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True,
               type=click_dt(format='%Y-%m-%d'))
-def rebuild_ft_billing_for_month_and_service(service_id, day):
+def rebuild_ft_billing_for_day(service_id, day):
     """
     Rebuild the data in ft_billing for the given service_id and date
     """
-    # confirm the service exists
-    dao_fetch_service_by_id(service_id)
-    transit_data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
-    for data in transit_data:
-        update_fact_billing(data, day)
+    def rebuild_ft_data(process_day, service):
+        transit_data = fetch_billing_data_for_day(process_day=process_day, service_id=service)
+        for data in transit_data:
+            update_fact_billing(data, process_day)
+    if service_id:
+        # confirm the service exists
+        dao_fetch_service_by_id(service_id)
+        rebuild_ft_data(day, service_id)
+    else:
+        services = get_service_ids_that_need_billing_populated(day, day)
+        for service_id in services:
+            rebuild_ft_data(day, service_id)
 
 
 @notify_command(name='compare-ft-billing-to-monthly-billing')

app/dao/fact_billing_dao.py

@@ -1,6 +1,5 @@
 from datetime import datetime, timedelta, time
 
-from flask import current_app
 from sqlalchemy import func, case, desc, Date
 
 from app import db
@@ -20,6 +19,31 @@ from app.models import (
 from app.utils import convert_utc_to_bst, convert_bst_to_utc
 
 
+def fetch_billing_totals_for_year(service_id, year):
+    year_start_date, year_end_date = get_financial_year(year)
+
+    yearly_data = db.session.query(
+        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
+        FactBilling.service_id,
+        FactBilling.rate,
+        FactBilling.notification_type
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date
+    ).group_by(
+        FactBilling.service_id,
+        FactBilling.rate,
+        FactBilling.notification_type
+    ).order_by(
+        FactBilling.service_id,
+        FactBilling.notification_type
+    ).all()
+
+    return yearly_data
+
+
 def fetch_monthly_billing_for_year(service_id, year):
     year_start_date, year_end_date = get_financial_year(year)
     utcnow = datetime.utcnow()
@@ -97,6 +121,7 @@ def fetch_billing_data_for_day(process_day, service_id=None):
     )
     if service_id:
         transit_data = transit_data.filter(Notification.service_id == service_id)
+
     return transit_data.all()
 
 
@@ -121,7 +146,6 @@ def update_fact_billing(data, process_day):
     inserted_records = 0
     updated_records = 0
     non_letter_rates, letter_rates = get_rates_for_billing()
-    print("process_day: {} {}".format(type(process_day), process_day))
     update_count = FactBilling.query.filter(
         FactBilling.bst_date == datetime.date(process_day),
         FactBilling.template_id == data.template_id,
@@ -147,8 +171,6 @@ def update_fact_billing(data, process_day):
         inserted_records += 1
     updated_records += update_count
     db.session.commit()
-    current_app.logger.info('ft_billing for {}: {} rows updated, {} rows inserted'
-                            .format(process_day, updated_records, inserted_records))
 
 
 def create_billing_record(data, rate, process_day):

app/models.py

@@ -1784,10 +1784,10 @@ class FactBilling(db.Model):
     service_id = db.Column(UUID(as_uuid=True), nullable=False, index=True)
     notification_type = db.Column(db.Text, nullable=False, primary_key=True)
     provider = db.Column(db.Text, nullable=True, primary_key=True)
-    rate_multiplier = db.Column(db.Numeric(), nullable=True, primary_key=True)
+    rate_multiplier = db.Column(db.Integer(), nullable=True, primary_key=True)
     international = db.Column(db.Boolean, nullable=False, primary_key=False)
     rate = db.Column(db.Numeric(), nullable=True)
-    billable_units = db.Column(db.Numeric(), nullable=True)
+    billable_units = db.Column(db.Integer(), nullable=True)
     notifications_sent = db.Column(db.Integer(), nullable=True)