Merge pull request #2599 from alphagov/billing-for-all-services

Billing for all services

app/billing/rest.py

@@ -12,7 +12,9 @@ from app.dao.annual_billing_dao import (
     dao_update_annual_billing_for_future_years
 )
 from app.dao.date_util import get_current_financial_year_start_year
-from app.dao.fact_billing_dao import fetch_monthly_billing_for_year, fetch_billing_totals_for_year
+from app.dao.fact_billing_dao import (
+    fetch_monthly_billing_for_year, fetch_billing_totals_for_year,
+)
 
 from app.errors import InvalidRequest
 from app.errors import register_errors

app/commands.py

@@ -21,6 +21,7 @@ from app.celery.nightly_tasks import send_total_sent_notifications_to_performanc
 from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.celery.letters_pdf_tasks import create_letters_pdf
 from app.config import QueueNames
+from app.dao.annual_billing_dao import dao_create_or_update_annual_billing_for_year
 from app.dao.fact_billing_dao import (
     delete_billing_data_for_service_for_day,
     fetch_billing_data_for_day,
@@ -248,29 +249,34 @@ def backfill_processing_time(start_date, end_date):
         send_processing_time_for_start_and_end(process_start_date, process_end_date)
 
 
-@notify_command()
-def populate_annual_billing():
+@notify_command(name='populate-annual-billing')
+@click.option('-y', '--year', required=True, type=int,
+              help="""The year to populate the annual billing data for, i.e. 2019""")
+def populate_annual_billing(year):
     """
-    add annual_billing for 2016, 2017 and 2018.
+    add annual_billing for given year.
     """
-    financial_year = [2016, 2017, 2018]
-
-    for fy in financial_year:
-        populate_data = """
-        INSERT INTO annual_billing(id, service_id, free_sms_fragment_limit, financial_year_start,
-                created_at, updated_at)
-            SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, 250000, {}, '{}', '{}'
-            FROM services
-            WHERE id NOT IN(
-                SELECT service_id
-                FROM annual_billing
-                WHERE financial_year_start={})
-        """.format(fy, datetime.utcnow(), datetime.utcnow(), fy)
-
-        services_result1 = db.session.execute(populate_data)
-        db.session.commit()
-
-        print("Populated annual billing {} for {} services".format(fy, services_result1.rowcount))
+    sql = """
+        Select id from services where active = true
+        except
+        select service_id
+        from annual_billing
+        where financial_year_start = :year
+    """
+    services_without_annual_billing = db.session.execute(sql, {"year": year})
+    for row in services_without_annual_billing:
+        latest_annual_billing = """
+            Select free_sms_fragment_limit
+            from annual_billing
+            where service_id = :service_id
+            order by financial_year_start desc limit 1
+        """
+        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
+        free_allowance = [x[0]for x in free_allowance_rows]
+        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
+        dao_create_or_update_annual_billing_for_year(service_id=row.id,
+                                                     free_sms_fragment_limit=free_allowance[0],
+                                                     financial_year_start=int(year))
 
 
 @notify_command(name='list-routes')

app/dao/date_util.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date, time
 
 from notifications_utils.timezones import convert_bst_to_utc
 import pytz
@@ -61,3 +61,14 @@ def get_current_financial_year_start_year():
     if now < start_date:
         financial_year_start = financial_year_start - 1
     return financial_year_start
+
+
+def get_financial_year_for_datetime(start_date):
+    if type(start_date) == date:
+        start_date = datetime.combine(start_date, time.min)
+
+    year = int(start_date.strftime('%Y'))
+    if start_date < get_april_fools(year):
+        return year - 1
+    else:
+        return year

app/dao/fact_billing_dao.py

@@ -1,12 +1,16 @@
-from datetime import datetime, timedelta, time
+from datetime import datetime, timedelta, time, date
 
 from flask import current_app
 from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
 from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy import func, case, desc, Date, Integer
+from sqlalchemy import func, case, desc, Date, Integer, and_
 
 from app import db
-from app.dao.date_util import get_financial_year
+from app.dao.date_util import (
+    get_financial_year,
+    get_financial_year_for_datetime
+)
+
 from app.models import (
     FactBilling,
     Notification,
@@ -19,11 +23,163 @@ from app.models import (
     NOTIFICATION_STATUS_TYPES_BILLABLE,
     NotificationHistory,
     EMAIL_TYPE,
-    NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS
+    NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS,
+    AnnualBilling,
+    Organisation,
 )
 from app.utils import get_london_midnight_in_utc
 
 
+def fetch_sms_free_allowance_remainder(start_date):
+    # ASSUMPTION: AnnualBilling has been populated for year.
+    billing_year = get_financial_year_for_datetime(start_date)
+    start_of_year = date(billing_year, 4, 1)
+
+    billable_units = func.coalesce(func.sum(FactBilling.billable_units * FactBilling.rate_multiplier), 0)
+
+    query = db.session.query(
+        AnnualBilling.service_id.label("service_id"),
+        AnnualBilling.free_sms_fragment_limit,
+        billable_units.label('billable_units'),
+        func.greatest((AnnualBilling.free_sms_fragment_limit - billable_units).cast(Integer), 0).label('sms_remainder')
+    ).outerjoin(
+        # if there are no ft_billing rows for a service we still want to return the annual billing so we can use the
+        # free_sms_fragment_limit)
+        FactBilling, and_(
+            AnnualBilling.service_id == FactBilling.service_id,
+            FactBilling.bst_date >= start_of_year,
+            FactBilling.bst_date < start_date,
+            FactBilling.notification_type == SMS_TYPE,
+        )
+    ).filter(
+        AnnualBilling.financial_year_start == billing_year,
+    ).group_by(
+        AnnualBilling.service_id,
+        AnnualBilling.free_sms_fragment_limit,
+    )
+    return query
+
+
+def fetch_sms_billing_for_all_services(start_date, end_date):
+
+    # ASSUMPTION: AnnualBilling has been populated for year.
+    free_allowance_remainder = fetch_sms_free_allowance_remainder(start_date).subquery()
+
+    sms_billable_units = func.sum(FactBilling.billable_units * FactBilling.rate_multiplier)
+    sms_remainder = func.coalesce(
+        free_allowance_remainder.c.sms_remainder,
+        free_allowance_remainder.c.free_sms_fragment_limit
+    )
+    chargeable_sms = func.greatest(sms_billable_units - sms_remainder, 0)
+    sms_cost = chargeable_sms * FactBilling.rate
+
+    query = db.session.query(
+        Organisation.name.label('organisation_name'),
+        Organisation.id.label('organisation_id'),
+        Service.name.label("service_name"),
+        Service.id.label("service_id"),
+        free_allowance_remainder.c.free_sms_fragment_limit,
+        FactBilling.rate.label('sms_rate'),
+        sms_remainder.label("sms_remainder"),
+        sms_billable_units.label('sms_billable_units'),
+        chargeable_sms.label("chargeable_billable_sms"),
+        sms_cost.label('sms_cost'),
+    ).select_from(
+        Service
+    ).outerjoin(
+        free_allowance_remainder, Service.id == free_allowance_remainder.c.service_id
+    ).outerjoin(
+        Service.organisation
+    ).join(
+        FactBilling, FactBilling.service_id == Service.id,
+    ).filter(
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == SMS_TYPE,
+    ).group_by(
+        Organisation.name,
+        Organisation.id,
+        Service.id,
+        Service.name,
+        free_allowance_remainder.c.free_sms_fragment_limit,
+        free_allowance_remainder.c.sms_remainder,
+        FactBilling.rate,
+    ).order_by(
+        Organisation.name,
+        Service.name
+    )
+
+    return query.all()
+
+
+def fetch_letter_costs_for_all_services(start_date, end_date):
+    query = db.session.query(
+        Organisation.name.label("organisation_name"),
+        Organisation.id.label("organisation_id"),
+        Service.name.label("service_name"),
+        Service.id.label("service_id"),
+        func.sum(FactBilling.notifications_sent * FactBilling.rate).label("letter_cost")
+    ).select_from(
+        Service
+    ).outerjoin(
+        Service.organisation
+    ).join(
+        FactBilling, FactBilling.service_id == Service.id,
+    ).filter(
+        FactBilling.service_id == Service.id,
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == LETTER_TYPE,
+    ).group_by(
+        Organisation.name,
+        Organisation.id,
+        Service.id,
+        Service.name,
+    ).order_by(
+        Organisation.name,
+        Service.name
+    )
+
+    return query.all()
+
+
+def fetch_letter_line_items_for_all_services(start_date, end_date):
+    query = db.session.query(
+        Organisation.name.label("organisation_name"),
+        Organisation.id.label("organisation_id"),
+        Service.name.label("service_name"),
+        Service.id.label("service_id"),
+        FactBilling.billable_units.label("sheet_count"),
+        FactBilling.rate.label("letter_rate"),
+        FactBilling.postage.label("postage"),
+        func.sum(FactBilling.notifications_sent).label("letters_sent"),
+    ).select_from(
+        Service
+    ).outerjoin(
+        Service.organisation
+    ).join(
+        FactBilling, FactBilling.service_id == Service.id,
+    ).filter(
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+        FactBilling.notification_type == LETTER_TYPE,
+    ).group_by(
+        Organisation.name,
+        Organisation.id,
+        Service.id,
+        Service.name,
+        FactBilling.billable_units,
+        FactBilling.rate,
+        FactBilling.postage
+    ).order_by(
+        Organisation.name,
+        Service.name,
+        FactBilling.postage.desc(),
+        FactBilling.rate,
+    )
+    return query.all()
+
+
 def fetch_billing_totals_for_year(service_id, year):
     year_start_date, year_end_date = get_financial_year(year)
     """

app/platform_stats/rest.py

@@ -2,11 +2,17 @@ from datetime import datetime
 
 from flask import Blueprint, jsonify, request
 
+from app.dao.date_util import get_financial_year_for_datetime
+from app.dao.fact_billing_dao import (
+    fetch_sms_billing_for_all_services, fetch_letter_costs_for_all_services,
+    fetch_letter_line_items_for_all_services
+)
 from app.dao.fact_notification_status_dao import fetch_notification_status_totals_for_all_services
-from app.errors import register_errors
+from app.errors import register_errors, InvalidRequest
 from app.platform_stats.platform_stats_schema import platform_stats_request
 from app.service.statistics import format_admin_stats
 from app.schema_validation import validate
+from app.utils import get_london_midnight_in_utc
 
 platform_stats_blueprint = Blueprint('platform_stats', __name__)
 
@@ -27,3 +33,76 @@ def get_platform_stats():
     stats = format_admin_stats(data)
 
     return jsonify(stats)
+
+
+def validate_date_range_is_within_a_financial_year(start_date, end_date):
+    try:
+        start_date = datetime.strptime(start_date, "%Y-%m-%d").date()
+        end_date = datetime.strptime(end_date, "%Y-%m-%d").date()
+    except ValueError:
+        raise InvalidRequest(message="Input must be a date in the format: YYYY-MM-DD", status_code=400)
+    if end_date < start_date:
+        raise InvalidRequest(message="Start date must be before end date", status_code=400)
+
+    start_fy = get_financial_year_for_datetime(get_london_midnight_in_utc(start_date))
+    end_fy = get_financial_year_for_datetime(get_london_midnight_in_utc(end_date))
+
+    if start_fy != end_fy:
+        raise InvalidRequest(message="Date must be in a single financial year.", status_code=400)
+
+    return start_date, end_date
+
+
+@platform_stats_blueprint.route('usage-for-all-services')
+def get_usage_for_all_services():
+    start_date = request.args.get('start_date')
+    end_date = request.args.get('end_date')
+
+    start_date, end_date = validate_date_range_is_within_a_financial_year(start_date, end_date)
+
+    sms_costs = fetch_sms_billing_for_all_services(start_date, end_date)
+    letter_costs = fetch_letter_costs_for_all_services(start_date, end_date)
+    letter_breakdown = fetch_letter_line_items_for_all_services(start_date, end_date)
+
+    lb_by_service = [
+        (lb.service_id, "{} {} class letters at {}p".format(lb.letters_sent, lb.postage, int(lb.letter_rate * 100)))
+        for lb in letter_breakdown
+    ]
+    combined = {}
+    for s in sms_costs:
+        entry = {
+            "organisation_id": str(s.organisation_id) if s.organisation_id else "",
+            "organisation_name": s.organisation_name or "",
+            "service_id": str(s.service_id),
+            "service_name": s.service_name,
+            "sms_cost": float(s.sms_cost),
+            "sms_fragments": s.chargeable_billable_sms,
+            "letter_cost": 0,
+            "letter_breakdown": ""
+        }
+        combined[s.service_id] = entry
+
+    for l in letter_costs:
+        if l.service_id in combined:
+            combined[l.service_id].update({'letter_cost': float(l.letter_cost)})
+        else:
+            letter_entry = {
+                "organisation_id": str(l.organisation_id) if l.organisation_id else "",
+                "organisation_name": l.organisation_name or "",
+                "service_id": str(l.service_id),
+                "service_name": l.service_name,
+                "sms_cost": 0,
+                "sms_fragments": 0,
+                "letter_cost": float(l.letter_cost),
+                "letter_breakdown": ""
+            }
+            combined[l.service_id] = letter_entry
+    for service_id, breakdown in lb_by_service:
+        combined[service_id]['letter_breakdown'] += (breakdown + '\n')
+
+    # sorting first by name == '' means that blank orgs will be sorted last.
+    return jsonify(sorted(combined.values(), key=lambda x: (
+        x['organisation_name'] == '',
+        x['organisation_name'],
+        x['service_name']
+    )))