- Only rebuild current month for monthly_billing if today is in the current year.

- Change the usage queries to a union so that billing_units is correct for all notification types. Removing the business logic from the schema.
- Added tests for different fragment counts, rates and sheet counts.

app/billing/billing_schemas.py

@@ -16,12 +16,12 @@ create_or_update_free_sms_fragment_limit_schema = {
 def serialize_ft_billing_remove_emails(data):
     results = []
     billed_notifications = [x for x in data if x.notification_type != 'email']
-    for notifications in billed_notifications:
+    for notification in billed_notifications:
         json_result = {
-            "month": (datetime.strftime(notifications.month, "%B")),
-            "notification_type": notifications.notification_type,
-            "billing_units": int(notifications.billable_units),
-            "rate": float(notifications.rate),
+            "month": (datetime.strftime(notification.month, "%B")),
+            "notification_type": notification.notification_type,
+            "billing_units": notification.billable_units,
+            "rate": float(notification.rate),
         }
         results.append(json_result)
     return results

app/dao/fact_billing_dao.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta, time
 
+from flask import current_app
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy import func, case, desc, Date
 
@@ -16,31 +17,55 @@ from app.models import (
     SMS_TYPE,
     Rate,
     LetterRate,
-    NotificationHistory
+    NotificationHistory,
+    EMAIL_TYPE
 )
 from app.utils import convert_utc_to_bst, convert_bst_to_utc
 
 
 def fetch_billing_totals_for_year(service_id, year):
     year_start_date, year_end_date = get_financial_year(year)
-
-    yearly_data = db.session.query(
+    """
+      Billing for email: only record the total number of emails.
+      Billing for letters: The billing units is used to fetch the correct rate for the sheet count of the letter. 
+            Total cost is notifications_sent * rate.
+      Rate multiplier does not apply to email or letters.
+    """
+    email_and_letters = db.session.query(
+        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
+        func.sum(FactBilling.notifications_sent).label("billable_units"),
+        FactBilling.rate.label('rate'),
+        FactBilling.notification_type.label('notification_type')
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date,
+        FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE])
+    ).group_by(
+        FactBilling.rate,
+        FactBilling.notification_type
+    )
+    """
+    Billing for SMS using the billing_units * rate_multiplier. Billing unit of SMS is the fragment count of a message
+    """
+    sms = db.session.query(
         func.sum(FactBilling.notifications_sent).label("notifications_sent"),
         func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
-        FactBilling.service_id,
         FactBilling.rate,
         FactBilling.notification_type
     ).filter(
         FactBilling.service_id == service_id,
         FactBilling.bst_date >= year_start_date,
-        FactBilling.bst_date <= year_end_date
+        FactBilling.bst_date <= year_end_date,
+        FactBilling.notification_type == SMS_TYPE
     ).group_by(
-        FactBilling.service_id,
         FactBilling.rate,
         FactBilling.notification_type
-    ).order_by(
-        FactBilling.service_id,
-        FactBilling.notification_type
+    )
+
+    yearly_data = email_and_letters.union_all(sms).order_by(
+        'notification_type',
+        'rate'
     ).all()
 
     return yearly_data
@@ -58,26 +83,44 @@ def fetch_monthly_billing_for_year(service_id, year):
             for d in data:
                 update_fact_billing(data=d, process_day=day)
 
-    yearly_data = db.session.query(
+    email_and_letters = db.session.query(
+        func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
+        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
+        func.sum(FactBilling.notifications_sent).label("billable_units"),
+        FactBilling.rate.label('rate'),
+        FactBilling.notification_type.label('notification_type')
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start_date,
+        FactBilling.bst_date <= year_end_date,
+        FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE])
+    ).group_by(
+        'month',
+        FactBilling.rate,
+        FactBilling.notification_type
+    )
+
+    sms = db.session.query(
         func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
         func.sum(FactBilling.notifications_sent).label("notifications_sent"),
         func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
-        FactBilling.service_id,
         FactBilling.rate,
         FactBilling.notification_type
     ).filter(
         FactBilling.service_id == service_id,
         FactBilling.bst_date >= year_start_date,
-        FactBilling.bst_date <= year_end_date
+        FactBilling.bst_date <= year_end_date,
+        FactBilling.notification_type == SMS_TYPE
     ).group_by(
         'month',
-        FactBilling.service_id,
         FactBilling.rate,
         FactBilling.notification_type
-    ).order_by(
-        FactBilling.service_id,
+    )
+
+    yearly_data = email_and_letters.union_all(sms).order_by(
         'month',
-        FactBilling.notification_type
+        'notification_type',
+        'rate'
     ).all()
 
     return yearly_data
@@ -88,6 +131,7 @@ def fetch_billing_data_for_day(process_day, service_id=None):
     end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
     # use notification_history if process day is older than 7 days
     # this is useful if we need to rebuild the ft_billing table for a date older than 7 days ago.
+    current_app.logger.info("Populate ft_billing for {} to {}".format(start_date, end_date))
     table = Notification
     if start_date < datetime.utcnow() - timedelta(days=7):
         table = NotificationHistory

app/dao/monthly_billing_dao.py

@@ -116,11 +116,11 @@ def get_monthly_billing_by_notification_type(service_id, billing_month, notifica
 
 @statsd(namespace="dao")
 def get_billing_data_for_financial_year(service_id, year, notification_types=[SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]):
-    # Update totals to the latest so we include data for today
     now = convert_utc_to_bst(datetime.utcnow())
-    create_or_update_monthly_billing(service_id=service_id, billing_month=now)
-
     start_date, end_date = get_financial_year(year)
+    if start_date <= now <= end_date:
+        # Update totals to the latest so we include data for today
+        create_or_update_monthly_billing(service_id=service_id, billing_month=now)
 
     results = get_yearly_billing_data_for_date_range(
         service_id, start_date, end_date, notification_types

tests/app/billing/test_billing.py

@@ -522,6 +522,7 @@ def set_up_yearly_data():
                               service=service,
                               template=email_template,
                               notification_type='email',
+                              billable_unit=0,
                               rate=0)
             create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
                               service=service,
@@ -559,7 +560,7 @@ def set_up_yearly_data():
     return service
 
 
-def test_get_yearly_billing_usage_summary_from_ft_billing_comapre_to_monthyl_billing(
+def test_get_yearly_billing_usage_summary_from_ft_billing_compare_to_monthly_billing(
         client, notify_db_session
 ):
     service = set_up_yearly_data()
@@ -621,3 +622,147 @@ def test_get_yearly_billing_usage_summary_from_ft_billing(client, notify_db_sess
     assert json_response[2]['billing_units'] == 825
     assert json_response[2]['rate'] == 0.0162
     assert json_response[2]['letter_total'] == 0
+
+
+def test_get_yearly_usage_by_monthly_from_ft_billing_all_cases(client, notify_db_session):
+    service = set_up_data_for_all_cases()
+    response = client.get('service/{}/billing/ft-monthly-usage?year=2018'.format(service.id),
+                          headers=[('Content-Type', 'application/json'), create_authorization_header()])
+
+    assert response.status_code == 200
+    json_response = json.loads(response.get_data(as_text=True))
+    assert len(json_response) == 5
+    assert json_response[0]['month'] == 'May'
+    assert json_response[0]['notification_type'] == 'letter'
+    assert json_response[0]['rate'] == 0.33
+    assert json_response[0]['billing_units'] == 1
+
+    assert json_response[1]['month'] == 'May'
+    assert json_response[1]['notification_type'] == 'letter'
+    assert json_response[1]['rate'] == 0.36
+    assert json_response[1]['billing_units'] == 1
+
+    assert json_response[2]['month'] == 'May'
+    assert json_response[2]['notification_type'] == 'letter'
+    assert json_response[2]['rate'] == 0.39
+    assert json_response[2]['billing_units'] == 1
+
+    assert json_response[3]['month'] == 'May'
+    assert json_response[3]['notification_type'] == 'sms'
+    assert json_response[3]['rate'] == 0.0150
+    assert json_response[3]['billing_units'] == 4
+
+    assert json_response[4]['month'] == 'May'
+    assert json_response[4]['notification_type'] == 'sms'
+    assert json_response[4]['rate'] == 0.162
+    assert json_response[4]['billing_units'] == 5
+
+
+def test_get_yearly_billing_usage_summary_from_ft_billing_all_cases(client, notify_db_session):
+    service = set_up_data_for_all_cases()
+    response = client.get('/service/{}/billing/ft-yearly-usage-summary?year=2018'.format(service.id),
+                          headers=[create_authorization_header()])
+    assert response.status_code == 200
+    json_response = json.loads(response.get_data(as_text=True))
+
+    assert len(json_response) == 6
+    assert json_response[0]["notification_type"] == 'email'
+    assert json_response[0]["billing_units"] == 1
+    assert json_response[0]["rate"] == 0
+    assert json_response[0]["letter_total"] == 0
+
+    assert json_response[1]["notification_type"] == 'letter'
+    assert json_response[1]["billing_units"] == 1
+    assert json_response[1]["rate"] == 0.33
+    assert json_response[1]["letter_total"] == 0.33
+
+    assert json_response[2]["notification_type"] == 'letter'
+    assert json_response[2]["billing_units"] == 1
+    assert json_response[2]["rate"] == 0.36
+    assert json_response[2]["letter_total"] == 0.36
+
+    assert json_response[3]["notification_type"] == 'letter'
+    assert json_response[3]["billing_units"] == 1
+    assert json_response[3]["rate"] == 0.39
+    assert json_response[3]["letter_total"] == 0.39
+
+    assert json_response[4]["notification_type"] == 'sms'
+    assert json_response[4]["billing_units"] == 4
+    assert json_response[4]["rate"] == 0.0150
+    assert json_response[4]["letter_total"] == 0
+
+    assert json_response[5]["notification_type"] == 'sms'
+    assert json_response[5]["billing_units"] == 5
+    assert json_response[5]["rate"] == 0.162
+    assert json_response[5]["letter_total"] == 0
+
+
+def set_up_data_for_all_cases():
+    service = create_service()
+    sms_template = create_template(service=service, template_type="sms")
+    email_template = create_template(service=service, template_type="email")
+    letter_template = create_template(service=service, template_type="letter")
+    create_ft_billing(bst_date='2018-05-16',
+                      notification_type='sms',
+                      template=sms_template,
+                      service=service,
+                      rate_multiplier=1,
+                      international=False,
+                      rate=0.162,
+                      billable_unit=1,
+                      notifications_sent=1)
+    create_ft_billing(bst_date='2018-05-17',
+                      notification_type='sms',
+                      template=sms_template,
+                      service=service,
+                      rate_multiplier=2,
+                      international=False,
+                      rate=0.162,
+                      billable_unit=2,
+                      notifications_sent=1)
+    create_ft_billing(bst_date='2018-05-16',
+                      notification_type='sms',
+                      template=sms_template,
+                      service=service,
+                      rate_multiplier=2,
+                      international=False,
+                      rate=0.0150,
+                      billable_unit=2,
+                      notifications_sent=1)
+    create_ft_billing(bst_date='2018-05-16',
+                      notification_type='email',
+                      template=email_template,
+                      service=service,
+                      rate_multiplier=1,
+                      international=False,
+                      rate=0,
+                      billable_unit=0,
+                      notifications_sent=1)
+    create_ft_billing(bst_date='2018-05-16',
+                      notification_type='letter',
+                      template=letter_template,
+                      service=service,
+                      rate_multiplier=1,
+                      international=False,
+                      rate=0.33,
+                      billable_unit=1,
+                      notifications_sent=1)
+    create_ft_billing(bst_date='2018-05-17',
+                      notification_type='letter',
+                      template=letter_template,
+                      service=service,
+                      rate_multiplier=1,
+                      international=False,
+                      rate=0.36,
+                      billable_unit=2,
+                      notifications_sent=1)
+    create_ft_billing(bst_date='2018-05-18',
+                      notification_type='letter',
+                      template=letter_template,
+                      service=service,
+                      rate_multiplier=1,
+                      international=False,
+                      rate=0.39,
+                      billable_unit=3,
+                      notifications_sent=1)
+    return service

tests/app/dao/test_ft_billing_dao.py

@@ -266,14 +266,12 @@ def test_fetch_monthly_billing_for_year(notify_db_session):
     assert str(results[0].month) == "2018-06-01"
     assert results[0].notifications_sent == 30
     assert results[0].billable_units == Decimal('60')
-    assert results[0].service_id == service.id
     assert results[0].rate == Decimal('0.162')
     assert results[0].notification_type == 'sms'
 
     assert str(results[1].month) == "2018-07-01"
     assert results[1].notifications_sent == 31
     assert results[1].billable_units == Decimal('31')
-    assert results[1].service_id == service.id
     assert results[1].rate == Decimal('0.158')
     assert results[1].notification_type == 'sms'
 
@@ -330,19 +328,16 @@ def test_fetch_billing_totals_for_year(notify_db_session):
 
     assert len(results) == 3
     assert results[0].notification_type == 'email'
-    assert results[0].service_id == service.id
     assert results[0].notifications_sent == 365
     assert results[0].billable_units == 365
     assert results[0].rate == Decimal('0')
 
     assert results[1].notification_type == 'letter'
-    assert results[1].service_id == service.id
     assert results[1].notifications_sent == 365
     assert results[1].billable_units == 365
     assert results[1].rate == Decimal('0.33')
 
     assert results[2].notification_type == 'sms'
-    assert results[2].service_id == service.id
     assert results[2].notifications_sent == 365
     assert results[2].billable_units == 365
     assert results[2].rate == Decimal('0.162')

tests/app/dao/test_monthly_billing.py

@@ -511,3 +511,13 @@ def test_get_yearly_billing_data_for_year_includes_current_day_totals(sample_tem
     )
 
     assert billing_data[0].monthly_totals[0]['billing_units'] == 3
+
+
+@freeze_time("2017-06-16 13:00:00")
+def test_get_billing_data_for_financial_year_updated_monthly_billing_if_today_is_in_current_year(
+        sample_service,
+        mocker
+):
+    mock = mocker.patch("app.dao.monthly_billing_dao.create_or_update_monthly_billing")
+    get_billing_data_for_financial_year(sample_service.id, 2016)
+    mock.assert_not_called()