Update monthly billing for SMS and Email:

* Refactor code to update for EMAIL_TYPE and SMS_TYPE
* Make tests more robust (overlapping rates, multiple months, multiple noti
* types)
*

app/dao/monthly_billing_dao.py

@@ -2,38 +2,64 @@ from datetime import datetime
 
 from app import db
 from app.dao.dao_utils import transactional
-from app.dao.date_util import get_month_start_end_date
+from app.dao.date_util import get_month_start_and_end_date_in_utc, get_financial_year
 from app.dao.notification_usage_dao import get_billing_data_for_month
-from app.models import MonthlyBilling, SMS_TYPE, NotificationHistory
+from app.models import (
+    SMS_TYPE,
+    EMAIL_TYPE,
+    MonthlyBilling,
+    NotificationHistory
+)
 from app.statsd_decorators import statsd
+from app.utils import convert_utc_to_bst
 
 
-def get_service_ids_that_need_sms_billing_populated(start_date, end_date):
+def get_service_ids_that_need_billing_populated(start_date, end_date):
     return db.session.query(
         NotificationHistory.service_id
     ).filter(
         NotificationHistory.created_at >= start_date,
         NotificationHistory.created_at <= end_date,
-        NotificationHistory.notification_type == SMS_TYPE,
+        NotificationHistory.notification_type.in_([SMS_TYPE, EMAIL_TYPE]),
         NotificationHistory.billable_units != 0
     ).distinct().all()
 
 
-@transactional
+def create_or_update_monthly_billing(service_id, billing_month):
-def create_or_update_monthly_billing_sms(service_id, billing_month):
+    start_date, end_date = get_month_start_and_end_date_in_utc(billing_month)
-    start_date, end_date = get_month_start_end_date(billing_month)
+    _update_monthly_billing(service_id, start_date, end_date, SMS_TYPE)
-    monthly = get_billing_data_for_month(service_id=service_id, start_date=start_date, end_date=end_date)
+    _update_monthly_billing(service_id, start_date, end_date, EMAIL_TYPE)
-    # update monthly
-    monthly_totals = _monthly_billing_data_to_json(monthly)
-    row = get_monthly_billing_entry(service_id, start_date, SMS_TYPE)
 
+
+def _monthly_billing_data_to_json(monthly):
+    # total cost must take into account the free allowance.
+    # might be a good idea to capture free allowance in this table
+    return [{
+        "billing_units": x.billing_units,
+        "rate_multiplier": x.rate_multiplier,
+        "international": x.international,
+        "rate": x.rate,
+        "total_cost": (x.billing_units * x.rate_multiplier) * x.rate
+    } for x in monthly]
+
+
+@transactional
+def _update_monthly_billing(service_id, start_date, end_date, notification_type):
+    billing_data = get_billing_data_for_month(
+        service_id=service_id,
+        start_date=start_date,
+        end_date=end_date,
+        notification_type=notification_type
+    )
+    monthly_totals = _monthly_billing_data_to_json(billing_data)
+    row = get_monthly_billing_entry(service_id, start_date, notification_type)
     if row:
         row.monthly_totals = monthly_totals
         row.updated_at = datetime.utcnow()
     else:
         row = MonthlyBilling(
             service_id=service_id,
-            notification_type=SMS_TYPE,
+            notification_type=notification_type,
             monthly_totals=monthly_totals,
             start_date=start_date,
             end_date=end_date

tests/app/dao/test_monthly_billing.py

@@ -1,19 +1,56 @@
 from datetime import datetime, timedelta
+from dateutil.relativedelta import relativedelta
 from freezegun import freeze_time
-from freezegun.api import FakeDatetime
+from functools import partial
 
 from app import db
 from app.dao.monthly_billing_dao import (
-    create_or_update_monthly_billing_sms,
+    create_or_update_monthly_billing,
     get_monthly_billing_entry,
-    get_monthly_billing_sms,
+    get_monthly_billing_by_notification_type,
-    get_service_ids_that_need_sms_billing_populated
+    get_service_ids_that_need_billing_populated,
+    get_billing_data_for_financial_year
 )
-from app.models import MonthlyBilling, SMS_TYPE
+from app.models import MonthlyBilling, SMS_TYPE, EMAIL_TYPE
-from tests.app.db import create_notification, create_rate, create_service, create_template
+from tests.app.db import (
+    create_notification,
+    create_rate,
+    create_service,
+    create_template,
+    create_monthly_billing_entry
+)
+
+FEB_2016_MONTH_START = datetime(2016, 2, 1)
+FEB_2016_MONTH_END = datetime(2016, 2, 29, 23, 59, 59, 99999)
+
+MAR_2016_MONTH_START = datetime(2016, 3, 1)
+MAR_2016_MONTH_END = datetime(2016, 3, 31, 22, 59, 59, 99999)
+
+APR_2016_MONTH_START = datetime(2016, 3, 31, 23, 00, 00)
+APR_2016_MONTH_END = datetime(2016, 4, 30, 22, 59, 59, 99999)
+
+MAY_2016_MONTH_START = datetime(2016, 5, 31, 23, 00, 00)
+MAY_2016_MONTH_END = MAY_2016_MONTH_START + relativedelta(months=1, seconds=-1)
+
+APR_2017_MONTH_START = datetime(2017, 3, 31, 23, 00, 00)
+APR_2017_MONTH_END = datetime(2017, 4, 30, 23, 59, 59, 99999)
+
+JAN_2017_MONTH_START = datetime(2017, 1, 1)
+JAN_2017_MONTH_END = datetime(2017, 1, 31, 23, 59, 59, 99999)
+
+FEB_2017 = datetime(2017, 2, 15)
+APR_2016 = datetime(2016, 4, 10)
+
+EMPTY_BILLING_DATA = {
+    'billing_units': 0,
+    'international': False,
+    'rate': 0,
+    'rate_multiplier': 1,
+    'total_cost': 0,
+}
 
 
-def create_sample_monthly_billing_entry(
+def _create_sample_monthly_billing_entry(
     service_id,
     monthly_totals,
     start_date,
@@ -33,118 +70,276 @@ def create_sample_monthly_billing_entry(
     return entry
 
 
-def test_add_monthly_billing(sample_template):
+def _assert_monthly_billing(monthly_billing, service_id, notification_type, month_start, month_end):
-    jan = datetime(2017, 1, 1)
+    assert monthly_billing.service_id == service_id
-    feb = datetime(2017, 2, 15)
+    assert monthly_billing.notification_type == notification_type
-    create_rate(start_date=jan, value=0.0158, notification_type=SMS_TYPE)
+    assert monthly_billing.start_date == month_start
-    create_rate(start_date=datetime(2017, 3, 31, 23, 00, 00), value=0.123, notification_type=SMS_TYPE)
+    assert monthly_billing.end_date == month_end
-    create_notification(template=sample_template, created_at=jan, billable_units=1, status='delivered')
+
-    create_notification(template=sample_template, created_at=feb, billable_units=2, status='delivered')
+
+def _assert_monthly_billing_totals(monthly_billing_totals, expected_dict):
+    assert sorted(monthly_billing_totals.keys()) == sorted(expected_dict.keys())
+    assert sorted(monthly_billing_totals.values()) == sorted(expected_dict.values())
+
+
+def test_get_monthly_billing_by_notification_type_returns_correct_totals(notify_db, notify_db_session):
+    service = create_service(service_name="Service One")
+
+    _create_sample_monthly_billing_entry(
+        service_id=service.id,
+        monthly_totals=[{
+            "billing_units": 12,
+            "rate": 0.0158,
+            "rate_multiplier": 5,
+            "total_cost": 2.1804,
+            "international": False
+        }],
+        start_date=APR_2016_MONTH_START,
+        end_date=APR_2016_MONTH_END,
+        notification_type=SMS_TYPE
+    )
+
+    monthly_billing_data = get_monthly_billing_by_notification_type(service.id, APR_2016, SMS_TYPE)
+
+    _assert_monthly_billing(
+        monthly_billing_data, service.id, 'sms', APR_2016_MONTH_START, APR_2016_MONTH_END
+    )
+    _assert_monthly_billing_totals(monthly_billing_data.monthly_totals[0], {
+        "billing_units": 12,
+        "rate_multiplier": 5,
+        "international": False,
+        "rate": 0.0158,
+        "total_cost": 2.1804
+    })
+
+
+def test_get_monthly_billing_by_notification_type_filters_by_type(notify_db, notify_db_session):
+    service = create_service(service_name="Service One")
+
+    _create_sample_monthly_billing_entry(
+        service_id=service.id,
+        monthly_totals=[{
+            "billing_units": 138,
+            "rate": 0.0158,
+            "rate_multiplier": 1,
+            "total_cost": 2.1804,
+            "international": None
+        }],
+        start_date=APR_2016_MONTH_START,
+        end_date=APR_2016_MONTH_END,
+        notification_type=SMS_TYPE
+    )
+
+    _create_sample_monthly_billing_entry(
+        service_id=service.id,
+        monthly_totals=[],
+        start_date=APR_2016_MONTH_START,
+        end_date=APR_2016_MONTH_END,
+        notification_type=EMAIL_TYPE
+    )
+
+    monthly_billing_data = get_monthly_billing_by_notification_type(service.id, APR_2016, EMAIL_TYPE)
+
+    _assert_monthly_billing(
+        monthly_billing_data, service.id, 'email', APR_2016_MONTH_START, APR_2016_MONTH_END
+    )
+    assert monthly_billing_data.monthly_totals == []
+
+
+def test_get_monthly_billing_by_notification_type_normalises_start_date(notify_db, notify_db_session):
+    service = create_service(service_name="Service One")
+
+    _create_sample_monthly_billing_entry(
+        service_id=service.id,
+        monthly_totals=[{
+            "billing_units": 321,
+            "rate": 0.0158,
+            "rate_multiplier": 1,
+            "total_cost": 2.1804,
+            "international": None
+        }],
+        start_date=APR_2016_MONTH_START,
+        end_date=APR_2016_MONTH_END,
+        notification_type=SMS_TYPE
+    )
+
+    monthly_billing_data = get_monthly_billing_by_notification_type(service.id, APR_2016 + timedelta(days=5), SMS_TYPE)
+
+    assert monthly_billing_data.start_date == APR_2016_MONTH_START
+    assert monthly_billing_data.monthly_totals[0]['billing_units'] == 321
+
+
+def test_add_monthly_billing_for_single_month_populates_correctly(
+    sample_template, sample_email_template
+):
+    create_rate(start_date=JAN_2017_MONTH_START, value=0.0158, notification_type=SMS_TYPE)
+    create_notification(
+        template=sample_template, created_at=JAN_2017_MONTH_START,
+        billable_units=1, rate_multiplier=2, status='delivered'
+    )
+
+    create_or_update_monthly_billing(
+        service_id=sample_template.service_id,
+        billing_month=JAN_2017_MONTH_START
+    )
+
+    monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all()
 
-    create_or_update_monthly_billing_sms(service_id=sample_template.service_id,
-                                         billing_month=jan)
-    create_or_update_monthly_billing_sms(service_id=sample_template.service_id,
-                                         billing_month=feb)
-    monthly_billing = MonthlyBilling.query.all()
     assert len(monthly_billing) == 2
-    assert monthly_billing[0].start_date == datetime(2017, 1, 1)
+    _assert_monthly_billing(
-    assert monthly_billing[1].start_date == datetime(2017, 2, 1)
+        monthly_billing[0], sample_template.service.id, 'email', JAN_2017_MONTH_START, JAN_2017_MONTH_END
+    )
+    _assert_monthly_billing_totals(monthly_billing[0].monthly_totals[0], {
+        "billing_units": 0,
+        "rate_multiplier": 1,
+        "international": False,
+        "rate": 0,
+        "total_cost": 0
+    })
 
-    january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=jan)
+    _assert_monthly_billing(
-    expected_jan = {"billing_units": 1,
+        monthly_billing[1], sample_template.service.id, 'sms', JAN_2017_MONTH_START, JAN_2017_MONTH_END
-                    "rate_multiplier": 1,
+    )
-                    "international": False,
+    _assert_monthly_billing_totals(monthly_billing[1].monthly_totals[0], {
-                    "rate": 0.0158,
+        "billing_units": 1,
-                    "total_cost": 1 * 0.0158}
+        "rate_multiplier": 2,
-    assert_monthly_billing(january, sample_template.service_id, 1, expected_jan,
+        "international": False,
-                           start_date=datetime(2017, 1, 1), end_date=datetime(2017, 1, 31))
+        "rate": 0.0158,
-
+        "total_cost": 1 * 2 * 0.0158
-    february = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=feb)
+    })
-    expected_feb = {"billing_units": 2,
-                    "rate_multiplier": 1,
-                    "international": False,
-                    "rate": 0.0158,
-                    "total_cost": 2 * 0.0158}
-    assert_monthly_billing(february, sample_template.service_id, 1, expected_feb,
-                           start_date=datetime(2017, 2, 1), end_date=datetime(2017, 2, 28))
 
 
-def test_add_monthly_billing_multiple_rates_in_a_month(sample_template):
+def test_add_monthly_billing_for_multiple_months_populate_correctly(
-    rate_1 = datetime(2016, 12, 1)
+    sample_template, sample_email_template
-    rate_2 = datetime(2017, 1, 15)
+):
-    create_rate(start_date=rate_1, value=0.0158, notification_type=SMS_TYPE)
+    create_rate(start_date=FEB_2016_MONTH_START - timedelta(days=1), value=0.12, notification_type=SMS_TYPE)
-    create_rate(start_date=rate_2, value=0.0124, notification_type=SMS_TYPE)
+    create_notification(
+        template=sample_template, created_at=FEB_2016_MONTH_START,
+        billable_units=1, rate_multiplier=2, status='delivered'
+    )
+    create_notification(
+        template=sample_template, created_at=MAR_2016_MONTH_START,
+        billable_units=2, rate_multiplier=3, status='delivered'
+    )
 
-    create_notification(template=sample_template, created_at=datetime(2017, 1, 1), billable_units=1, status='delivered')
+    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=FEB_2016_MONTH_START)
-    create_notification(template=sample_template, created_at=datetime(2017, 1, 14, 23, 59), billable_units=1,
+    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=MAR_2016_MONTH_START)
-                        status='delivered')
 
-    create_notification(template=sample_template, created_at=datetime(2017, 1, 15), billable_units=2,
-                        status='delivered')
-    create_notification(template=sample_template, created_at=datetime(2017, 1, 17, 13, 30, 57), billable_units=4,
-                        status='delivered')
-
-    create_or_update_monthly_billing_sms(service_id=sample_template.service_id,
-                                         billing_month=rate_2)
     monthly_billing = MonthlyBilling.query.all()
-    assert len(monthly_billing) == 1
-    assert monthly_billing[0].start_date == datetime(2017, 1, 1)
 
-    january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=rate_2)
+    assert len(monthly_billing) == 4
-    first_row = {"billing_units": 2,
+    _assert_monthly_billing(
-                 "rate_multiplier": 1,
+        monthly_billing[0], sample_template.service.id, 'sms', FEB_2016_MONTH_START, FEB_2016_MONTH_END
-                 "international": False,
+    )
-                 "rate": 0.0158,
+    _assert_monthly_billing_totals(monthly_billing[0].monthly_totals[0], {
-                 "total_cost": 3 * 0.0158}
+        "billing_units": 1,
-    assert_monthly_billing(january, sample_template.service_id, 2, first_row,
+        "rate_multiplier": 2,
-                           start_date=datetime(2017, 1, 1), end_date=datetime(2017, 1, 1))
+        "international": False,
-    second_row = {"billing_units": 6,
+        "rate": 0.12,
-                  "rate_multiplier": 1,
+        "total_cost": 0.24
-                  "international": False,
+    })
-                  "rate": 0.0124,
+
-                  "total_cost": 1 * 0.0124}
+    _assert_monthly_billing(
-    assert sorted(january.monthly_totals[1]) == sorted(second_row)
+        monthly_billing[1], sample_template.service.id, 'email', FEB_2016_MONTH_START, FEB_2016_MONTH_END
+    )
+    _assert_monthly_billing_totals(monthly_billing[1].monthly_totals[0], EMPTY_BILLING_DATA)
+
+    _assert_monthly_billing(
+        monthly_billing[2], sample_template.service.id, 'sms', MAR_2016_MONTH_START, MAR_2016_MONTH_END
+    )
+    _assert_monthly_billing_totals(monthly_billing[2].monthly_totals[0], {
+        "billing_units": 2,
+        "rate_multiplier": 3,
+        "international": False,
+        "rate": 0.12,
+        "total_cost": 0.72
+    })
+
+    _assert_monthly_billing(
+        monthly_billing[3], sample_template.service.id, 'email', MAR_2016_MONTH_START, MAR_2016_MONTH_END
+    )
+    _assert_monthly_billing_totals(monthly_billing[3].monthly_totals[0], EMPTY_BILLING_DATA)
+
+
+def test_add_monthly_billing_with_multiple_rates_populate_correctly(
+    sample_template
+):
+    create_rate(start_date=JAN_2017_MONTH_START, value=0.0158, notification_type=SMS_TYPE)
+    create_rate(start_date=JAN_2017_MONTH_START + timedelta(days=5), value=0.123, notification_type=SMS_TYPE)
+    create_notification(template=sample_template, created_at=JAN_2017_MONTH_START, billable_units=1, status='delivered')
+    create_notification(
+        template=sample_template, created_at=JAN_2017_MONTH_START + timedelta(days=6),
+        billable_units=2, status='delivered'
+    )
+
+    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=JAN_2017_MONTH_START)
+
+    monthly_billing = MonthlyBilling.query.all()
+
+    assert len(monthly_billing) == 2
+    _assert_monthly_billing(
+        monthly_billing[0], sample_template.service.id, 'sms', JAN_2017_MONTH_START, JAN_2017_MONTH_END
+    )
+    _assert_monthly_billing_totals(monthly_billing[0].monthly_totals[0], {
+        "billing_units": 1,
+        "rate_multiplier": 1,
+        "international": False,
+        "rate": 0.0158,
+        "total_cost": 0.0158
+    })
+    _assert_monthly_billing_totals(monthly_billing[0].monthly_totals[1], {
+        "billing_units": 2,
+        "rate_multiplier": 1,
+        "international": False,
+        "rate": 0.123,
+        "total_cost": 0.246
+    })
+
+    _assert_monthly_billing(
+        monthly_billing[1], sample_template.service.id, 'email', JAN_2017_MONTH_START, JAN_2017_MONTH_END
+    )
+    _assert_monthly_billing_totals(monthly_billing[1].monthly_totals[0], EMPTY_BILLING_DATA)
 
 
 def test_update_monthly_billing_overwrites_old_totals(sample_template):
-    july = datetime(2017, 7, 1)
+    create_rate(APR_2016_MONTH_START, 0.123, SMS_TYPE)
-    create_rate(july, 0.123, SMS_TYPE)
+    create_notification(template=sample_template, created_at=APR_2016_MONTH_START, billable_units=1, status='delivered')
-    create_notification(template=sample_template, created_at=datetime(2017, 7, 2), billable_units=1, status='delivered')
-    with freeze_time('2017-07-20 02:30:00'):
-        create_or_update_monthly_billing_sms(sample_template.service_id, july)
-    first_update = get_monthly_billing_sms(sample_template.service_id, july)
-    expected = {"billing_units": 1,
-                "rate_multiplier": 1,
-                "international": False,
-                "rate": 0.123,
-                "total_cost": 1 * 0.123}
-    assert_monthly_billing(first_update, sample_template.service_id, 1, expected,
-                           start_date=datetime(2017, 6, 30, 23), end_date=datetime(2017, 7, 31, 23, 59, 59, 99999))
-    first_updated_at = first_update.updated_at
-    with freeze_time('2017-07-20 03:30:00'):
-        create_notification(template=sample_template, created_at=datetime(2017, 7, 5), billable_units=2,
-                            status='delivered')
 
-        create_or_update_monthly_billing_sms(sample_template.service_id, july)
+    create_or_update_monthly_billing(sample_template.service_id, APR_2016_MONTH_END)
-    second_update = get_monthly_billing_sms(sample_template.service_id, july)
+    first_update = get_monthly_billing_by_notification_type(sample_template.service_id, APR_2016_MONTH_START, SMS_TYPE)
-    expected_update = {"billing_units": 3,
+
-                       "rate_multiplier": 1,
+    _assert_monthly_billing(
-                       "international": False,
+        first_update, sample_template.service.id, 'sms', APR_2016_MONTH_START, APR_2016_MONTH_END
-                       "rate": 0.123,
+    )
-                       "total_cost": 3 * 0.123}
+    _assert_monthly_billing_totals(first_update.monthly_totals[0], {
-    assert_monthly_billing(second_update, sample_template.service_id, 1, expected_update,
+        "billing_units": 1,
-                           start_date=datetime(2017, 6, 30, 23), end_date=datetime(2017, 7, 31, 23, 59, 59, 99999))
+        "rate_multiplier": 1,
-    assert second_update.updated_at == FakeDatetime(2017, 7, 20, 3, 30)
+        "international": False,
+        "rate": 0.123,
+        "total_cost": 0.123
+    })
+
+    first_updated_at = first_update.updated_at
+
+    with freeze_time(APR_2016_MONTH_START + timedelta(days=3)):
+        create_notification(template=sample_template, billable_units=2, status='delivered')
+        create_or_update_monthly_billing(sample_template.service_id, APR_2016_MONTH_END)
+
+    second_update = get_monthly_billing_by_notification_type(sample_template.service_id, APR_2016_MONTH_START, SMS_TYPE)
+
+    _assert_monthly_billing_totals(second_update.monthly_totals[0], {
+        "billing_units": 3,
+        "rate_multiplier": 1,
+        "international": False,
+        "rate": 0.123,
+        "total_cost": 0.369
+    })
+
+    assert second_update.updated_at == APR_2016_MONTH_START + timedelta(days=3)
     assert first_updated_at != second_update.updated_at
 
 
-def assert_monthly_billing(monthly_billing, service_id, expected_len, first_row, start_date, end_date):
+def test_get_service_ids_that_need_billing_populated_return_correctly(notify_db_session):
-    assert monthly_billing.service_id == service_id
-    assert len(monthly_billing.monthly_totals) == expected_len
-    assert sorted(monthly_billing.monthly_totals[0]) == sorted(first_row)
-
-
-def test_get_service_id(notify_db_session):
     service_1 = create_service(service_name="Service One")
     template_1 = create_template(service=service_1)
     service_2 = create_service(service_name="Service Two")
@@ -153,8 +348,9 @@ def test_get_service_id(notify_db_session):
     create_notification(template=template_1, created_at=datetime(2017, 7, 1, 14, 30), status='delivered')
     create_notification(template=template_2, created_at=datetime(2017, 7, 15, 13, 30))
     create_notification(template=template_2, created_at=datetime(2017, 7, 31, 13, 30))
-    services = get_service_ids_that_need_sms_billing_populated(start_date=datetime(2017, 7, 1),
+    services = get_service_ids_that_need_billing_populated(
-                                                               end_date=datetime(2017, 7, 16))
+        start_date=datetime(2017, 7, 1), end_date=datetime(2017, 7, 16)
+    )
     expected_services = [service_1.id, service_2.id]
     assert sorted([x.service_id for x in services]) == sorted(expected_services)
 
@@ -164,14 +360,14 @@ def test_get_monthly_billing_entry_filters_by_service(notify_db, notify_db_sessi
     service_2 = create_service(service_name="Service Two")
     now = datetime.utcnow()
 
-    create_sample_monthly_billing_entry(
+    _create_sample_monthly_billing_entry(
         service_id=service_1.id,
         monthly_totals=[],
         start_date=now,
         end_date=now + timedelta(days=30)
     )
 
-    create_sample_monthly_billing_entry(
+    _create_sample_monthly_billing_entry(
         service_id=service_2.id,
         monthly_totals=[],
         start_date=now,