Monthly billing - part 1

This is still a work in progress but it would be good to get some eyes on it.
This commit includes creating and updating a row in the monthly billing table and a method to fetch the results.
There is a command to populate the monthly billing for a service and month so we can try it out.
The total cost at the moment are wrong, they do not take into account the free allowance - see notes below about adding that to the table.
Left to do:
create a nightly task to run to update the monthly totals.
create an endpoint to return the yearly billing, the current day will need to be calculated on the fly and added to the totals.
Add the free allowance into the total costs.

app/commands.py

@@ -5,7 +5,8 @@ from flask.ext.script import Command, Manager, Option
 
 
 from app import db
-from app.models import (PROVIDERS, Service, User, NotificationHistory)
+from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms
+from app.models import (PROVIDERS, User)
 from app.dao.services_dao import (
     delete_service_and_all_associated_db_objects,
     dao_fetch_all_services_by_user
@@ -146,3 +147,18 @@ class CustomDbScript(Command):
             print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
             db.session.commit()
             result = db.session.execute(subq_hist).fetchall()
+
+
+class PopulateMonthlyBilling(Command):
+        option_list = (
+            Option('-s', '-service-id', dest='service_id',
+                   help="Service id to populate monthly billing for"),
+            Option('-m', '-month', dest="month", help="Use for integer value for month, e.g. 7 for July"),
+            Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017")
+        )
+
+        def run(self, service_id, month, year):
+            create_or_update_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
+            results = get_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
+            print("Finished populating data for {} for service id {}".format(month, service_id))
+            print(results.monthly_totals)

app/dao/date_util.py

@@ -16,3 +16,16 @@ def get_april_fools(year):
     """
     return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(
         tzinfo=None)
+
+
+def get_month_start_end_date(month_year):
+    """
+     This function return the start and date of the month_year as UTC,
+     :param month_year: the datetime to calculate the start and end date for that month
+     :return: start_date, end_date, month
+    """
+    import calendar
+    _, num_days = calendar.monthrange(month_year.year, month_year.month)
+    first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0)
+    last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999)
+    return first_day, last_day

app/dao/monthly_billing_dao.py

@@ -1,6 +1,44 @@
+from datetime import datetime
+
 from app import db
+from app.dao.notification_usage_dao import get_billing_data_for_month
+from app.models import MonthlyBilling, SMS_TYPE
 
 
-def update_monthly_billing(monthly_billing):
-    db.session.add(monthly_billing)
+def create_or_update_monthly_billing_sms(service_id, billing_month):
+    monthly = get_billing_data_for_month(service_id=service_id, billing_month=billing_month)
+    # update monthly
+    monthly_totals = _monthly_billing_data_to_json(monthly)
+    row = MonthlyBilling.query.filter_by(year=billing_month.year,
+                                         month=datetime.strftime(billing_month, "%B"),
+                                         notification_type='sms').first()
+    if row:
+        row.monthly_totals = monthly_totals
+    else:
+        row = MonthlyBilling(service_id=service_id,
+                             notification_type=SMS_TYPE,
+                             year=billing_month.year,
+                             month=datetime.strftime(billing_month, "%B"),
+                             monthly_totals=monthly_totals)
+    db.session.add(row)
     db.session.commit()
+
+
+def get_monthly_billing_sms(service_id, billing_month):
+    monthly = MonthlyBilling.query.filter_by(service_id=service_id,
+                                             year=billing_month.year,
+                                             month=datetime.strftime(billing_month, "%B"),
+                                             notification_type=SMS_TYPE).first()
+    return monthly
+
+
+def _monthly_billing_data_to_json(monthly):
+    # ('April', 6, 1, False, 'sms', 0.014)
+    #  (month, billing_units, rate_multiplier, international, notification_type, rate)
+    # total cost must take into account the free allowance.
+    # might be a good idea to capture free allowance in this table
+    return [{"billing_units": x[1],
+             "rate_multiplier": x[2],
+             "international": x[3],
+             "rate": x[5],
+             "total_cost": (x[1] * x[2]) * x[5]} for x in monthly]

app/dao/notification_usage_dao.py

@@ -6,7 +6,7 @@ from sqlalchemy import func, case, cast
 from sqlalchemy import literal_column
 
 from app import db
-from app.dao.date_util import get_financial_year
+from app.dao.date_util import get_financial_year, get_month_start_end_date
 from app.models import (NotificationHistory,
                         Rate,
                         NOTIFICATION_STATUS_TYPES_BILLABLE,
@@ -35,6 +35,21 @@ def get_yearly_billing_data(service_id, year):
     return sum(result, [])
 
 
+@statsd(namespace="dao")
+def get_billing_data_for_month(service_id, billing_month):
+    start_date, end_date = get_month_start_end_date(billing_month)
+    rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
+    result = []
+    # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought
+    for r, n in zip(rates, rates[1:]):
+        result.extend(sms_billing_data_per_month_query(r.rate, service_id, max(r.valid_from, start_date),
+                                                       min(n.valid_from, end_date)))
+    result.extend(
+        sms_billing_data_per_month_query(rates[-1].rate, service_id, max(rates[-1].valid_from, start_date), end_date))
+
+    return result
+
+
 @statsd(namespace="dao")
 def get_monthly_billing_data(service_id, year):
     start_date, end_date = get_financial_year(year)

app/dao/provider_rates_dao.py

@@ -9,3 +9,8 @@ def create_provider_rates(provider_identifier, valid_from, rate):
 
     provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate)
     db.session.add(provider_rates)
+
+
+@transactional
+def create_sms_rate(rate):
+    db.session.add(rate)

app/delivery/send_to_providers.py

@@ -18,7 +18,7 @@ from app.dao.templates_dao import dao_get_template_by_id
 from app.models import SMS_TYPE, KEY_TYPE_TEST, BRANDING_ORG, EMAIL_TYPE, NOTIFICATION_TECHNICAL_FAILURE, \
     NOTIFICATION_SENT, NOTIFICATION_SENDING
 
-from app.celery.statistics_tasks import record_initial_job_statistics, create_initial_notification_statistic_tasks
+from app.celery.statistics_tasks import create_initial_notification_statistic_tasks
 
 
 def send_sms_to_provider(notification):

app/models.py

@@ -1261,3 +1261,12 @@ class MonthlyBilling(db.Model):
     __table_args__ = (
         UniqueConstraint('service_id', 'month', 'year', 'notification_type', name='uix_monthly_billing'),
     )
+
+    def serialized(self):
+        return {
+            "month": self.month,
+            "year": self.year,
+            "service_id": str(self.service_id),
+            "notification_type": self.notification_type,
+            "monthly_totals": self.monthly_totals
+        }

app/service/rest.py

@@ -68,7 +68,6 @@ from app.schemas import (
     user_schema,
     permission_schema,
     notification_with_template_schema,
-    notification_with_personalisation_schema,
     notifications_filter_schema,
     detailed_service_schema
 )

application.py

@@ -16,6 +16,7 @@ manager.add_command('db', MigrateCommand)
 manager.add_command('create_provider_rate', commands.CreateProviderRateCommand)
 manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand)
 manager.add_command('custom_db_script', commands.CustomDbScript)
+manager.add_command('populate_monthly_billing', commands.PopulateMonthlyBilling)
 
 
 @manager.command

tests/app/dao/test_date_utils.py

@@ -1,4 +1,8 @@
-from app.dao.date_util import get_financial_year, get_april_fools
+from datetime import datetime
+
+import pytest
+
+from app.dao.date_util import get_financial_year, get_april_fools, get_month_start_end_date
 
 
 def test_get_financial_year():
@@ -11,3 +15,16 @@ def test_get_april_fools():
     april_fools = get_april_fools(2016)
     assert str(april_fools) == '2016-03-31 23:00:00'
     assert april_fools.tzinfo is None
+
+
+@pytest.mark.parametrize("month, year, expected_end",
+                         [(7, 2017, 31),
+                          (2, 2016, 29),
+                          (2, 2017, 28),
+                          (9, 2018, 30),
+                          (12, 2019, 31)])
+def test_get_month_start_end_date(month, year, expected_end):
+    month_year = datetime(year, month, 10, 13, 30, 00)
+    result = get_month_start_end_date(month_year)
+    assert result[0] == datetime(year, month, 1, 0, 0, 0, 0)
+    assert result[1] == datetime(year, month, expected_end, 23, 59, 59, 99999)

tests/app/dao/test_monthly_billing.py

@@ -1,33 +1,107 @@
-import uuid
+from datetime import datetime
 
-import pytest
-from sqlalchemy.exc import IntegrityError
-
-from app.dao.monthly_billing_dao import update_monthly_billing
+from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms
 from app.models import MonthlyBilling
+from tests.app.db import create_notification, create_rate
 
 
-def test_add_monthly_billing_only_allows_one_row_per_service_month_type(sample_service):
-    first = MonthlyBilling(id=uuid.uuid4(),
-                           service_id=sample_service.id,
-                           notification_type='sms',
-                           month='January',
-                           year='2017',
-                           monthly_totals={'billing_units': 100,
-                                           'rate': 0.0158})
+def test_add_monthly_billing(sample_template):
+    jan = datetime(2017, 1, 1)
+    feb = datetime(2017, 2, 15)
+    create_rate(start_date=jan, value=0.0158, notification_type='sms')
+    create_notification(template=sample_template, created_at=jan, billable_units=1, status='delivered')
+    create_notification(template=sample_template, created_at=feb, billable_units=2, status='delivered')
 
-    second = MonthlyBilling(id=uuid.uuid4(),
-                            service_id=sample_service.id,
-                            notification_type='sms',
-                            month='January',
-                            year='2017',
-                            monthly_totals={'billing_units': 50,
-                                            'rate': 0.0162})
+    create_or_update_monthly_billing_sms(service_id=sample_template.service_id,
+                                         billing_month=jan)
+    create_or_update_monthly_billing_sms(service_id=sample_template.service_id,
+                                         billing_month=feb)
+    monthly_billing = MonthlyBilling.query.all()
+    assert len(monthly_billing) == 2
+    assert monthly_billing[0].month == 'January'
+    assert monthly_billing[1].month == 'February'
 
-    update_monthly_billing(first)
-    with pytest.raises(IntegrityError):
-        update_monthly_billing(second)
-        monthly = MonthlyBilling.query.all()
-        assert len(monthly) == 1
-        assert monthly[0].monthly_totals == {'billing_units': 100,
-                                             'rate': 0.0158}
+    january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=jan)
+    expected_jan = {"billing_units": 1,
+                    "rate_multiplier": 1,
+                    "international": False,
+                    "rate": 0.0158,
+                    "total_cost": 1 * 0.0158}
+    assert_monthly_billing(january, 2017, "January", sample_template.service_id, 1, expected_jan)
+
+    february = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=feb)
+    expected_feb = {"billing_units": 2,
+                    "rate_multiplier": 1,
+                    "international": False,
+                    "rate": 0.0158,
+                    "total_cost": 2 * 0.0158}
+    assert_monthly_billing(february, 2017, "February", sample_template.service_id, 1, expected_feb)
+
+
+def test_add_monthly_billing_multiple_rates_in_a_month(sample_template):
+    rate_1 = datetime(2016, 12, 1)
+    rate_2 = datetime(2017, 1, 15)
+    create_rate(start_date=rate_1, value=0.0158, notification_type='sms')
+    create_rate(start_date=rate_2, value=0.0124, notification_type='sms')
+
+    create_notification(template=sample_template, created_at=datetime(2017, 1, 1), billable_units=1, status='delivered')
+    create_notification(template=sample_template, created_at=datetime(2017, 1, 14, 23, 59), billable_units=1,
+                        status='delivered')
+
+    create_notification(template=sample_template, created_at=datetime(2017, 1, 15), billable_units=2,
+                        status='delivered')
+    create_notification(template=sample_template, created_at=datetime(2017, 1, 17, 13, 30, 57), billable_units=4,
+                        status='delivered')
+
+    create_or_update_monthly_billing_sms(service_id=sample_template.service_id,
+                                         billing_month=rate_2)
+    monthly_billing = MonthlyBilling.query.all()
+    assert len(monthly_billing) == 1
+    assert monthly_billing[0].month == 'January'
+
+    january = get_monthly_billing_sms(service_id=sample_template.service_id, billing_month=rate_2)
+    first_row = {"billing_units": 2,
+                 "rate_multiplier": 1,
+                 "international": False,
+                 "rate": 0.0158,
+                 "total_cost": 3 * 0.0158}
+    assert_monthly_billing(january, 2017, "January", sample_template.service_id, 2, first_row)
+    second_row = {"billing_units": 6,
+                  "rate_multiplier": 1,
+                  "international": False,
+                  "rate": 0.0124,
+                  "total_cost": 1 * 0.0124}
+    assert sorted(january.monthly_totals[1]) == sorted(second_row)
+
+
+def test_update_monthly_billing_overwrites_old_totals(sample_template):
+    july = datetime(2017, 7, 1)
+    create_rate(july, 0.123, 'sms')
+    create_notification(template=sample_template, created_at=datetime(2017, 7, 2), billable_units=1, status='delivered')
+
+    create_or_update_monthly_billing_sms(sample_template.service_id, july)
+    first_update = get_monthly_billing_sms(sample_template.service_id, july)
+    expected = {"billing_units": 1,
+                "rate_multiplier": 1,
+                "international": False,
+                "rate": 0.123,
+                "total_cost": 1 * 0.123}
+    assert_monthly_billing(first_update, 2017, "July", sample_template.service_id, 1, expected)
+
+    create_notification(template=sample_template, created_at=datetime(2017, 7, 5), billable_units=2, status='delivered')
+    create_or_update_monthly_billing_sms(sample_template.service_id, july)
+    second_update = get_monthly_billing_sms(sample_template.service_id, july)
+    expected_update = {"billing_units": 3,
+                       "rate_multiplier": 1,
+                       "international": False,
+                       "rate": 0.123,
+                       "total_cost": 3 * 0.123}
+    assert_monthly_billing(second_update, 2017, "July", sample_template.service_id, 1, expected_update)
+
+
+def assert_monthly_billing(monthly_billing, year, month, service_id, expected_len, first_row):
+    assert monthly_billing.year == year
+    assert monthly_billing.month == month
+    assert monthly_billing.service_id == service_id
+    assert len(monthly_billing.monthly_totals) == expected_len
+    assert sorted(monthly_billing.monthly_totals[0]) == sorted(first_row)

tests/app/dao/test_provider_rates_dao.py

@@ -1,7 +1,8 @@
+import uuid
 from datetime import datetime
 from decimal import Decimal
-from app.dao.provider_rates_dao import create_provider_rates
-from app.models import ProviderRates, ProviderDetails
+from app.dao.provider_rates_dao import create_provider_rates, create_sms_rate
+from app.models import ProviderRates, ProviderDetails, Rate
 
 
 def test_create_provider_rates(notify_db, notify_db_session, mmg_provider):
@@ -15,3 +16,11 @@ def test_create_provider_rates(notify_db, notify_db_session, mmg_provider):
     assert ProviderRates.query.first().rate == rate
     assert ProviderRates.query.first().valid_from == now
     assert ProviderRates.query.first().provider_id == provider.id
+
+
+def test_create_sms_rate():
+    rate = Rate(id=uuid.uuid4(), valid_from=datetime.now(), rate=0.014, notification_type='sms')
+    create_sms_rate(rate)
+    rates = Rate.query.all()
+    assert len(rates) == 1
+    assert rates[0] == rate

tests/app/db.py

@@ -1,8 +1,8 @@
 from datetime import datetime
 import uuid
 
-
 from app.dao.jobs_dao import dao_create_job
+from app.dao.provider_rates_dao import create_sms_rate
 from app.dao.service_inbound_api_dao import save_service_inbound_api
 from app.models import (
     Service,
@@ -11,6 +11,7 @@ from app.models import (
     Notification,
     ScheduledNotification,
     ServicePermission,
+    Rate,
     Job,
     InboundSms,
     Organisation,
@@ -239,3 +240,9 @@ def create_organisation(colour='blue', logo='test_x2.png', name='test_org_1'):
     dao_create_organisation(organisation)
 
     return organisation
+
+
+def create_rate(start_date, value, notification_type):
+    rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=notification_type)
+    create_sms_rate(rate)
+    return rate