Merge pull request #1144 from alphagov/imdad-feat-schedule-billing-and-adjust-backfill

Populate monthly billing on a schedule and make backfill less granular

app/commands.py

@@ -153,12 +153,19 @@ class PopulateMonthlyBilling(Command):
         option_list = (
             Option('-s', '-service-id', dest='service_id',
                    help="Service id to populate monthly billing for"),
-            Option('-m', '-month', dest="month", help="Use for integer value for month, e.g. 7 for July"),
             Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017")
         )
 
-        def run(self, service_id, month, year):
-            print('Starting populating monthly billing')
+        def run(self, service_id, year):
+            start, end = 1, 13
+            if year == '2016':
+                start = 4
+
+            print('Starting populating monthly billing for {}'.format(year))
+            for i in range(start, end):
+                self.populate(service_id, year, i)
+
+        def populate(self, service_id, year, month):
             create_or_update_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
             results = get_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
             print("Finished populating data for {} for service id {}".format(month, service_id))

app/config.py

@@ -221,6 +221,11 @@ class Config(object):
             'task': 'timeout-job-statistics',
             'schedule': crontab(minute=0, hour=5),
             'options': {'queue': QueueNames.PERIODIC}
+        },
+        'populate_monthly_billing': {
+            'task': 'populate_monthly_billing',
+            'schedule': crontab(minute=10, hour=5),
+            'options': {'queue': QueueNames.PERIODIC}
         }
     }
     CELERY_QUEUES = []

app/dao/notification_usage_dao.py

@@ -22,14 +22,31 @@ def get_yearly_billing_data(service_id, year):
     start_date, end_date = get_financial_year(year)
     rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
 
+    if not rates:
+        return []
+
     def get_valid_from(valid_from):
         return start_date if valid_from < start_date else valid_from
 
     result = []
     for r, n in zip(rates, rates[1:]):
-        result.append(sms_yearly_billing_data_query(r.rate, service_id, get_valid_from(r.valid_from), n.valid_from))
+        result.append(
+            sms_yearly_billing_data_query(
+                r.rate,
+                service_id,
+                get_valid_from(r.valid_from),
+                n.valid_from
+            )
+        )
     result.append(
-        sms_yearly_billing_data_query(rates[-1].rate, service_id, get_valid_from(rates[-1].valid_from), end_date))
+        sms_yearly_billing_data_query(
+            rates[-1].rate,
+            service_id,
+            get_valid_from(rates[-1].valid_from),
+            end_date
+        )
+    )
+
     result.append(email_yearly_billing_data_query(service_id, start_date, end_date))
 
     return sum(result, [])
@@ -38,6 +55,10 @@ def get_yearly_billing_data(service_id, year):
 @statsd(namespace="dao")
 def get_billing_data_for_month(service_id, start_date, end_date):
     rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
+
+    if not rates:
+        return []
+
     result = []
     # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought
     for r, n in zip(rates, rates[1:]):
@@ -54,6 +75,9 @@ def get_monthly_billing_data(service_id, year):
     start_date, end_date = get_financial_year(year)
     rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
 
+    if not rates:
+        return []
+
     result = []
     for r, n in zip(rates, rates[1:]):
         result.extend(sms_billing_data_per_month_query(r.rate, service_id, r.valid_from, n.valid_from))

tests/app/dao/test_notification_usage_dao.py

@@ -1,13 +1,15 @@
+import pytest
 import uuid
 from datetime import datetime, timedelta
+from freezegun import freeze_time
 
-import pytest
 from flask import current_app
 
 from app.dao.date_util import get_financial_year
 from app.dao.notification_usage_dao import (
     get_rates_for_daterange,
     get_yearly_billing_data,
+    get_billing_data_for_month,
     get_monthly_billing_data,
     get_total_billable_units_for_sent_sms_notifications_in_date_range,
     discover_rate_bounds_for_billing_query
@@ -16,11 +18,16 @@ from app.models import (
     Rate,
     NOTIFICATION_DELIVERED,
     NOTIFICATION_STATUS_TYPES_BILLABLE,
-    NOTIFICATION_STATUS_TYPES_NON_BILLABLE)
-from tests.app.conftest import sample_notification, sample_email_template, sample_letter_template, sample_service
-from tests.app.db import create_notification
-from freezegun import freeze_time
-
+    NOTIFICATION_STATUS_TYPES_NON_BILLABLE,
+    SMS_TYPE,
+)
+from tests.app.conftest import (
+    sample_notification,
+    sample_email_template,
+    sample_letter_template,
+    sample_service
+)
+from tests.app.db import create_notification, create_rate
 from tests.conftest import set_config
 
 
@@ -722,3 +729,46 @@ def test_deducts_free_tier_from_bill_across_rate_boundaries(
         )[1] == expected_cost
     finally:
         current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] = start_value
+
+
+def test_get_yearly_billing_data_for_start_date_before_rate_returns_empty(
+    sample_template
+):
+    create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE)
+
+    results = get_yearly_billing_data(
+        service_id=sample_template.service_id,
+        year=2015
+    )
+
+    assert not results
+
+
+@freeze_time("2016-05-01")
+def test_get_billing_data_for_month_where_start_date_before_rate_returns_empty(
+    sample_template
+):
+    create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE)
+
+    results = get_monthly_billing_data(
+        service_id=sample_template.service_id,
+        year=2015
+    )
+
+    assert not results
+
+
+@freeze_time("2016-05-01")
+def test_get_monthly_billing_data_where_start_date_before_rate_returns_empty(
+    sample_template
+):
+    now = datetime.utcnow()
+    create_rate(now, 0.014, SMS_TYPE)
+
+    results = get_billing_data_for_month(
+        service_id=sample_template.service_id,
+        start_date=now - timedelta(days=2),
+        end_date=now - timedelta(days=1)
+    )
+
+    assert not results