Merge pull request #1144 from alphagov/imdad-feat-schedule-billing-and-adjust-backfill

Populate monthly billing on a schedule and make backfill less granular

app/commands.py

@@ -153,12 +153,19 @@ class PopulateMonthlyBilling(Command):
         option_list = (
             Option('-s', '-service-id', dest='service_id',
                    help="Service id to populate monthly billing for"),
-            Option('-m', '-month', dest="month", help="Use for integer value for month, e.g. 7 for July"),
             Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017")
         )
 
-        def run(self, service_id, month, year):
-            print('Starting populating monthly billing')
+        def run(self, service_id, year):
+            start, end = 1, 13
+            if year == '2016':
+                start = 4
+
+            print('Starting populating monthly billing for {}'.format(year))
+            for i in range(start, end):
+                self.populate(service_id, year, i)
+
+        def populate(self, service_id, year, month):
             create_or_update_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
             results = get_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
             print("Finished populating data for {} for service id {}".format(month, service_id))

app/config.py

@@ -221,6 +221,11 @@ class Config(object):
             'task': 'timeout-job-statistics',
             'schedule': crontab(minute=0, hour=5),
             'options': {'queue': QueueNames.PERIODIC}
+        },
+        'populate_monthly_billing': {
+            'task': 'populate_monthly_billing',
+            'schedule': crontab(minute=10, hour=5),
+            'options': {'queue': QueueNames.PERIODIC}
         }
     }
     CELERY_QUEUES = []

app/dao/notification_usage_dao.py

@@ -22,14 +22,31 @@ def get_yearly_billing_data(service_id, year):
     start_date, end_date = get_financial_year(year)
     rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
 
+    if not rates:
+        return []
+
     def get_valid_from(valid_from):
         return start_date if valid_from < start_date else valid_from
 
     result = []
     for r, n in zip(rates, rates[1:]):
-        result.append(sms_yearly_billing_data_query(r.rate, service_id, get_valid_from(r.valid_from), n.valid_from))
+        result.append(
+            sms_yearly_billing_data_query(
+                r.rate,
+                service_id,
+                get_valid_from(r.valid_from),
+                n.valid_from
+            )
+        )
     result.append(
-        sms_yearly_billing_data_query(rates[-1].rate, service_id, get_valid_from(rates[-1].valid_from), end_date))
+        sms_yearly_billing_data_query(
+            rates[-1].rate,
+            service_id,
+            get_valid_from(rates[-1].valid_from),
+            end_date
+        )
+    )
+
     result.append(email_yearly_billing_data_query(service_id, start_date, end_date))
 
     return sum(result, [])
@@ -38,6 +55,10 @@ def get_yearly_billing_data(service_id, year):
 @statsd(namespace="dao")
 def get_billing_data_for_month(service_id, start_date, end_date):
     rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
+
+    if not rates:
+        return []
+
     result = []
     # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought
     for r, n in zip(rates, rates[1:]):
@@ -54,6 +75,9 @@ def get_monthly_billing_data(service_id, year):
     start_date, end_date = get_financial_year(year)
     rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
 
+    if not rates:
+        return []
+
     result = []
     for r, n in zip(rates, rates[1:]):
         result.extend(sms_billing_data_per_month_query(r.rate, service_id, r.valid_from, n.valid_from))