From a08de0939bc55b467620178a3c5762bd86b7a007 Mon Sep 17 00:00:00 2001 From: Imdad Ahad Date: Tue, 1 Aug 2017 11:36:30 +0100 Subject: [PATCH 1/3] Adjust command to backfill (less granular) Rates began from 05-2016 This adjusts the command to backfill by year. If 2016, let's backfill from May. If 2017, let's backfill from the beginning of the year. --- app/commands.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/app/commands.py b/app/commands.py index 14e26b887..99eb95608 100644 --- a/app/commands.py +++ b/app/commands.py @@ -153,12 +153,19 @@ class PopulateMonthlyBilling(Command): option_list = ( Option('-s', '-service-id', dest='service_id', help="Service id to populate monthly billing for"), - Option('-m', '-month', dest="month", help="Use for integer value for month, e.g. 7 for July"), Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017") ) - def run(self, service_id, month, year): - print('Starting populating monthly billing') + def run(self, service_id, year): + start, end = 1, 13 + if year == '2016': + start = 6 + + print('Starting populating monthly billing for {}'.format(year)) + for i in range(start, end): + self.populate(service_id, year, i) + + def populate(self, service_id, year, month): create_or_update_monthly_billing_sms(service_id, datetime(int(year), int(month), 1)) results = get_monthly_billing_sms(service_id, datetime(int(year), int(month), 1)) print("Finished populating data for {} for service id {}".format(month, service_id)) From 5b9377c69740db8328afe3ef176a1b033c03dd7a Mon Sep 17 00:00:00 2001 From: Imdad Ahad Date: Tue, 1 Aug 2017 11:38:25 +0100 Subject: [PATCH 2/3] Start populating monthly billing on a schedule --- app/commands.py | 2 +- app/config.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/commands.py b/app/commands.py index 99eb95608..a2e692ebd 100644 --- a/app/commands.py +++ b/app/commands.py @@ -159,7 +159,7 @@ class PopulateMonthlyBilling(Command): def run(self, service_id, year): start, end = 1, 13 if year == '2016': - start = 6 + start = 4 print('Starting populating monthly billing for {}'.format(year)) for i in range(start, end): diff --git a/app/config.py b/app/config.py index cfd37fa33..12d52df9b 100644 --- a/app/config.py +++ b/app/config.py @@ -221,6 +221,11 @@ class Config(object): 'task': 'timeout-job-statistics', 'schedule': crontab(minute=0, hour=5), 'options': {'queue': QueueNames.PERIODIC} + }, + 'populate_monthly_billing': { + 'task': 'populate_monthly_billing', + 'schedule': crontab(minute=10, hour=5), + 'options': {'queue': QueueNames.PERIODIC} } } CELERY_QUEUES = [] From 824063ddb8d20aa75b4e8c62ece7087d1171d731 Mon Sep 17 00:00:00 2001 From: Imdad Ahad Date: Wed, 2 Aug 2017 15:24:14 +0100 Subject: [PATCH 3/3] Fix to return billing data before a rate begins --- app/dao/notification_usage_dao.py | 28 ++++++++- tests/app/dao/test_notification_usage_dao.py | 62 ++++++++++++++++++-- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 79f786dd0..cf90fe499 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -22,14 +22,31 @@ def get_yearly_billing_data(service_id, year): start_date, end_date = get_financial_year(year) rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) + if not rates: + return [] + def get_valid_from(valid_from): return start_date if valid_from < start_date else valid_from result = [] for r, n in zip(rates, rates[1:]): - result.append(sms_yearly_billing_data_query(r.rate, service_id, get_valid_from(r.valid_from), n.valid_from)) + result.append( + sms_yearly_billing_data_query( + r.rate, + service_id, + get_valid_from(r.valid_from), + n.valid_from + ) + ) result.append( - sms_yearly_billing_data_query(rates[-1].rate, service_id, get_valid_from(rates[-1].valid_from), end_date)) + sms_yearly_billing_data_query( + rates[-1].rate, + service_id, + get_valid_from(rates[-1].valid_from), + end_date + ) + ) + result.append(email_yearly_billing_data_query(service_id, start_date, end_date)) return sum(result, []) @@ -38,6 +55,10 @@ def get_yearly_billing_data(service_id, year): @statsd(namespace="dao") def get_billing_data_for_month(service_id, start_date, end_date): rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) + + if not rates: + return [] + result = [] # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought for r, n in zip(rates, rates[1:]): @@ -54,6 +75,9 @@ def get_monthly_billing_data(service_id, year): start_date, end_date = get_financial_year(year) rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE) + if not rates: + return [] + result = [] for r, n in zip(rates, rates[1:]): result.extend(sms_billing_data_per_month_query(r.rate, service_id, r.valid_from, n.valid_from)) diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py index 9eab4f089..394c71540 100644 --- a/tests/app/dao/test_notification_usage_dao.py +++ b/tests/app/dao/test_notification_usage_dao.py @@ -1,13 +1,15 @@ +import pytest import uuid from datetime import datetime, timedelta +from freezegun import freeze_time -import pytest from flask import current_app from app.dao.date_util import get_financial_year from app.dao.notification_usage_dao import ( get_rates_for_daterange, get_yearly_billing_data, + get_billing_data_for_month, get_monthly_billing_data, get_total_billable_units_for_sent_sms_notifications_in_date_range, discover_rate_bounds_for_billing_query @@ -16,11 +18,16 @@ from app.models import ( Rate, NOTIFICATION_DELIVERED, NOTIFICATION_STATUS_TYPES_BILLABLE, - NOTIFICATION_STATUS_TYPES_NON_BILLABLE) -from tests.app.conftest import sample_notification, sample_email_template, sample_letter_template, sample_service -from tests.app.db import create_notification -from freezegun import freeze_time - + NOTIFICATION_STATUS_TYPES_NON_BILLABLE, + SMS_TYPE, +) +from tests.app.conftest import ( + sample_notification, + sample_email_template, + sample_letter_template, + sample_service +) +from tests.app.db import create_notification, create_rate from tests.conftest import set_config @@ -722,3 +729,46 @@ def test_deducts_free_tier_from_bill_across_rate_boundaries( )[1] == expected_cost finally: current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT'] = start_value + + +def test_get_yearly_billing_data_for_start_date_before_rate_returns_empty( + sample_template +): + create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE) + + results = get_yearly_billing_data( + service_id=sample_template.service_id, + year=2015 + ) + + assert not results + + +@freeze_time("2016-05-01") +def test_get_billing_data_for_month_where_start_date_before_rate_returns_empty( + sample_template +): + create_rate(datetime(2016, 4, 1), 0.014, SMS_TYPE) + + results = get_monthly_billing_data( + service_id=sample_template.service_id, + year=2015 + ) + + assert not results + + +@freeze_time("2016-05-01") +def test_get_monthly_billing_data_where_start_date_before_rate_returns_empty( + sample_template +): + now = datetime.utcnow() + create_rate(now, 0.014, SMS_TYPE) + + results = get_billing_data_for_month( + service_id=sample_template.service_id, + start_date=now - timedelta(days=2), + end_date=now - timedelta(days=1) + ) + + assert not results