Make sure we update the correct month for billing:

When populating the monthly billing records on a schedule, we need
to ensure the correct month is being updated.

As an example, if the current datetime is 31 Mar 2016, 23:00. The
BST equivalent is the 1st April. Therefore we need to ensure we
update billing for April, not March. This takes care of that.

app/celery/scheduled_tasks.py

@@ -9,7 +9,7 @@ from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
 from app import notify_celery
 from app import performance_platform_client
-from app.dao.date_util import get_month_start_end_date
+from app.dao.date_util import get_month_start_and_end_date_in_utc
 from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
 from app.dao.jobs_dao import (
@@ -17,8 +17,8 @@ from app.dao.jobs_dao import (
     dao_get_jobs_older_than_limited_by
 )
 from app.dao.monthly_billing_dao import (
-    get_service_ids_that_need_sms_billing_populated,
-    create_or_update_monthly_billing_sms
+    get_service_ids_that_need_billing_populated,
+    create_or_update_monthly_billing
 )
 from app.dao.notifications_dao import (
     dao_timeout_notifications,
@@ -37,6 +37,7 @@ from app.notifications.process_notifications import send_notification_to_queue
 from app.statsd_decorators import statsd
 from app.celery.tasks import process_job
 from app.config import QueueNames
+from app.utils import convert_utc_to_bst
 
 
 @notify_celery.task(name="remove_csv_files")
@@ -297,6 +298,7 @@ def populate_monthly_billing():
     # for every service with billable units this month update billing totals for yesterday
     # this will overwrite the existing amount.
     yesterday = datetime.utcnow() - timedelta(days=1)
-    start_date, end_date = get_month_start_end_date(yesterday)
-    services = get_service_ids_that_need_sms_billing_populated(start_date=start_date, end_date=end_date)
-    [create_or_update_monthly_billing_sms(service_id=s.service_id, billing_month=yesterday) for s in services]
+    yesterday_in_bst = convert_utc_to_bst(yesterday)
+    start_date, end_date = get_month_start_and_end_date_in_utc(yesterday_in_bst)
+    services = get_service_ids_that_need_billing_populated(start_date=start_date, end_date=end_date)
+    [create_or_update_monthly_billing(service_id=s.service_id, billing_month=end_date) for s in services]

tests/app/celery/test_scheduled_tasks.py

@@ -38,7 +38,7 @@ from app.models import (
     Service, Template,
     SMS_TYPE, LETTER_TYPE,
     MonthlyBilling)
-from app.utils import get_london_midnight_in_utc
+from app.utils import get_london_midnight_in_utc, convert_utc_to_bst
 from tests.app.db import create_notification, create_service, create_template, create_job, create_rate
 from tests.app.conftest import (
     sample_job as create_sample_job,
@@ -612,27 +612,71 @@ def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files(
 
 
 @freeze_time("2017-07-12 02:00:00")
-def test_populate_monthly_billing(sample_template):
+def test_populate_monthly_billing_populates_correctly(sample_template):
     yesterday = datetime(2017, 7, 11, 13, 30)
+    jul_month_start = datetime(2017, 6, 30, 23)
+    jul_month_end = datetime(2017, 7, 31, 22, 59, 59, 99999)
     create_rate(datetime(2016, 1, 1), 0.0123, 'sms')
+
     create_notification(template=sample_template, status='delivered', created_at=yesterday)
     create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1))
     create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1))
     # not included in billing
     create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30))
 
-    assert len(MonthlyBilling.query.all()) == 0
     populate_monthly_billing()
 
-    monthly_billing = MonthlyBilling.query.all()
-    assert len(monthly_billing) == 1
+    monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all()
+
+    assert len(monthly_billing) == 2
+
     assert monthly_billing[0].service_id == sample_template.service_id
-    assert monthly_billing[0].start_date == datetime(2017, 6, 30, 23)
-    assert monthly_billing[0].end_date == datetime(2017, 7, 31, 22, 59, 59, 99999)
-    assert monthly_billing[0].notification_type == 'sms'
-    assert len(monthly_billing[0].monthly_totals) == 1
-    assert sorted(monthly_billing[0].monthly_totals[0]) == sorted({'international': False,
-                                                                   'rate_multiplier': 1,
-                                                                   'billing_units': 3,
-                                                                   'rate': 0.0123,
-                                                                   'total_cost': 0.0369})
+    assert monthly_billing[0].start_date == jul_month_start
+    assert monthly_billing[0].end_date == jul_month_end
+    assert monthly_billing[0].notification_type == 'email'
+    assert monthly_billing[0].notification_type == 'email'
+    assert len(monthly_billing[1].monthly_totals) == 1
+    assert monthly_billing[0].monthly_totals[0]['billing_units'] == 0
+    assert monthly_billing[0].monthly_totals[0]['total_cost'] == 0
+
+    assert monthly_billing[1].service_id == sample_template.service_id
+    assert monthly_billing[1].start_date == jul_month_start
+    assert monthly_billing[1].end_date == jul_month_end
+    assert monthly_billing[1].notification_type == 'sms'
+    assert len(monthly_billing[1].monthly_totals) == 1
+    assert sorted(monthly_billing[1].monthly_totals[0]) == sorted(
+        {
+            'international': False,
+            'rate_multiplier': 1,
+            'billing_units': 3,
+            'rate': 0.0123,
+            'total_cost': 0.0369
+        }
+    )
+
+
+@freeze_time("2016-04-01 23:00:00")
+def test_populate_monthly_billing_updates_correct_month_in_bst(sample_template):
+    yesterday = datetime.utcnow() - timedelta(days=1)
+    apr_month_start = datetime(2016, 3, 31, 23)
+    apr_month_end = datetime(2016, 4, 30, 22, 59, 59, 99999)
+    create_rate(datetime(2016, 1, 1), 0.0123, 'sms')
+    create_notification(template=sample_template, status='delivered', created_at=yesterday)
+    populate_monthly_billing()
+
+    monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all()
+
+    assert len(monthly_billing) == 2
+
+    assert monthly_billing[0].service_id == sample_template.service_id
+    assert monthly_billing[0].start_date == apr_month_start
+    assert monthly_billing[0].end_date == apr_month_end
+    assert monthly_billing[0].notification_type == 'email'
+    assert monthly_billing[0].monthly_totals[0]['billing_units'] == 0
+
+    assert monthly_billing[1].service_id == sample_template.service_id
+    assert monthly_billing[1].start_date == apr_month_start
+    assert monthly_billing[1].end_date == apr_month_end
+    assert monthly_billing[1].notification_type == 'sms'
+    assert monthly_billing[1].monthly_totals[0]['billing_units'] == 1
+    assert monthly_billing[1].monthly_totals[0]['total_cost'] == 0.0123

tests/app/db.py

@@ -6,9 +6,13 @@ from app.dao.jobs_dao import dao_create_job
 from app.dao.service_inbound_api_dao import save_service_inbound_api
 from app.models import (
     ApiKey,
+    EMAIL_TYPE,
+    SMS_TYPE,
+    KEY_TYPE_NORMAL,
     Service,
     User,
     Template,
+    MonthlyBilling,
     Notification,
     ScheduledNotification,
     ServicePermission,
@@ -17,10 +21,8 @@ from app.models import (
     InboundSms,
     InboundNumber,
     Organisation,
-    EMAIL_TYPE,
-    SMS_TYPE,
-    KEY_TYPE_NORMAL,
-    ServiceInboundApi)
+    ServiceInboundApi
+)
 from app.dao.users_dao import save_model_user
 from app.dao.notifications_dao import dao_create_notification, dao_created_scheduled_notification
 from app.dao.templates_dao import dao_create_template
@@ -258,7 +260,12 @@ def create_organisation(colour='blue', logo='test_x2.png', name='test_org_1'):
 
 
 def create_rate(start_date, value, notification_type):
-    rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type=notification_type)
+    rate = Rate(
+        id=uuid.uuid4(),
+        valid_from=start_date,
+        rate=value,
+        notification_type=notification_type
+    )
     db.session.add(rate)
     db.session.commit()
     return rate
@@ -290,3 +297,24 @@ def create_inbound_number(number, provider='mmg', active=True, service_id=None):
     db.session.add(inbound_number)
     db.session.commit()
     return inbound_number
+
+
+def create_monthly_billing_entry(
+    service,
+    start_date,
+    end_date,
+    notification_type,
+    monthly_totals=[]
+):
+    entry = MonthlyBilling(
+        service_id=service.id,
+        notification_type=notification_type,
+        monthly_totals=monthly_totals,
+        start_date=start_date,
+        end_date=end_date
+    )
+
+    db.session.add(entry)
+    db.session.commit()
+
+    return entry