- Added a scheduled task to create or update billing for the month, yesterday is used to calculate the start and end date for the month.

- The new task has not been added to the beat application yet.
- Added an updated_at column to the monthly billing table, we may want to only calculate from the last updated date rather than the entire month.

app/celery/scheduled_tasks.py

@@ -9,9 +9,17 @@ from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
 from app import notify_celery
 from app import performance_platform_client
+from app.dao.date_util import get_month_start_end_date
 from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
-from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by
+from app.dao.jobs_dao import (
+    dao_set_scheduled_jobs_to_pending,
+    dao_get_jobs_older_than_limited_by
+)
+from app.dao.monthly_billing_dao import (
+    get_service_ids_that_need_sms_billing_populated,
+    create_or_update_monthly_billing_sms
+)
 from app.dao.notifications_dao import (
     dao_timeout_notifications,
     is_delivery_slow_for_provider,
@@ -281,3 +289,14 @@ def delete_dvla_response_files_older_than_seven_days():
     except SQLAlchemyError as e:
         current_app.logger.exception("Failed to delete dvla response files")
         raise
+
+
+@notify_celery.task(name="populate_monthly_billing")
+@statsd(namespace="tasks")
+def populate_monthly_billing():
+    # for every service with billable units this month update billing totals for yesterday
+    # this will overwrite the existing amount.
+    yesterday = datetime.utcnow() - timedelta(days=1)
+    start_date, end_date = get_month_start_end_date(yesterday)
+    services = get_service_ids_that_need_sms_billing_populated(start_date, end_date=end_date)
+    [create_or_update_monthly_billing_sms(service_id=s.service_id, billing_month=start_date) for s in services]

app/dao/monthly_billing_dao.py

@@ -1,12 +1,25 @@
 from datetime import datetime
 
 from app import db
+from app.dao.date_util import get_month_start_end_date
 from app.dao.notification_usage_dao import get_billing_data_for_month
-from app.models import MonthlyBilling, SMS_TYPE
+from app.models import MonthlyBilling, SMS_TYPE, NotificationHistory
+
+
+def get_service_ids_that_need_sms_billing_populated(start_date, end_date):
+    return db.session.query(
+        NotificationHistory.service_id
+    ).filter(
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at <= end_date,
+        NotificationHistory.notification_type == SMS_TYPE,
+        NotificationHistory.billable_units != 0
+    ).distinct().all()
 
 
 def create_or_update_monthly_billing_sms(service_id, billing_month):
-    monthly = get_billing_data_for_month(service_id=service_id, billing_month=billing_month)
+    start_date, end_date = get_month_start_end_date(billing_month)
+    monthly = get_billing_data_for_month(service_id=service_id, start_date=start_date, end_date=end_date)
     # update monthly
     monthly_totals = _monthly_billing_data_to_json(monthly)
     row = MonthlyBilling.query.filter_by(year=billing_month.year,

app/dao/notification_usage_dao.py

@@ -36,8 +36,7 @@ def get_yearly_billing_data(service_id, year):
 
 
 @statsd(namespace="dao")
-def get_billing_data_for_month(service_id, billing_month):
-    start_date, end_date = get_month_start_end_date(billing_month)
+def get_billing_data_for_month(service_id, start_date, end_date):
     rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
     result = []
     # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought

app/models.py

@@ -1257,6 +1257,7 @@ class MonthlyBilling(db.Model):
     year = db.Column(db.Float(asdecimal=False), nullable=False)
     notification_type = db.Column(notification_types, nullable=False)
     monthly_totals = db.Column(JSON, nullable=False)
+    updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
 
     __table_args__ = (
         UniqueConstraint('service_id', 'month', 'year', 'notification_type', name='uix_monthly_billing'),

migrations/versions/0110_monthly_billing.py

@@ -26,6 +26,7 @@ def upgrade():
                               postgresql.ENUM('email', 'sms', 'letter', name='notification_type', create_type=False),
                               nullable=False),
                     sa.Column('monthly_totals', postgresql.JSON(), nullable=False),
+                    sa.Column('updated_at', sa.DateTime, nullable=False),
                     sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
                     sa.PrimaryKeyConstraint('id')
                     )

tests/app/celery/test_scheduled_tasks.py

@@ -25,8 +25,8 @@ from app.celery.scheduled_tasks import (
     send_scheduled_notifications,
     switch_current_sms_provider_on_slow_delivery,
     timeout_job_statistics,
-    timeout_notifications
-)
+    timeout_notifications,
+    populate_monthly_billing)
 from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
 from app.dao.jobs_dao import dao_get_job_by_id
 from app.dao.notifications_dao import dao_get_scheduled_notifications
@@ -36,10 +36,10 @@ from app.dao.provider_details_dao import (
 )
 from app.models import (
     Service, Template,
-    SMS_TYPE, LETTER_TYPE
-)
+    SMS_TYPE, LETTER_TYPE,
+    MonthlyBilling)
 from app.utils import get_london_midnight_in_utc
-from tests.app.db import create_notification, create_service, create_template, create_job
+from tests.app.db import create_notification, create_service, create_template, create_job, create_rate
 from tests.app.conftest import (
     sample_job as create_sample_job,
     sample_notification_history as create_notification_history,
@@ -98,6 +98,8 @@ def test_should_have_decorated_tasks_functions():
         'remove_transformed_dvla_files'
     assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \
         'delete_dvla_response_files_older_than_seven_days'
+    assert populate_monthly_billing.__wrapped__.__name__ == \
+        'populate_monthly_billing'
 
 
 @pytest.fixture(scope='function')
@@ -607,3 +609,30 @@ def test_delete_dvla_response_files_older_than_seven_days_does_not_remove_files(
     delete_dvla_response_files_older_than_seven_days()
 
     remove_s3_mock.assert_not_called()
+
+
+@freeze_time("2017-07-12 02:00:00")
+def test_populate_monthly_billing(sample_template):
+    yesterday = datetime(2017, 7, 11, 13, 30)
+    create_rate(datetime(2016, 1, 1), 0.0123, 'sms')
+    create_notification(template=sample_template, status='delivered', created_at=yesterday)
+    create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1))
+    create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1))
+    # not included in billing
+    create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30))
+
+    assert len(MonthlyBilling.query.all()) == 0
+    populate_monthly_billing()
+
+    monthly_billing = MonthlyBilling.query.all()
+    assert len(monthly_billing) == 1
+    assert monthly_billing[0].service_id == sample_template.service_id
+    assert monthly_billing[0].year == 2017
+    assert monthly_billing[0].month == 'July'
+    assert monthly_billing[0].notification_type == 'sms'
+    assert len(monthly_billing[0].monthly_totals) == 1
+    assert sorted(monthly_billing[0].monthly_totals[0]) == sorted({'international': False,
+                                                                   'rate_multiplier': 1,
+                                                                   'billing_units': 3,
+                                                                   'rate': 0.0123,
+                                                                   'total_cost': 0.0369})

tests/app/dao/test_monthly_billing.py

@@ -1,8 +1,12 @@
 from datetime import datetime
 
-from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms
+from app.dao.monthly_billing_dao import (
+    create_or_update_monthly_billing_sms,
+    get_monthly_billing_sms,
+    get_service_ids_that_need_sms_billing_populated
+)
 from app.models import MonthlyBilling
-from tests.app.db import create_notification, create_rate
+from tests.app.db import create_notification, create_rate, create_service, create_template
 
 
 def test_add_monthly_billing(sample_template):
@@ -105,3 +109,18 @@ def assert_monthly_billing(monthly_billing, year, month, service_id, expected_le
     assert monthly_billing.service_id == service_id
     assert len(monthly_billing.monthly_totals) == expected_len
     assert sorted(monthly_billing.monthly_totals[0]) == sorted(first_row)
+
+
+def test_get_service_id():
+    service_1 = create_service(service_name="Service One")
+    template_1 = create_template(service=service_1)
+    service_2 = create_service(service_name="Service Two")
+    template_2 = create_template(service=service_2)
+    create_notification(template=template_1, created_at=datetime(2017, 6, 30, 13, 30), status='delivered')
+    create_notification(template=template_1, created_at=datetime(2017, 7, 1, 14, 30), status='delivered')
+    create_notification(template=template_2, created_at=datetime(2017, 7, 15, 13, 30))
+    create_notification(template=template_2, created_at=datetime(2017, 7, 31, 13, 30))
+    services = get_service_ids_that_need_sms_billing_populated(start_date=datetime(2017, 7, 1),
+                                                               end_date=datetime(2017, 7, 16))
+    expected_services = [service_1.id, service_2.id]
+    assert sorted([x.service_id for x in services]) == sorted(expected_services)