- Added a scheduled task to create or update billing for the month, yesterday is used to calculate the start and end date for the month.

- The new task has not been added to the beat application yet.
- Added an updated_at column to the monthly billing table, we may want to only calculate from the last updated date rather than the entire month.

app/celery/scheduled_tasks.py

@@ -9,9 +9,17 @@ from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
 from app import notify_celery
 from app import performance_platform_client
+from app.dao.date_util import get_month_start_end_date
 from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
-from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by
+from app.dao.jobs_dao import (
+    dao_set_scheduled_jobs_to_pending,
+    dao_get_jobs_older_than_limited_by
+)
+from app.dao.monthly_billing_dao import (
+    get_service_ids_that_need_sms_billing_populated,
+    create_or_update_monthly_billing_sms
+)
 from app.dao.notifications_dao import (
     dao_timeout_notifications,
     is_delivery_slow_for_provider,
@@ -281,3 +289,14 @@ def delete_dvla_response_files_older_than_seven_days():
     except SQLAlchemyError as e:
         current_app.logger.exception("Failed to delete dvla response files")
         raise
+
+
+@notify_celery.task(name="populate_monthly_billing")
+@statsd(namespace="tasks")
+def populate_monthly_billing():
+    # for every service with billable units this month update billing totals for yesterday
+    # this will overwrite the existing amount.
+    yesterday = datetime.utcnow() - timedelta(days=1)
+    start_date, end_date = get_month_start_end_date(yesterday)
+    services = get_service_ids_that_need_sms_billing_populated(start_date, end_date=end_date)
+    [create_or_update_monthly_billing_sms(service_id=s.service_id, billing_month=start_date) for s in services]

app/dao/monthly_billing_dao.py

@@ -1,12 +1,25 @@
 from datetime import datetime
 
 from app import db
+from app.dao.date_util import get_month_start_end_date
 from app.dao.notification_usage_dao import get_billing_data_for_month
-from app.models import MonthlyBilling, SMS_TYPE
+from app.models import MonthlyBilling, SMS_TYPE, NotificationHistory
+
+
+def get_service_ids_that_need_sms_billing_populated(start_date, end_date):
+    return db.session.query(
+        NotificationHistory.service_id
+    ).filter(
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at <= end_date,
+        NotificationHistory.notification_type == SMS_TYPE,
+        NotificationHistory.billable_units != 0
+    ).distinct().all()
 
 
 def create_or_update_monthly_billing_sms(service_id, billing_month):
-    monthly = get_billing_data_for_month(service_id=service_id, billing_month=billing_month)
+    start_date, end_date = get_month_start_end_date(billing_month)
+    monthly = get_billing_data_for_month(service_id=service_id, start_date=start_date, end_date=end_date)
     # update monthly
     monthly_totals = _monthly_billing_data_to_json(monthly)
     row = MonthlyBilling.query.filter_by(year=billing_month.year,

app/dao/notification_usage_dao.py

@@ -36,8 +36,7 @@ def get_yearly_billing_data(service_id, year):
 
 
 @statsd(namespace="dao")
-def get_billing_data_for_month(service_id, billing_month):
-    start_date, end_date = get_month_start_end_date(billing_month)
+def get_billing_data_for_month(service_id, start_date, end_date):
     rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
     result = []
     # so the start end date in the query are the valid from the rate, not the month - this is going to take some thought

app/models.py

@@ -1257,6 +1257,7 @@ class MonthlyBilling(db.Model):
     year = db.Column(db.Float(asdecimal=False), nullable=False)
     notification_type = db.Column(notification_types, nullable=False)
     monthly_totals = db.Column(JSON, nullable=False)
+    updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
 
     __table_args__ = (
         UniqueConstraint('service_id', 'month', 'year', 'notification_type', name='uix_monthly_billing'),