Merge pull request #1994 from alphagov/fix-billing-update

make rebuild-ft-billing-for-day remove unused rows

app/commands.py

@@ -19,7 +19,11 @@ from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.celery.letters_pdf_tasks import create_letters_pdf
 from app.config import QueueNames
 from app.dao.date_util import get_financial_year
-from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing
+from app.dao.fact_billing_dao import (
+    fetch_billing_data_for_day,
+    update_fact_billing,
+    delete_billing_data_for_service_for_day
+)
 from app.dao.monthly_billing_dao import (
     create_or_update_monthly_billing,
     get_monthly_billing_by_notification_type,
@@ -526,17 +530,34 @@ def rebuild_ft_billing_for_day(service_id, day):
     Rebuild the data in ft_billing for the given service_id and date
     """
     def rebuild_ft_data(process_day, service):
+        deleted_rows = delete_billing_data_for_service_for_day(process_day, service)
+        current_app.logger.info('deleted {} existing billing rows for {} on {}'.format(
+            deleted_rows,
+            service,
+            process_day
+        ))
         transit_data = fetch_billing_data_for_day(process_day=process_day, service_id=service)
+        # transit_data = every row that should exist
         for data in transit_data:
+            # upsert existing rows
             update_fact_billing(data, process_day)
+        current_app.logger.info('added/updated {} billing rows for {} on {}'.format(
+            len(transit_data),
+            service,
+            process_day
+        ))
+
     if service_id:
         # confirm the service exists
         dao_fetch_service_by_id(service_id)
         rebuild_ft_data(day, service_id)
     else:
-        services = get_service_ids_that_need_billing_populated(day, day)
-        for service_id in services:
-            rebuild_ft_data(day, service_id)
+        services = get_service_ids_that_need_billing_populated(
+            get_london_midnight_in_utc(day),
+            get_london_midnight_in_utc(day + timedelta(days=1))
+        )
+        for row in services:
+            rebuild_ft_data(day, row.service_id)
 
 
 @notify_command(name='compare-ft-billing-to-monthly-billing')

app/dao/fact_billing_dao.py

@@ -125,6 +125,18 @@ def fetch_monthly_billing_for_year(service_id, year):
     return yearly_data
 
 
+def delete_billing_data_for_service_for_day(process_day, service_id):
+    """
+    Delete all ft_billing data for a given service on a given bst_date
+
+    Returns how many rows were deleted
+    """
+    return FactBilling.query.filter(
+        FactBilling.bst_date == process_day,
+        FactBilling.service_id == service_id
+    ).delete()
+
+
 def fetch_billing_data_for_day(process_day, service_id=None):
     start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
     end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))

tests/app/dao/test_ft_billing_dao.py

@@ -6,9 +6,12 @@ from freezegun import freeze_time
 
 from app import db
 from app.dao.fact_billing_dao import (
-    fetch_monthly_billing_for_year, fetch_billing_data_for_day, get_rates_for_billing,
-    get_rate,
+    delete_billing_data_for_service_for_day,
+    fetch_billing_data_for_day,
     fetch_billing_totals_for_year,
+    fetch_monthly_billing_for_year,
+    get_rate,
+    get_rates_for_billing,
 )
 from app.models import FactBilling, Notification
 from app.utils import convert_utc_to_bst
@@ -353,3 +356,25 @@ def test_fetch_billing_totals_for_year(notify_db_session):
     assert results[3].notifications_sent == 365
     assert results[3].billable_units == 365
     assert results[3].rate == Decimal('0.162')
+
+
+def test_delete_billing_data(notify_db_session):
+    service_1 = create_service(service_name='1')
+    service_2 = create_service(service_name='2')
+    sms_template = create_template(service_1, 'sms')
+    email_template = create_template(service_1, 'email')
+    other_service_template = create_template(service_2, 'sms')
+
+    existing_rows_to_delete = [  # noqa
+        create_ft_billing('2018-01-01', 'sms', sms_template, service_1, billable_unit=1),
+        create_ft_billing('2018-01-01', 'email', email_template, service_1, billable_unit=2)
+    ]
+    other_day = create_ft_billing('2018-01-02', 'sms', sms_template, service_1, billable_unit=3)
+    other_service = create_ft_billing('2018-01-01', 'sms', other_service_template, service_2, billable_unit=4)
+
+    delete_billing_data_for_service_for_day('2018-01-01', service_1.id)
+
+    current_rows = FactBilling.query.all()
+    assert sorted(x.billable_units for x in current_rows) == sorted(
+        [other_day.billable_units, other_service.billable_units]
+    )