Add command to backfill Performance Platform totals

We don’t have any way of playing back the totals we send to performance
platform.

This commit copies the command used to backfill the processing time and
adapts it to backfill the totals instead. Under the hood it uses the
same code that we use in the scheduled tasks to update performance
platform on a daily basis. I had to modify this code to take a `day`
argument because it was hardcoded to only work for ‘yesterday’.

app/celery/scheduled_tasks.py

@@ -213,20 +213,21 @@ def timeout_notifications():
 @statsd(namespace="tasks")
 def send_daily_performance_platform_stats():
     if performance_platform_client.active:
-        send_total_sent_notifications_to_performance_platform()
+        yesterday = datetime.utcnow() - timedelta(days=1)
+        send_total_sent_notifications_to_performance_platform(yesterday)
         processing_time.send_processing_time_to_performance_platform()
 
 
-def send_total_sent_notifications_to_performance_platform():
-    count_dict = total_sent_notifications.get_total_sent_notifications_yesterday()
+def send_total_sent_notifications_to_performance_platform(day):
+    count_dict = total_sent_notifications.get_total_sent_notifications_for_day(day)
     email_sent_count = count_dict.get('email').get('count')
     sms_sent_count = count_dict.get('sms').get('count')
     letter_sent_count = count_dict.get('letter').get('count')
     start_date = count_dict.get('start_date')
 
     current_app.logger.info(
-        "Attempting to update performance platform for date {} with email count {} and sms count {}"
-        .format(start_date, email_sent_count, sms_sent_count)
+        "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
+        .format(start_date, email_sent_count, sms_sent_count, letter_sent_count)
     )
 
     total_sent_notifications.send_total_notifications_sent_for_day_stats(

app/commands.py

@@ -22,7 +22,8 @@ from app.dao.services_dao import (
 from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
 from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
 from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
-from app.performance_platform.processing_time import send_processing_time_for_start_and_end
+from app.performance_platform.processing_time import (send_processing_time_for_start_and_end)
+from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
 
 
 @click.group(name='command', help='Additional commands')
@@ -209,12 +210,35 @@ def populate_monthly_billing(year):
             populate(service_id, year, i)
 
 
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_performance_platform_totals(start_date, end_date):
+    """
+    Send historical total messages sent to Performance Platform.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+
+        process_date = start_date + timedelta(days=i)
+
+        print('Sending total messages sent for {}'.format(
+            process_date.isoformat()
+        ))
+
+        send_total_sent_notifications_to_performance_platform(process_date)
+
+
 @notify_command()
 @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
 @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
 def backfill_processing_time(start_date, end_date):
     """
-    Send historical performance platform stats.
+    Send historical processing time to Performance Platform.
     """
 
     delta = end_date - start_date

app/performance_platform/total_sent_notifications.py

@@ -1,11 +1,8 @@
-from datetime import datetime
+from datetime import timedelta
 
 from app import performance_platform_client
 from app.dao.notifications_dao import get_total_sent_notifications_in_date_range
-from app.utils import (
-    get_london_midnight_in_utc,
-    get_midnight_for_day_before
-)
+from app.utils import get_london_midnight_in_utc
 
 
 def send_total_notifications_sent_for_day_stats(date, notification_type, count):
@@ -20,10 +17,9 @@ def send_total_notifications_sent_for_day_stats(date, notification_type, count):
     performance_platform_client.send_stats_to_performance_platform(payload)
 
 
-def get_total_sent_notifications_yesterday():
-    today = datetime.utcnow()
-    start_date = get_midnight_for_day_before(today)
-    end_date = get_london_midnight_in_utc(today)
+def get_total_sent_notifications_for_day(day):
+    start_date = get_london_midnight_in_utc(day)
+    end_date = start_date + timedelta(days=1)
 
     email_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'email')
     sms_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'sms')

tests/app/celery/test_scheduled_tasks.py

@@ -335,7 +335,7 @@ def test_send_total_sent_notifications_to_performance_platform_calls_with_correc
         new_callable=PropertyMock
     ) as mock_active:
         mock_active.return_value = True
-        send_total_sent_notifications_to_performance_platform()
+        send_total_sent_notifications_to_performance_platform(yesterday)
 
         perf_mock.assert_has_calls([
             call(get_london_midnight_in_utc(yesterday), 'sms', 2),

tests/app/performance_platform/test_total_sent_notifications.py

@@ -6,7 +6,7 @@ from freezegun import freeze_time
 from app.utils import get_midnight_for_day_before
 from app.performance_platform.total_sent_notifications import (
     send_total_notifications_sent_for_day_stats,
-    get_total_sent_notifications_yesterday
+    get_total_sent_notifications_for_day
 )
 
 from tests.app.conftest import (
@@ -55,6 +55,7 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
 
     # Create some notifications for the day before
     yesterday = datetime(2016, 1, 10, 15, 30, 0, 0)
+    ereyesterday = datetime(2016, 1, 9, 15, 30, 0, 0)
     with freeze_time(yesterday):
         notification_history(notification_type='letter')
         notification_history(notification_type='sms')
@@ -63,7 +64,7 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
         notification_history(notification_type='email')
         notification_history(notification_type='email')
 
-    total_count_dict = get_total_sent_notifications_yesterday()
+    total_count_dict = get_total_sent_notifications_for_day(yesterday)
 
     assert total_count_dict == {
         "start_date": get_midnight_for_day_before(datetime.utcnow()),
@@ -77,3 +78,12 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
             "count": 1
         }
     }
+
+    another_day = get_total_sent_notifications_for_day(ereyesterday)
+
+    assert another_day == {
+        'email': {'count': 0},
+        'letter': {'count': 0},
+        'sms': {'count': 0},
+        'start_date': datetime(2016, 1, 9, 0, 0),
+    }

tests/app/test_commands.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from app.commands import backfill_processing_time
+from app.commands import backfill_performance_platform_totals, backfill_processing_time
 
 
 def test_backfill_processing_time_works_for_correct_dates(mocker, notify_api):
@@ -14,3 +14,16 @@ def test_backfill_processing_time_works_for_correct_dates(mocker, notify_api):
     send_mock.assert_any_call(datetime(2017, 7, 31, 23, 0), datetime(2017, 8, 1, 23, 0))
     send_mock.assert_any_call(datetime(2017, 8, 1, 23, 0), datetime(2017, 8, 2, 23, 0))
     send_mock.assert_any_call(datetime(2017, 8, 2, 23, 0), datetime(2017, 8, 3, 23, 0))
+
+
+def test_backfill_totals_works_for_correct_dates(mocker, notify_api):
+    send_mock = mocker.patch('app.commands.send_total_sent_notifications_to_performance_platform')
+
+    # backfill_processing_time is a click.Command object - if you try invoking the callback on its own, it
+    # throws a `RuntimeError: There is no active click context.` - so get at the original function using __wrapped__
+    backfill_performance_platform_totals.callback.__wrapped__(datetime(2017, 8, 1), datetime(2017, 8, 3))
+
+    assert send_mock.call_count == 3
+    send_mock.assert_any_call(datetime(2017, 8, 1))
+    send_mock.assert_any_call(datetime(2017, 8, 2))
+    send_mock.assert_any_call(datetime(2017, 8, 3))