Merge pull request #1739 from alphagov/backfill-performance-totals

Add command to backfill Performance Platform totals

app/celery/scheduled_tasks.py

@@ -213,20 +213,21 @@ def timeout_notifications():
 @statsd(namespace="tasks")
 def send_daily_performance_platform_stats():
     if performance_platform_client.active:
-        send_total_sent_notifications_to_performance_platform()
+        yesterday = datetime.utcnow() - timedelta(days=1)
+        send_total_sent_notifications_to_performance_platform(yesterday)
         processing_time.send_processing_time_to_performance_platform()
 
 
-def send_total_sent_notifications_to_performance_platform():
-    count_dict = total_sent_notifications.get_total_sent_notifications_yesterday()
+def send_total_sent_notifications_to_performance_platform(day):
+    count_dict = total_sent_notifications.get_total_sent_notifications_for_day(day)
     email_sent_count = count_dict.get('email').get('count')
     sms_sent_count = count_dict.get('sms').get('count')
     letter_sent_count = count_dict.get('letter').get('count')
     start_date = count_dict.get('start_date')
 
     current_app.logger.info(
-        "Attempting to update performance platform for date {} with email count {} and sms count {}"
-        .format(start_date, email_sent_count, sms_sent_count)
+        "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters"
+        .format(start_date, email_sent_count, sms_sent_count, letter_sent_count)
     )
 
     total_sent_notifications.send_total_notifications_sent_for_day_stats(

app/commands.py

@@ -22,7 +22,8 @@ from app.dao.services_dao import (
 from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
 from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
 from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
-from app.performance_platform.processing_time import send_processing_time_for_start_and_end
+from app.performance_platform.processing_time import (send_processing_time_for_start_and_end)
+from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
 
 
 @click.group(name='command', help='Additional commands')
@@ -209,12 +210,35 @@ def populate_monthly_billing(year):
             populate(service_id, year, i)
 
 
+@notify_command()
+@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
+@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
+def backfill_performance_platform_totals(start_date, end_date):
+    """
+    Send historical total messages sent to Performance Platform.
+    """
+
+    delta = end_date - start_date
+
+    print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date))
+
+    for i in range(delta.days + 1):
+
+        process_date = start_date + timedelta(days=i)
+
+        print('Sending total messages sent for {}'.format(
+            process_date.isoformat()
+        ))
+
+        send_total_sent_notifications_to_performance_platform(process_date)
+
+
 @notify_command()
 @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
 @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
 def backfill_processing_time(start_date, end_date):
     """
-    Send historical performance platform stats.
+    Send historical processing time to Performance Platform.
     """
 
     delta = end_date - start_date

app/performance_platform/total_sent_notifications.py

@@ -1,11 +1,8 @@
-from datetime import datetime
+from datetime import timedelta
 
 from app import performance_platform_client
 from app.dao.notifications_dao import get_total_sent_notifications_in_date_range
-from app.utils import (
-    get_london_midnight_in_utc,
-    get_midnight_for_day_before
-)
+from app.utils import get_london_midnight_in_utc
 
 
 def send_total_notifications_sent_for_day_stats(date, notification_type, count):
@@ -20,10 +17,9 @@ def send_total_notifications_sent_for_day_stats(date, notification_type, count):
     performance_platform_client.send_stats_to_performance_platform(payload)
 
 
-def get_total_sent_notifications_yesterday():
-    today = datetime.utcnow()
-    start_date = get_midnight_for_day_before(today)
-    end_date = get_london_midnight_in_utc(today)
+def get_total_sent_notifications_for_day(day):
+    start_date = get_london_midnight_in_utc(day)
+    end_date = start_date + timedelta(days=1)
 
     email_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'email')
     sms_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'sms')

tests/app/celery/test_scheduled_tasks.py

@@ -335,7 +335,7 @@ def test_send_total_sent_notifications_to_performance_platform_calls_with_correc
         new_callable=PropertyMock
     ) as mock_active:
         mock_active.return_value = True
-        send_total_sent_notifications_to_performance_platform()
+        send_total_sent_notifications_to_performance_platform(yesterday)
 
         perf_mock.assert_has_calls([
             call(get_london_midnight_in_utc(yesterday), 'sms', 2),

tests/app/performance_platform/test_total_sent_notifications.py

@@ -6,7 +6,7 @@ from freezegun import freeze_time
 from app.utils import get_midnight_for_day_before
 from app.performance_platform.total_sent_notifications import (
     send_total_notifications_sent_for_day_stats,
-    get_total_sent_notifications_yesterday
+    get_total_sent_notifications_for_day
 )
 
 from tests.app.conftest import (
@@ -55,6 +55,7 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
 
     # Create some notifications for the day before
     yesterday = datetime(2016, 1, 10, 15, 30, 0, 0)
+    ereyesterday = datetime(2016, 1, 9, 15, 30, 0, 0)
     with freeze_time(yesterday):
         notification_history(notification_type='letter')
         notification_history(notification_type='sms')
@@ -63,7 +64,7 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
         notification_history(notification_type='email')
         notification_history(notification_type='email')
 
-    total_count_dict = get_total_sent_notifications_yesterday()
+    total_count_dict = get_total_sent_notifications_for_day(yesterday)
 
     assert total_count_dict == {
         "start_date": get_midnight_for_day_before(datetime.utcnow()),
@@ -77,3 +78,12 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict(
             "count": 1
         }
     }
+
+    another_day = get_total_sent_notifications_for_day(ereyesterday)
+
+    assert another_day == {
+        'email': {'count': 0},
+        'letter': {'count': 0},
+        'sms': {'count': 0},
+        'start_date': datetime(2016, 1, 9, 0, 0),
+    }

tests/app/test_commands.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from app.commands import backfill_processing_time
+from app.commands import backfill_performance_platform_totals, backfill_processing_time
 
 
 def test_backfill_processing_time_works_for_correct_dates(mocker, notify_api):
@@ -14,3 +14,16 @@ def test_backfill_processing_time_works_for_correct_dates(mocker, notify_api):
     send_mock.assert_any_call(datetime(2017, 7, 31, 23, 0), datetime(2017, 8, 1, 23, 0))
     send_mock.assert_any_call(datetime(2017, 8, 1, 23, 0), datetime(2017, 8, 2, 23, 0))
     send_mock.assert_any_call(datetime(2017, 8, 2, 23, 0), datetime(2017, 8, 3, 23, 0))
+
+
+def test_backfill_totals_works_for_correct_dates(mocker, notify_api):
+    send_mock = mocker.patch('app.commands.send_total_sent_notifications_to_performance_platform')
+
+    # backfill_processing_time is a click.Command object - if you try invoking the callback on its own, it
+    # throws a `RuntimeError: There is no active click context.` - so get at the original function using __wrapped__
+    backfill_performance_platform_totals.callback.__wrapped__(datetime(2017, 8, 1), datetime(2017, 8, 3))
+
+    assert send_mock.call_count == 3
+    send_mock.assert_any_call(datetime(2017, 8, 1))
+    send_mock.assert_any_call(datetime(2017, 8, 2))
+    send_mock.assert_any_call(datetime(2017, 8, 3))