From ca167206d5dd913e888429b25e1821268ed73e2d Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 5 Mar 2018 16:44:20 +0000 Subject: [PATCH] Add command to backfill Performance Platform totals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don’t have any way of playing back the totals we send to performance platform. This commit copies the command used to backfill the processing time and adapts it to backfill the totals instead. Under the hood it uses the same code that we use in the scheduled tasks to update performance platform on a daily basis. I had to modify this code to take a `day` argument because it was hardcoded to only work for ‘yesterday’. --- app/celery/scheduled_tasks.py | 11 ++++---- app/commands.py | 28 +++++++++++++++++-- .../total_sent_notifications.py | 14 ++++------ tests/app/celery/test_scheduled_tasks.py | 2 +- .../test_total_sent_notifications.py | 14 ++++++++-- tests/app/test_commands.py | 15 +++++++++- 6 files changed, 64 insertions(+), 20 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 7500111a7..b5f904f2b 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -213,20 +213,21 @@ def timeout_notifications(): @statsd(namespace="tasks") def send_daily_performance_platform_stats(): if performance_platform_client.active: - send_total_sent_notifications_to_performance_platform() + yesterday = datetime.utcnow() - timedelta(days=1) + send_total_sent_notifications_to_performance_platform(yesterday) processing_time.send_processing_time_to_performance_platform() -def send_total_sent_notifications_to_performance_platform(): - count_dict = total_sent_notifications.get_total_sent_notifications_yesterday() +def send_total_sent_notifications_to_performance_platform(day): + count_dict = total_sent_notifications.get_total_sent_notifications_for_day(day) email_sent_count = count_dict.get('email').get('count') sms_sent_count = count_dict.get('sms').get('count') letter_sent_count = count_dict.get('letter').get('count') start_date = count_dict.get('start_date') current_app.logger.info( - "Attempting to update performance platform for date {} with email count {} and sms count {}" - .format(start_date, email_sent_count, sms_sent_count) + "Attempting to update Performance Platform for {} with {} emails, {} text messages and {} letters" + .format(start_date, email_sent_count, sms_sent_count, letter_sent_count) ) total_sent_notifications.send_total_notifications_sent_for_day_stats( diff --git a/app/commands.py b/app/commands.py index 06a9b7a75..d8775f217 100644 --- a/app/commands.py +++ b/app/commands.py @@ -22,7 +22,8 @@ from app.dao.services_dao import ( from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates from app.dao.users_dao import (delete_model_user, delete_user_verify_codes) from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc -from app.performance_platform.processing_time import send_processing_time_for_start_and_end +from app.performance_platform.processing_time import (send_processing_time_for_start_and_end) +from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform @click.group(name='command', help='Additional commands') @@ -209,12 +210,35 @@ def populate_monthly_billing(year): populate(service_id, year, i) +@notify_command() +@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d')) +@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d')) +def backfill_performance_platform_totals(start_date, end_date): + """ + Send historical total messages sent to Performance Platform. + """ + + delta = end_date - start_date + + print('Sending total messages sent for all days between {} and {}'.format(start_date, end_date)) + + for i in range(delta.days + 1): + + process_date = start_date + timedelta(days=i) + + print('Sending total messages sent for {}'.format( + process_date.isoformat() + )) + + send_total_sent_notifications_to_performance_platform(process_date) + + @notify_command() @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d')) @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d')) def backfill_processing_time(start_date, end_date): """ - Send historical performance platform stats. + Send historical processing time to Performance Platform. """ delta = end_date - start_date diff --git a/app/performance_platform/total_sent_notifications.py b/app/performance_platform/total_sent_notifications.py index 4aa62c786..14695e9e9 100644 --- a/app/performance_platform/total_sent_notifications.py +++ b/app/performance_platform/total_sent_notifications.py @@ -1,11 +1,8 @@ -from datetime import datetime +from datetime import timedelta from app import performance_platform_client from app.dao.notifications_dao import get_total_sent_notifications_in_date_range -from app.utils import ( - get_london_midnight_in_utc, - get_midnight_for_day_before -) +from app.utils import get_london_midnight_in_utc def send_total_notifications_sent_for_day_stats(date, notification_type, count): @@ -20,10 +17,9 @@ def send_total_notifications_sent_for_day_stats(date, notification_type, count): performance_platform_client.send_stats_to_performance_platform(payload) -def get_total_sent_notifications_yesterday(): - today = datetime.utcnow() - start_date = get_midnight_for_day_before(today) - end_date = get_london_midnight_in_utc(today) +def get_total_sent_notifications_for_day(day): + start_date = get_london_midnight_in_utc(day) + end_date = start_date + timedelta(days=1) email_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'email') sms_count = get_total_sent_notifications_in_date_range(start_date, end_date, 'sms') diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 6c2094cf8..8e3a3a33e 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -335,7 +335,7 @@ def test_send_total_sent_notifications_to_performance_platform_calls_with_correc new_callable=PropertyMock ) as mock_active: mock_active.return_value = True - send_total_sent_notifications_to_performance_platform() + send_total_sent_notifications_to_performance_platform(yesterday) perf_mock.assert_has_calls([ call(get_london_midnight_in_utc(yesterday), 'sms', 2), diff --git a/tests/app/performance_platform/test_total_sent_notifications.py b/tests/app/performance_platform/test_total_sent_notifications.py index 2ad2a93f6..3aecf7447 100644 --- a/tests/app/performance_platform/test_total_sent_notifications.py +++ b/tests/app/performance_platform/test_total_sent_notifications.py @@ -6,7 +6,7 @@ from freezegun import freeze_time from app.utils import get_midnight_for_day_before from app.performance_platform.total_sent_notifications import ( send_total_notifications_sent_for_day_stats, - get_total_sent_notifications_yesterday + get_total_sent_notifications_for_day ) from tests.app.conftest import ( @@ -55,6 +55,7 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict( # Create some notifications for the day before yesterday = datetime(2016, 1, 10, 15, 30, 0, 0) + ereyesterday = datetime(2016, 1, 9, 15, 30, 0, 0) with freeze_time(yesterday): notification_history(notification_type='letter') notification_history(notification_type='sms') @@ -63,7 +64,7 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict( notification_history(notification_type='email') notification_history(notification_type='email') - total_count_dict = get_total_sent_notifications_yesterday() + total_count_dict = get_total_sent_notifications_for_day(yesterday) assert total_count_dict == { "start_date": get_midnight_for_day_before(datetime.utcnow()), @@ -77,3 +78,12 @@ def test_get_total_sent_notifications_yesterday_returns_expected_totals_dict( "count": 1 } } + + another_day = get_total_sent_notifications_for_day(ereyesterday) + + assert another_day == { + 'email': {'count': 0}, + 'letter': {'count': 0}, + 'sms': {'count': 0}, + 'start_date': datetime(2016, 1, 9, 0, 0), + } diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index d5313d128..e7b8ac360 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -1,6 +1,6 @@ from datetime import datetime -from app.commands import backfill_processing_time +from app.commands import backfill_performance_platform_totals, backfill_processing_time def test_backfill_processing_time_works_for_correct_dates(mocker, notify_api): @@ -14,3 +14,16 @@ def test_backfill_processing_time_works_for_correct_dates(mocker, notify_api): send_mock.assert_any_call(datetime(2017, 7, 31, 23, 0), datetime(2017, 8, 1, 23, 0)) send_mock.assert_any_call(datetime(2017, 8, 1, 23, 0), datetime(2017, 8, 2, 23, 0)) send_mock.assert_any_call(datetime(2017, 8, 2, 23, 0), datetime(2017, 8, 3, 23, 0)) + + +def test_backfill_totals_works_for_correct_dates(mocker, notify_api): + send_mock = mocker.patch('app.commands.send_total_sent_notifications_to_performance_platform') + + # backfill_processing_time is a click.Command object - if you try invoking the callback on its own, it + # throws a `RuntimeError: There is no active click context.` - so get at the original function using __wrapped__ + backfill_performance_platform_totals.callback.__wrapped__(datetime(2017, 8, 1), datetime(2017, 8, 3)) + + assert send_mock.call_count == 3 + send_mock.assert_any_call(datetime(2017, 8, 1)) + send_mock.assert_any_call(datetime(2017, 8, 2)) + send_mock.assert_any_call(datetime(2017, 8, 3))