diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 68f0ea781..d6172fd98 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -453,6 +453,34 @@ def fetch_aggregate_stats_by_date_range_for_all_services(start_date, end_date, i return query.all() +@statsd(namespace='dao') +def fetch_new_aggregate_stats_by_date_range_for_all_services(start_date, end_date): + start_date = get_london_midnight_in_utc(start_date) + end_date = get_london_midnight_in_utc(end_date + timedelta(days=1)) + table = NotificationHistory + + if start_date >= datetime.utcnow() - timedelta(days=7): + table = Notification + + query = db.session.query( + table.notification_type, + table.status, + table.key_type, + func.count(table.id).label('count') + ).filter( + table.created_at >= start_date, + table.created_at < end_date + ).group_by( + table.notification_type, + table.key_type, + table.status + ).order_by( + table.notification_type, + ) + + return query.all() + + @transactional @version_class(Service) @version_class(ApiKey) diff --git a/app/service/rest.py b/app/service/rest.py index 4889fd3b1..13b77c317 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -44,6 +44,7 @@ from app.dao.services_dao import ( dao_suspend_service, dao_update_service, fetch_aggregate_stats_by_date_range_for_all_services, + fetch_new_aggregate_stats_by_date_range_for_all_services, fetch_stats_by_date_range_for_all_services ) from app.dao.service_whitelist_dao import ( @@ -131,6 +132,19 @@ def get_platform_stats(): return result +@service_blueprint.route('/platform-stats-new', methods=['GET']) +def get_new_platform_stats(): + # If start and end date are not set, we are expecting today's stats. + today = str(datetime.utcnow().date()) + + start_date = datetime.strptime(request.args.get('start_date', today), '%Y-%m-%d').date() + end_date = datetime.strptime(request.args.get('end_date', today), '%Y-%m-%d').date() + data = fetch_new_aggregate_stats_by_date_range_for_all_services(start_date=start_date, end_date=end_date) + stats = statistics.format_admin_stats(data) + + return jsonify(stats) + + @service_blueprint.route('', methods=['GET']) def get_services(): only_active = request.args.get('only_active') == 'True' diff --git a/app/service/statistics.py b/app/service/statistics.py index 26b3fb289..df49b7ff2 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -14,6 +14,37 @@ def format_statistics(statistics): return counts +def format_admin_stats(statistics): + counts = create_stats_dict() + + for row in statistics: + if row.key_type == 'test': + counts[row.notification_type]['test-key'] += row.count + else: + counts[row.notification_type]['total'] += row.count + if row.status in ('technical-failure', 'permanent-failure', 'temporary-failure', 'virus-scan-failed'): + counts[row.notification_type]['failures'][row.status] += row.count + + return counts + + +def create_stats_dict(): + stats_dict = {} + for template in TEMPLATE_TYPES: + stats_dict[template] = {} + + for status in ('total', 'test-key'): + stats_dict[template][status] = 0 + + stats_dict[template]['failures'] = { + 'technical-failure': 0, + 'permanent-failure': 0, + 'temporary-failure': 0, + 'virus-scan-failed': 0, + } + return stats_dict + + def format_monthly_template_notification_stats(year, rows): stats = { datetime.strftime(date, '%Y-%m'): {} diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 997920fd5..8338ce6ed 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta import uuid import functools @@ -33,7 +33,8 @@ from app.dao.services_dao import ( dao_fetch_active_users_for_service, dao_fetch_service_by_inbound_number, dao_fetch_monthly_historical_stats_by_template, - dao_fetch_monthly_historical_usage_by_template_for_service) + dao_fetch_monthly_historical_usage_by_template_for_service, + fetch_new_aggregate_stats_by_date_range_for_all_services) from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission from app.dao.users_dao import save_model_user from app.models import ( @@ -674,7 +675,7 @@ def test_fetch_monthly_historical_stats_separates_months(notify_db, notify_db_se result = dao_fetch_monthly_historical_stats_for_service(sample_template.service_id, 2016) - for date, status, count in ( + for day, status, count in ( ('2016-04', 'sending', 0), ('2016-04', 'delivered', 0), ('2016-04', 'pending', 0), @@ -692,9 +693,9 @@ def test_fetch_monthly_historical_stats_separates_months(notify_db, notify_db_se ('2017-03', 'created', 2), ): - assert result[date]['sms'][status] == count - assert result[date]['email'][status] == 0 - assert result[date]['letter'][status] == 0 + assert result[day]['sms'][status] == count + assert result[day]['email'][status] == 0 + assert result[day]['letter'][status] == 0 assert result.keys() == { '2016-04', '2016-05', '2016-06', @@ -817,6 +818,66 @@ def test_fetch_stats_by_date_range_for_all_services(notify_db, notify_db_session result_one.service.created_at, 'sms', 'created', 2) +@pytest.mark.parametrize('table_name, days_ago', [ + ('Notification', 8), + ('NotificationHistory', 3), +]) +@freeze_time('2018-01-08') +def test_fetch_new_aggregate_stats_by_date_range_for_all_services_uses_the_correct_table( + mocker, + notify_db_session, + table_name, + days_ago +): + start_date = datetime.now().date() - timedelta(days=days_ago) + end_date = datetime.now().date() + + # mock the table that should not be used, then check it is not being called + unused_table_mock = mocker.patch('app.dao.services_dao.{}'.format(table_name)) + fetch_new_aggregate_stats_by_date_range_for_all_services(start_date, end_date) + + unused_table_mock.assert_not_called() + + +def test_fetch_new_aggregate_stats_by_date_range_for_all_services_returns_empty_list_when_no_stats(notify_db_session): + start_date = date(2018, 1, 1) + end_date = date(2018, 1, 5) + + result = fetch_new_aggregate_stats_by_date_range_for_all_services(start_date, end_date) + assert result == [] + + +@freeze_time('2018-01-08') +def test_fetch_new_aggregate_stats_by_date_range_for_all_services_groups_stats( + notify_db, + notify_db_session, + sample_template, + sample_email_template, + sample_letter_template, +): + today = datetime.now().date() + + for i in range(3): + create_notification(notify_db, notify_db_session, template=sample_email_template, status='permanent-failure', + created_at=today) + + create_notification(notify_db, notify_db_session, template=sample_email_template, status='sent', created_at=today) + create_notification(notify_db, notify_db_session, template=sample_template, status='sent', created_at=today) + create_notification(notify_db, notify_db_session, template=sample_template, status='sent', created_at=today, + key_type=KEY_TYPE_TEAM) + create_notification(notify_db, notify_db_session, template=sample_letter_template, status='virus-scan-failed', + created_at=today) + + result = fetch_new_aggregate_stats_by_date_range_for_all_services(today, today) + + assert len(result) == 5 + assert result[0] == ('email', 'permanent-failure', 'normal', 3) + assert result[1] == ('email', 'sent', 'normal', 1) + assert result[2] == ('sms', 'sent', 'normal', 1) + assert result[3] == ('sms', 'sent', 'team', 1) + assert result[4] == ('letter', 'virus-scan-failed', 'normal', 1) + + @freeze_time('2001-01-01T23:59:00') def test_dao_suspend_service_marks_service_as_inactive_and_expires_api_keys(sample_service, sample_api_key): dao_suspend_service(sample_service.id) diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index b9665dfc9..44b928c38 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -3085,6 +3085,28 @@ def test_get_platform_stats_creates_zero_stats(client, notify_db_session): assert json_resp['sms'] == {'failed': 0, 'requested': 4, 'delivered': 3} +@freeze_time('2018-06-01') +def test_get_new_platform_stats_uses_todays_date_if_no_start_or_end_date_is_provided(admin_request, mocker): + today = datetime.now().date() + dao_mock = mocker.patch('app.service.rest.fetch_new_aggregate_stats_by_date_range_for_all_services') + mocker.patch('app.service.rest.statistics.format_statistics') + + admin_request.get('service.get_new_platform_stats') + + dao_mock.assert_called_once_with(start_date=today, end_date=today) + + +def test_get_new_platform_stats_can_filter_by_date(admin_request, mocker): + start_date = date(2017, 1, 1) + end_date = date(2018, 1, 1) + dao_mock = mocker.patch('app.service.rest.fetch_new_aggregate_stats_by_date_range_for_all_services') + mocker.patch('app.service.rest.statistics.format_statistics') + + admin_request.get('service.get_new_platform_stats', start_date=start_date, end_date=end_date) + + dao_mock.assert_called_once_with(start_date=start_date, end_date=end_date) + + @pytest.mark.parametrize('today_only, stats', [ (False, {'requested': 2, 'delivered': 1, 'failed': 0}), (True, {'requested': 1, 'delivered': 0, 'failed': 0}) diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index 7f56a5b9a..856bee499 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -3,11 +3,14 @@ import collections import pytest from app.service.statistics import ( + format_admin_stats, format_statistics, + create_stats_dict, create_zeroed_stats_dicts, ) StatsRow = collections.namedtuple('row', ('notification_type', 'status', 'count')) +NewStatsRow = collections.namedtuple('row', ('notification_type', 'status', 'key_type', 'count')) # email_counts and sms_counts are 3-tuple of requested, delivered, failed @@ -65,5 +68,64 @@ def test_create_zeroed_stats_dicts(): } -def _stats(requested, delivered, failed): - return {'requested': requested, 'delivered': delivered, 'failed': failed} +def test_create_stats_dict(): + assert create_stats_dict() == { + 'sms': {'total': 0, + 'test-key': 0, + 'failures': {'technical-failure': 0, + 'permanent-failure': 0, + 'temporary-failure': 0, + 'virus-scan-failed': 0}}, + 'email': {'total': 0, + 'test-key': 0, + 'failures': {'technical-failure': 0, + 'permanent-failure': 0, + 'temporary-failure': 0, + 'virus-scan-failed': 0}}, + 'letter': {'total': 0, + 'test-key': 0, + 'failures': {'technical-failure': 0, + 'permanent-failure': 0, + 'temporary-failure': 0, + 'virus-scan-failed': 0}} + } + + +def test_format_admin_stats_only_includes_test_key_notifications_in_test_key_section(): + rows = [ + NewStatsRow('email', 'technical-failure', 'test', 3), + NewStatsRow('sms', 'permanent-failure', 'test', 4), + NewStatsRow('letter', 'virus-scan-failed', 'test', 5), + ] + stats_dict = format_admin_stats(rows) + + assert stats_dict['email']['total'] == 0 + assert stats_dict['email']['failures']['technical-failure'] == 0 + assert stats_dict['email']['test-key'] == 3 + + assert stats_dict['sms']['total'] == 0 + assert stats_dict['sms']['failures']['permanent-failure'] == 0 + assert stats_dict['sms']['test-key'] == 4 + + assert stats_dict['letter']['total'] == 0 + assert stats_dict['letter']['failures']['virus-scan-failed'] == 0 + assert stats_dict['letter']['test-key'] == 5 + + +def test_format_admin_stats_counts_non_test_key_notifications_correctly(): + rows = [ + NewStatsRow('email', 'technical-failure', 'normal', 1), + NewStatsRow('email', 'created', 'team', 3), + NewStatsRow('sms', 'temporary-failure', 'normal', 6), + NewStatsRow('sms', 'sent', 'normal', 2), + NewStatsRow('letter', 'pending-virus-check', 'normal', 1), + ] + stats_dict = format_admin_stats(rows) + + assert stats_dict['email']['total'] == 4 + assert stats_dict['email']['failures']['technical-failure'] == 1 + + assert stats_dict['sms']['total'] == 8 + assert stats_dict['sms']['failures']['permanent-failure'] == 0 + + assert stats_dict['letter']['total'] == 1