From b06850e6112a328cc66d010df7fb694f70067b8f Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Thu, 4 Mar 2021 16:10:53 +0000 Subject: [PATCH] Add an endpoint to return all the data required for the performance platform page. --- app/__init__.py | 4 + app/dao/fact_notification_status_dao.py | 31 +++++++ app/dao/fact_processing_time_dao.py | 15 ++++ app/dao/services_dao.py | 20 +++++ .../performance_platform_schema.py | 10 +++ app/performance_platform/rest.py | 85 +++++++++++++++++++ .../dao/test_fact_notification_status_dao.py | 26 +++++- .../app/dao/test_fact_processing_time_dao.py | 20 +++++ tests/app/dao/test_services_dao.py | 27 +++++- tests/app/performance_platform/test_rest.py | 48 +++++++++++ 10 files changed, 284 insertions(+), 2 deletions(-) create mode 100644 app/performance_platform/performance_platform_schema.py create mode 100644 app/performance_platform/rest.py create mode 100644 tests/app/performance_platform/test_rest.py diff --git a/app/__init__.py b/app/__init__.py index 9cc2e64fb..d62a7ec56 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -150,6 +150,7 @@ def register_blueprint(application): from app.organisation.rest import organisation_blueprint from app.organisation.invite_rest import organisation_invite_blueprint from app.complaint.complaint_rest import complaint_blueprint + from app.performance_platform.rest import performance_platform_blueprint from app.platform_stats.rest import platform_stats_blueprint from app.template_folder.rest import template_folder_blueprint from app.letter_branding.letter_branding_rest import letter_branding_blueprint @@ -228,6 +229,9 @@ def register_blueprint(application): complaint_blueprint.before_request(requires_admin_auth) application.register_blueprint(complaint_blueprint) + performance_platform_blueprint.before_request(requires_admin_auth) + application.register_blueprint(performance_platform_blueprint) + platform_stats_blueprint.before_request(requires_admin_auth) application.register_blueprint(platform_stats_blueprint, url_prefix='/platform-stats') diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index ad5fc03c3..36b448608 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -442,6 +442,37 @@ def get_total_sent_notifications_for_day_and_type(day, notification_type): return result or 0 +def get_total_notifications_for_date_range(start_date, end_date): + + result = db.session.query( + FactNotificationStatus.bst_date.cast(db.Text).label("bst_date"), + func.sum(case( + [ + (FactNotificationStatus.notification_type == 'email', FactNotificationStatus.notification_count) + ], + else_=0)).label('emails'), + func.sum(case( + [ + (FactNotificationStatus.notification_type == 'sms', FactNotificationStatus.notification_count) + ], + else_=0)).label('sms'), + func.sum(case( + [ + (FactNotificationStatus.notification_type == 'letter', FactNotificationStatus.notification_count) + ], + else_=0)).label('letters'), + ).filter( + FactNotificationStatus.key_type != KEY_TYPE_TEST, + FactNotificationStatus.bst_date >= start_date, + FactNotificationStatus.bst_date <= end_date + ).group_by( + FactNotificationStatus.bst_date + ).order_by( + FactNotificationStatus.bst_date + ) + return result.all() + + def fetch_monthly_notification_statuses_per_service(start_date, end_date): return db.session.query( func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date).label('date_created'), diff --git a/app/dao/fact_processing_time_dao.py b/app/dao/fact_processing_time_dao.py index 867519fa5..6b879186c 100644 --- a/app/dao/fact_processing_time_dao.py +++ b/app/dao/fact_processing_time_dao.py @@ -30,3 +30,18 @@ def insert_update_processing_time(processing_time): } ) db.session.connection().execute(stmt) + + +def get_processing_time_percentage_for_date_range(start_date, end_date): + query = db.session.query( + FactProcessingTime.bst_date.cast(db.Text).label("date"), + FactProcessingTime.messages_total, + FactProcessingTime.messages_within_10_secs, + ((FactProcessingTime.messages_within_10_secs / FactProcessingTime.messages_total.cast( + db.Float)) * 100).label("percentage") + ).filter( + FactProcessingTime.bst_date >= start_date, + FactProcessingTime.bst_date <= end_date + ).order_by(FactProcessingTime.bst_date) + + return query.all() diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index f678969ef..12dedefa6 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -614,3 +614,23 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10 ) return query.all() + + +def get_live_services_with_organisation(): + query = db.session.query( + Service.id.label("service_id"), + Service.name.label("service_name"), + Organisation.id.label("organisation_id"), + Organisation.name.label("organisation_name") + ).outerjoin( + Service.organisation + ).filter( + Service.count_as_live.is_(True), + Service.active.is_(True), + Service.restricted.is_(False) + ).order_by( + Organisation.name, + Service.name + ) + + return query.all() \ No newline at end of file diff --git a/app/performance_platform/performance_platform_schema.py b/app/performance_platform/performance_platform_schema.py new file mode 100644 index 000000000..78403c449 --- /dev/null +++ b/app/performance_platform/performance_platform_schema.py @@ -0,0 +1,10 @@ +performance_platform_request = { + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Performance platform request schema", + "type": "object", + "title": "Performance platform request", + "properties": { + "start_date": {"type": ["string", "null"], "format": "date"}, + "end_date": {"type": ["string", "null"], "format": "date"}, + } +} diff --git a/app/performance_platform/rest.py b/app/performance_platform/rest.py new file mode 100644 index 000000000..d819fc18d --- /dev/null +++ b/app/performance_platform/rest.py @@ -0,0 +1,85 @@ +from datetime import datetime + +from flask import Blueprint, request, jsonify + +from app.dao.fact_notification_status_dao import get_total_notifications_for_date_range +from app.dao.fact_processing_time_dao import get_processing_time_percentage_for_date_range +from app.dao.services_dao import get_live_services_with_organisation +from app.errors import register_errors +from app.performance_platform.performance_platform_schema import performance_platform_request +from app.schema_validation import validate + +performance_platform_blueprint = Blueprint('performance_platform', __name__, url_prefix='/performance-platform') + +register_errors(performance_platform_blueprint) + + +@performance_platform_blueprint.route('') +def get_performance_platform(): + # All statistics are as of last night this matches the existing performance platform + # and avoids the need to query notifications. + if request.args: + # Is it ok to reuse this? - should probably create a new one + validate(request.args, performance_platform_request) + + # If start and end date are not set, we are expecting today's stats. + today = str(datetime.utcnow().date()) + + start_date = datetime.strptime(request.args.get('start_date', today), '%Y-%m-%d').date() + end_date = datetime.strptime(request.args.get('end_date', today), '%Y-%m-%d').date() + results = get_total_notifications_for_date_range(start_date=start_date, end_date=end_date) + total_notifications, emails, sms, letters = transform_results_into_totals(results) + processing_time_results = get_processing_time_percentage_for_date_range(start_date=start_date, end_date=end_date) + services = get_live_services_with_organisation() + stats = { + "total_notifications": total_notifications, + "email_notifications": emails, + "sms_notifications": sms, + "letter_notifications": letters, + "notifications_by_type": transform_results_to_json(results), + "processing_time": transform_processing_time_results_to_json(processing_time_results), + "live_service_count": len(services), + "services_using_notify": transform_services_to_json(services) + + } + + return jsonify(stats) + + +def transform_results_into_totals(results): + total_notifications = 0 + emails = 0 + sms = 0 + letters = 0 + for x in results: + total_notifications += x.emails + total_notifications += x.sms + total_notifications += x.letters + emails += x.emails + sms += x.sms + letters += x.letters + return total_notifications, emails, sms, letters + + +def transform_results_to_json(results): + j = [] + for x in results: + j.append({"date": x.bst_date, "emails": x.emails, "sms": x.sms, "letters": x.letters}) + return j + + +def transform_processing_time_results_to_json(results): + j = [] + for x in results: + j.append({"date": x.date, "percentage_under_10_seconds": round(x.percentage, 1)}) + + return j + + +def transform_services_to_json(results): + j=[] + for x in results: + j.append({"service_id": x.service_id, "service_name": x.service_name, + "organisation_id": x.organisation_id, "organisation_name": x.organisation_name} + ) + return j \ No newline at end of file diff --git a/tests/app/dao/test_fact_notification_status_dao.py b/tests/app/dao/test_fact_notification_status_dao.py index 4fded8217..e24ec0d92 100644 --- a/tests/app/dao/test_fact_notification_status_dao.py +++ b/tests/app/dao/test_fact_notification_status_dao.py @@ -14,7 +14,7 @@ from app.dao.fact_notification_status_dao import ( fetch_notification_status_totals_for_all_services, fetch_notification_statuses_for_job, fetch_stats_for_all_services_by_date_range, fetch_monthly_template_usage_for_service, - get_total_sent_notifications_for_day_and_type + get_total_sent_notifications_for_day_and_type, get_total_notifications_for_date_range ) from app.models import ( FactNotificationStatus, @@ -703,3 +703,27 @@ def test_fetch_monthly_notification_statuses_per_service_for_rows_that_should_be results = fetch_monthly_notification_statuses_per_service(date(2019, 3, 1), date(2019, 3, 31)) assert len(results) == 0 + + +def test_get_total_notifications_for_date_range(sample_service): + template_sms = create_template(service=sample_service, template_type='sms', template_name='a') + template_email = create_template(service=sample_service, template_type='email', template_name='b') + template_letter = create_template(service=sample_service, template_type='letter', template_name='c') + create_ft_notification_status(bst_date=date(2021, 3, 1), + service=template_email.service, + template=template_email, + count=15) + create_ft_notification_status(bst_date=date(2021, 3, 1), + service=template_sms.service, + template=template_sms, + count=20) + create_ft_notification_status(bst_date=date(2021, 3, 1), + service=template_letter.service, + template=template_letter, + count=3) + + results = get_total_notifications_for_date_range(start_date=datetime(2021, 3, 1), end_date=datetime(2021, 3, 1)) + + assert len(results) == 1 + print(type(results[0].emails)) + assert results[0] == ("2021-03-01", 15, 20, 3) \ No newline at end of file diff --git a/tests/app/dao/test_fact_processing_time_dao.py b/tests/app/dao/test_fact_processing_time_dao.py index cfea71a73..8b79e2279 100644 --- a/tests/app/dao/test_fact_processing_time_dao.py +++ b/tests/app/dao/test_fact_processing_time_dao.py @@ -1,8 +1,10 @@ from datetime import datetime +from decimal import Decimal from freezegun import freeze_time from app.dao import fact_processing_time_dao +from app.dao.fact_processing_time_dao import get_processing_time_percentage_for_date_range from app.models import FactProcessingTime @@ -40,3 +42,21 @@ def test_insert_update_processing_time(notify_db_session): assert result[0].messages_within_10_secs == 3 assert result[0].created_at assert result[0].updated_at == datetime(2021, 2, 23, 13, 23, 33) + + +def test_get_processing_time_percentage_for_date_range(notify_db_session): + data = FactProcessingTime( + bst_date=datetime(2021, 2, 22).date(), + messages_total=3, + messages_within_10_secs=2 + ) + + fact_processing_time_dao.insert_update_processing_time(data) + + results = get_processing_time_percentage_for_date_range('2021-02-22', '2021-02-22') + + assert len(results) == 1 + assert results[0].date == '2021-02-22' + assert results[0].messages_total == 3 + assert results[0].messages_within_10_secs == 2 + assert round(results[0].percentage, 1) == 66.7 diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 03d1a8ab4..c29f20085 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -33,7 +33,7 @@ from app.dao.services_dao import (dao_add_user_to_service, dao_create_service, dao_update_service, delete_service_and_all_associated_db_objects, fetch_todays_total_message_count, - get_services_by_partial_name) + get_services_by_partial_name, get_live_services_with_organisation) from app.dao.users_dao import create_user_code, save_model_user from app.models import (EMAIL_TYPE, INTERNATIONAL_SMS_TYPE, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST, LETTER_TYPE, SMS_TYPE, @@ -1155,3 +1155,28 @@ def test_dao_find_services_with_high_failure_rates(notify_db_session, fake_uuid) assert len(result) == 1 assert str(result[0].service_id) == fake_uuid assert result[0].permanent_failure_rate == 0.25 + + +def test_get_live_services_with_organisation(sample_organisation): + trial_service = create_service(service_name='trial service', restricted=True) + live_service = create_service(service_name="count as live") + live_service_diff_org = create_service(service_name="live service different org") + dont_count_as_live = create_service(service_name="dont count as live", count_as_live=False) + inactive_service = create_service(service_name="inactive", active=False) + service_without_org = create_service(service_name="no org") + another_org = create_organisation(name='different org', ) + + dao_add_service_to_organisation(trial_service, sample_organisation.id) + dao_add_service_to_organisation(live_service, sample_organisation.id) + dao_add_service_to_organisation(dont_count_as_live, sample_organisation.id) + dao_add_service_to_organisation(inactive_service, sample_organisation.id) + dao_add_service_to_organisation(live_service_diff_org, another_org.id) + + services = get_live_services_with_organisation() + assert len(services) == 3 + assert ([(x.service_name, x.organisation_name) for x in services]) == [ + (live_service_diff_org.name, another_org.name), + (live_service.name, sample_organisation.name), + (service_without_org.name, None)] + + diff --git a/tests/app/performance_platform/test_rest.py b/tests/app/performance_platform/test_rest.py new file mode 100644 index 000000000..1f2b1d6e4 --- /dev/null +++ b/tests/app/performance_platform/test_rest.py @@ -0,0 +1,48 @@ +from datetime import date, datetime + +from app.dao import fact_processing_time_dao +from app.models import FactProcessingTime +from tests.app.db import create_template, create_ft_notification_status + + +def test_performance_platform(sample_service, admin_request): + template_sms = create_template(service=sample_service, template_type='sms', template_name='a') + template_email = create_template(service=sample_service, template_type='email', template_name='b') + template_letter = create_template(service=sample_service, template_type='letter', template_name='c') + create_ft_notification_status(bst_date=date(2021, 3, 1), + service=template_email.service, + template=template_email, + count=15) + create_ft_notification_status(bst_date=date(2021, 3, 1), + service=template_sms.service, + template=template_sms, + count=20) + create_ft_notification_status(bst_date=date(2021, 3, 1), + service=template_letter.service, + template=template_letter, + count=3) + + create_process_time() + + results = admin_request.get(endpoint="performance_platform.get_performance_platform", + start_date='2021-03-01', + end_date='2021-03-01') + + assert results['total_notifications'] == 15+20+3 + assert results['email_notifications'] == 15 + assert results['sms_notifications'] == 20 + assert results['letter_notifications'] == 3 + assert results['notifications_by_type'] == [{"date": '2021-03-01', "emails": 15, "sms": 20, "letters": 3}] + assert results['processing_time'] == [({"date": "2021-03-01", "percentage_under_10_seconds": 97.1})] + assert results["live_service_count"] == 1 + assert results["services_using_notify"][0]["service_name"] == sample_service.name + assert not results["services_using_notify"][0]["organisation_name"] + + +def create_process_time(): + data = FactProcessingTime( + bst_date=datetime(2021, 3, 1).date(), + messages_total=35, + messages_within_10_secs=34 + ) + fact_processing_time_dao.insert_update_processing_time(data)