From 5cd68e808143e341735d71e96a628ef69a10bdcc Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Wed, 22 May 2024 13:33:32 -0600 Subject: [PATCH 01/16] New endpoints for #1006 and #1007 --- app/dao/services_dao.py | 27 +++++++++++++++++++++++++++ app/service/rest.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index df8e59287..bc2a5ddf3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -425,6 +425,33 @@ def dao_fetch_todays_stats_for_service(service_id): ) +def dao_fetch_stats_for_service_from_day(service_id, day): + # today = datetime.now(timezone.utc).date() + # 2024-05-20 + # day_date = datetime.strptime(day, '%Y-%m-%d').date() + start_date = get_midnight_in_utc(day) + end_date = get_midnight_in_utc(day + timedelta(days=1)) + print(start_date) + return ( + db.session.query( + NotificationHistory.notification_type, + NotificationHistory.status, + func.count(NotificationHistory.id).label("count"), + ) + .filter( + NotificationHistory.service_id == service_id, + NotificationHistory.key_type != KeyType.TEST, + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at <= end_date, + ) + .group_by( + NotificationHistory.notification_type, + NotificationHistory.status, + ) + .all() + ) + + def dao_fetch_todays_stats_for_all_services( include_from_test_key=True, only_active=True ): diff --git a/app/service/rest.py b/app/service/rest.py index ce5083073..9c0295304 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,5 +1,5 @@ import itertools -from datetime import datetime +from datetime import datetime, timedelta from flask import Blueprint, current_app, jsonify, request from sqlalchemy.exc import IntegrityError @@ -63,6 +63,7 @@ from app.dao.services_dao import ( dao_fetch_all_services_by_user, dao_fetch_live_services_data, dao_fetch_service_by_id, + dao_fetch_stats_for_service_from_day, dao_fetch_todays_stats_for_all_services, dao_fetch_todays_stats_for_service, dao_remove_user_from_service, @@ -210,6 +211,36 @@ def get_service_notification_statistics(service_id): ) +@service_blueprint.route("//statistics//") +def get_service_notification_statistics_by_day(service_id, start, days): + return jsonify( + data=get_service_statistics_for_specific_days(service_id, start, int(days)) + ) + + +def get_service_statistics_for_specific_days(service_id, start, days=1): + start_date = datetime.strptime(start, "%Y-%m-%d").date() + + if days == 1: + stats = {} + stats[start] = { + "value": statistics.format_statistics( + dao_fetch_stats_for_service_from_day(service_id, start_date) + ) + } + else: + stats = {} + for d in range(days): + new_date = start_date + timedelta(days=d) + key = new_date.strftime("%Y-%m-%d") + value = statistics.format_statistics( + dao_fetch_stats_for_service_from_day(service_id, new_date) + ) + stats[key] = {"value": value} + + return stats + + @service_blueprint.route("", methods=["POST"]) def create_service(): data = request.get_json() From 5fbad5bd69cdd453900a97efc1a30c44d261fcf5 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Sat, 25 May 2024 20:59:08 -0600 Subject: [PATCH 02/16] Added endpoints for #1006 and #1007 --- app/dao/date_util.py | 5 +++ app/dao/services_dao.py | 54 +++++++++++++++++++++++- app/service/rest.py | 86 +++++++++++++++++++++++++++++++++++---- app/service/statistics.py | 3 +- 4 files changed, 137 insertions(+), 11 deletions(-) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 7aafd711f..66aadc9df 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -1,3 +1,4 @@ +import calendar from datetime import date, datetime, time, timedelta @@ -64,3 +65,7 @@ def get_calendar_year_for_datetime(start_date): return year - 1 else: return year + + +def get_number_of_days_for_month(year, month): + return calendar.monthrange(year, month)[1] diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index bc2a5ddf3..f6724f247 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -431,7 +431,6 @@ def dao_fetch_stats_for_service_from_day(service_id, day): # day_date = datetime.strptime(day, '%Y-%m-%d').date() start_date = get_midnight_in_utc(day) end_date = get_midnight_in_utc(day + timedelta(days=1)) - print(start_date) return ( db.session.query( NotificationHistory.notification_type, @@ -452,6 +451,33 @@ def dao_fetch_stats_for_service_from_day(service_id, day): ) +def dao_fetch_stats_for_service_from_day_for_user(service_id, day, user_id): + # today = datetime.now(timezone.utc).date() + # 2024-05-20 + # day_date = datetime.strptime(day, '%Y-%m-%d').date() + start_date = get_midnight_in_utc(day) + end_date = get_midnight_in_utc(day + timedelta(days=1)) + return ( + db.session.query( + NotificationHistory.notification_type, + NotificationHistory.status, + func.count(NotificationHistory.id).label("count"), + ) + .filter( + NotificationHistory.service_id == service_id, + NotificationHistory.key_type != KeyType.TEST, + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at <= end_date, + NotificationHistory.created_by_id == user_id, + ) + .group_by( + NotificationHistory.notification_type, + NotificationHistory.status, + ) + .all() + ) + + def dao_fetch_todays_stats_for_all_services( include_from_test_key=True, only_active=True ): @@ -633,3 +659,29 @@ def get_live_services_with_organization(): ) return query.all() + + +def fetch_notification_stats_for_service_by_month_by_user( + start_date, end_date, service_id, user_id +): + return ( + db.session.query( + func.date_trunc("month", NotificationHistory.created_at).label("month"), + NotificationHistory.notification_type, + (NotificationHistory.status).label("notification_status"), + func.count(NotificationHistory.id).label("count"), + ) + .filter( + NotificationHistory.service_id == service_id, + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date, + NotificationHistory.key_type != KeyType.TEST, + NotificationHistory.created_by_id == user_id, + ) + .group_by( + func.date_trunc("month", NotificationHistory.created_at).label("month"), + NotificationHistory.notification_type, + NotificationHistory.status, + ) + .all() + ) diff --git a/app/service/rest.py b/app/service/rest.py index 9c0295304..7d8d53c8e 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -17,7 +17,11 @@ from app.dao.api_key_dao import ( save_model_api_key, ) from app.dao.dao_utils import dao_rollback, transaction -from app.dao.date_util import get_calendar_year +from app.dao.date_util import ( + get_calendar_year, + get_month_start_and_end_date_in_utc, + get_number_of_days_for_month, +) from app.dao.fact_notification_status_dao import ( fetch_monthly_template_usage_for_service, fetch_notification_status_for_service_by_month, @@ -64,12 +68,14 @@ from app.dao.services_dao import ( dao_fetch_live_services_data, dao_fetch_service_by_id, dao_fetch_stats_for_service_from_day, + dao_fetch_stats_for_service_from_day_for_user, dao_fetch_todays_stats_for_all_services, dao_fetch_todays_stats_for_service, dao_remove_user_from_service, dao_resume_service, dao_suspend_service, dao_update_service, + fetch_notification_stats_for_service_by_month_by_user, get_services_by_partial_name, ) from app.dao.templates_dao import dao_get_template_by_id @@ -223,20 +229,17 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): if days == 1: stats = {} - stats[start] = { - "value": statistics.format_statistics( - dao_fetch_stats_for_service_from_day(service_id, start_date) - ) - } + stats[start] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day(service_id, start_date) + ) else: stats = {} for d in range(days): new_date = start_date + timedelta(days=d) key = new_date.strftime("%Y-%m-%d") - value = statistics.format_statistics( + stats[key] = statistics.format_statistics( dao_fetch_stats_for_service_from_day(service_id, new_date) ) - stats[key] = {"value": value} return stats @@ -612,6 +615,7 @@ def get_monthly_notification_stats(service_id): stats = fetch_notification_status_for_service_by_month( start_date, end_date, service_id ) + statistics.add_monthly_notification_status_stats(data, stats) now = datetime.utcnow() @@ -624,6 +628,72 @@ def get_monthly_notification_stats(service_id): return jsonify(data=data) +@service_blueprint.route( + "//notifications//monthly", methods=["GET"] +) +def get_monthly_notification_stats_by_user(service_id, user_id): + # check service_id validity + dao_fetch_service_by_id(service_id) + # user = get_user_by_id(user_id=user_id) + + try: + year = int(request.args.get("year", "NaN")) + except ValueError: + raise InvalidRequest("Year must be a number", status_code=400) + + start_date, end_date = get_calendar_year(year) + + data = statistics.create_empty_monthly_notification_status_stats_dict(year) + + stats = fetch_notification_stats_for_service_by_month_by_user( + start_date, end_date, service_id, user_id + ) + + statistics.add_monthly_notification_status_stats(data, stats) + + now = datetime.utcnow() + if end_date > now: + todays_deltas = fetch_notification_status_for_service_for_day( + now, service_id=service_id + ) + statistics.add_monthly_notification_status_stats(data, todays_deltas) + + return jsonify(data=data) + + +@service_blueprint.route( + "//notifications//month", methods=["GET"] +) +def get_single_month_notification_stats_by_user(service_id, user_id): + # check service_id validity + dao_fetch_service_by_id(service_id) + + try: + month = int(request.args.get("month", "NaN")) + year = int(request.args.get("year", "NaN")) + except ValueError: + raise InvalidRequest( + "Both a month and year are required as numbers", status_code=400 + ) + + month_year = datetime(year, month, 10, 00, 00, 00) + days = get_number_of_days_for_month(year, month) + start_date, end_date = get_month_start_and_end_date_in_utc(month_year) + + stats = {} + for d in range(days): + new_date = start_date + timedelta(days=d) + if new_date <= end_date: + key = new_date.strftime("%Y-%m-%d") + stats[key] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day_for_user( + service_id, new_date, user_id + ) + ) + + return jsonify(stats) + + def get_detailed_service(service_id, today_only=False): service = dao_fetch_service_by_id(service_id) diff --git a/app/service/statistics.py b/app/service/statistics.py index 90b933960..a6b58e067 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -113,7 +113,6 @@ def create_empty_monthly_notification_status_stats_dict(year): def add_monthly_notification_status_stats(data, stats): for row in stats: month = row.month.strftime("%Y-%m") - data[month][row.notification_type][row.notification_status] += row.count - + data[month][row.notification_type][StatisticsType.REQUESTED] += row.count return data From 140e40ebe0b3d0cb4f052e6365f785f4b8f0e06a Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Sat, 25 May 2024 22:14:51 -0600 Subject: [PATCH 03/16] Updated pytest for new return values --- tests/app/service/test_statistics.py | 17 ++++++++++++++--- tests/app/service/test_statistics_rest.py | 23 +++++++++++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index 28484a8d6..b58b85f78 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -307,6 +307,7 @@ def test_add_monthly_notification_status_stats(): add_monthly_notification_status_stats(data, rows) # first 3 months are empty + print("********* ", data) assert data == { "2018-01": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, "2018-02": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, @@ -315,12 +316,22 @@ def test_add_monthly_notification_status_stats(): NotificationType.SMS: { NotificationStatus.SENDING: 1, NotificationStatus.DELIVERED: 2, + StatisticsType.REQUESTED: 3, + }, + NotificationType.EMAIL: { + NotificationStatus.SENDING: 4, + StatisticsType.REQUESTED: 4, }, - NotificationType.EMAIL: {NotificationStatus.SENDING: 4}, }, "2018-05": { - NotificationType.SMS: {NotificationStatus.SENDING: 24}, - NotificationType.EMAIL: {NotificationStatus.SENDING: 32}, + NotificationType.SMS: { + NotificationStatus.SENDING: 24, + StatisticsType.REQUESTED: 8, + }, + NotificationType.EMAIL: { + NotificationStatus.SENDING: 32, + StatisticsType.REQUESTED: 32, + }, }, "2018-06": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, } diff --git a/tests/app/service/test_statistics_rest.py b/tests/app/service/test_statistics_rest.py index 522c3902b..735730d63 100644 --- a/tests/app/service/test_statistics_rest.py +++ b/tests/app/service/test_statistics_rest.py @@ -255,7 +255,8 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv assert response["data"]["2016-06"] == { NotificationType.SMS: { # it combines the two days - NotificationStatus.DELIVERED: 2 + NotificationStatus.DELIVERED: 2, + StatisticsType.REQUESTED: 2, }, NotificationType.EMAIL: {}, } @@ -264,8 +265,12 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv NotificationType.SMS: { NotificationStatus.CREATED: 1, NotificationStatus.DELIVERED: 2, + StatisticsType.REQUESTED: 3, + }, + NotificationType.EMAIL: { + StatisticsType.DELIVERED: 1, + StatisticsType.REQUESTED: 1, }, - NotificationType.EMAIL: {StatisticsType.DELIVERED: 1}, } @@ -311,7 +316,10 @@ def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( assert len(response["data"]) == 6 # January to June assert response["data"]["2016-05"] == { - NotificationType.SMS: {NotificationStatus.DELIVERED: 1}, + NotificationType.SMS: { + NotificationStatus.DELIVERED: 1, + StatisticsType.REQUESTED: 1, + }, NotificationType.EMAIL: {}, } assert response["data"]["2016-06"] == { @@ -319,6 +327,7 @@ def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( # combines the stats from the historic ft_notification_status and the current notifications NotificationStatus.CREATED: 3, NotificationStatus.DELIVERED: 1, + StatisticsType.REQUESTED: 4, }, NotificationType.EMAIL: {}, } @@ -354,6 +363,7 @@ def test_get_monthly_notification_stats_ignores_test_keys( assert response["data"]["2016-06"][NotificationType.SMS] == { NotificationStatus.DELIVERED: 3, + StatisticsType.REQUESTED: 3, } @@ -385,9 +395,11 @@ def test_get_monthly_notification_stats_checks_dates(admin_request, sample_servi assert "2017-04" not in response["data"] assert response["data"]["2016-04"][NotificationType.SMS] == { NotificationStatus.SENDING: 1, + StatisticsType.REQUESTED: 1, } assert response["data"]["2016-04"][NotificationType.SMS] == { NotificationStatus.SENDING: 1, + StatisticsType.REQUESTED: 1, } @@ -416,6 +428,9 @@ def test_get_monthly_notification_stats_only_gets_for_one_service( ) assert response["data"]["2016-06"] == { - NotificationType.SMS: {NotificationStatus.CREATED: 1}, + NotificationType.SMS: { + NotificationStatus.CREATED: 1, + StatisticsType.REQUESTED: 1, + }, NotificationType.EMAIL: {}, } From 41f24162162fe9e138a6e41ee81279aa083ea946 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Mon, 3 Jun 2024 21:38:29 -0600 Subject: [PATCH 04/16] reversed direction of the day count --- app/service/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/service/rest.py b/app/service/rest.py index 7d8d53c8e..0d015c08a 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -235,7 +235,7 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): else: stats = {} for d in range(days): - new_date = start_date + timedelta(days=d) + new_date = start_date - timedelta(days=d) key = new_date.strftime("%Y-%m-%d") stats[key] = statistics.format_statistics( dao_fetch_stats_for_service_from_day(service_id, new_date) From a5055a0cf991aa710999b17258746839f7de9421 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 6 Jun 2024 16:00:12 -0600 Subject: [PATCH 05/16] Updated endpoints to use the NotificationAllTimeView which is a view created to merge notifiations and notification_history --- app/dao/services_dao.py | 63 +++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 36427d8a8..67fa69da2 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -28,6 +28,7 @@ from app.models import ( Job, Notification, NotificationHistory, + NotificationAllTimeView, Organization, Permission, Service, @@ -434,19 +435,19 @@ def dao_fetch_stats_for_service_from_day(service_id, day): end_date = get_midnight_in_utc(day + timedelta(days=1)) return ( db.session.query( - NotificationHistory.notification_type, - NotificationHistory.status, - func.count(NotificationHistory.id).label("count"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.count(NotificationAllTimeView.id).label("count"), ) .filter( - NotificationHistory.service_id == service_id, - NotificationHistory.key_type != KeyType.TEST, - NotificationHistory.created_at >= start_date, - NotificationHistory.created_at <= end_date, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at <= end_date, ) .group_by( - NotificationHistory.notification_type, - NotificationHistory.status, + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, ) .all() ) @@ -460,20 +461,20 @@ def dao_fetch_stats_for_service_from_day_for_user(service_id, day, user_id): end_date = get_midnight_in_utc(day + timedelta(days=1)) return ( db.session.query( - NotificationHistory.notification_type, - NotificationHistory.status, - func.count(NotificationHistory.id).label("count"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.count(NotificationAllTimeView.id).label("count"), ) .filter( - NotificationHistory.service_id == service_id, - NotificationHistory.key_type != KeyType.TEST, - NotificationHistory.created_at >= start_date, - NotificationHistory.created_at <= end_date, - NotificationHistory.created_by_id == user_id, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at <= end_date, + NotificationAllTimeView.created_by_id == user_id, ) .group_by( - NotificationHistory.notification_type, - NotificationHistory.status, + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, ) .all() ) @@ -667,22 +668,22 @@ def fetch_notification_stats_for_service_by_month_by_user( ): return ( db.session.query( - func.date_trunc("month", NotificationHistory.created_at).label("month"), - NotificationHistory.notification_type, - (NotificationHistory.status).label("notification_status"), - func.count(NotificationHistory.id).label("count"), + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + (NotificationAllTimeView.status).label("notification_status"), + func.count(NotificationAllTimeView.id).label("count"), ) .filter( - NotificationHistory.service_id == service_id, - NotificationHistory.created_at >= start_date, - NotificationHistory.created_at < end_date, - NotificationHistory.key_type != KeyType.TEST, - NotificationHistory.created_by_id == user_id, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_by_id == user_id, ) .group_by( - func.date_trunc("month", NotificationHistory.created_at).label("month"), - NotificationHistory.notification_type, - NotificationHistory.status, + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, ) .all() ) From cd188180ca6f289411fcecbc067c747e488e55f9 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 6 Jun 2024 16:37:12 -0600 Subject: [PATCH 06/16] Added new endpoint for getting statistics for a service, for a user, by a number of days --- app/service/rest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/app/service/rest.py b/app/service/rest.py index 117b414df..dafccbbe3 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -243,6 +243,32 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): return stats +@service_blueprint.route("//statistics/user///") +def get_service_notification_statistics_by_day_by_user(service_id, user_id, start, days): + return jsonify( + data=get_service_statistics_for_specific_days_by_user(service_id, user_id, start, int(days)) + ) + + +def get_service_statistics_for_specific_days_by_user(service_id, user_id, start, days=1): + start_date = datetime.strptime(start, "%Y-%m-%d").date() + + if days == 1: + stats = {} + stats[start] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day_for_user(service_id, start_date, user_id) + ) + else: + stats = {} + for d in range(days): + new_date = start_date - timedelta(days=d) + key = new_date.strftime("%Y-%m-%d") + stats[key] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day_for_user(service_id, new_date, user_id) + ) + + return stats + @service_blueprint.route("", methods=["POST"]) def create_service(): From 0082ba3dd0527b43c73d7f7dab3f58958a2ed622 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 6 Jun 2024 22:11:24 -0600 Subject: [PATCH 07/16] Updated sort order, endpoint defenition, and tests for REQUESTED attribute --- app/dao/services_dao.py | 2 +- app/service/rest.py | 25 +++++++++++++++++++------ tests/app/service/test_statistics.py | 6 ++++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 67fa69da2..e2082a1ba 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -27,8 +27,8 @@ from app.models import ( InvitedUser, Job, Notification, - NotificationHistory, NotificationAllTimeView, + NotificationHistory, Organization, Permission, Service, diff --git a/app/service/rest.py b/app/service/rest.py index dafccbbe3..372dec7a5 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -243,20 +243,31 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): return stats -@service_blueprint.route("//statistics/user///") -def get_service_notification_statistics_by_day_by_user(service_id, user_id, start, days): + +@service_blueprint.route( + "//statistics/user///" +) +def get_service_notification_statistics_by_day_by_user( + service_id, user_id, start, days +): return jsonify( - data=get_service_statistics_for_specific_days_by_user(service_id, user_id, start, int(days)) + data=get_service_statistics_for_specific_days_by_user( + service_id, user_id, start, int(days) + ) ) -def get_service_statistics_for_specific_days_by_user(service_id, user_id, start, days=1): +def get_service_statistics_for_specific_days_by_user( + service_id, user_id, start, days=1 +): start_date = datetime.strptime(start, "%Y-%m-%d").date() if days == 1: stats = {} stats[start] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user(service_id, start_date, user_id) + dao_fetch_stats_for_service_from_day_for_user( + service_id, start_date, user_id + ) ) else: stats = {} @@ -264,7 +275,9 @@ def get_service_statistics_for_specific_days_by_user(service_id, user_id, start, new_date = start_date - timedelta(days=d) key = new_date.strftime("%Y-%m-%d") stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user(service_id, new_date, user_id) + dao_fetch_stats_for_service_from_day_for_user( + service_id, new_date, user_id + ) ) return stats diff --git a/tests/app/service/test_statistics.py b/tests/app/service/test_statistics.py index b58b85f78..c760d01b8 100644 --- a/tests/app/service/test_statistics.py +++ b/tests/app/service/test_statistics.py @@ -301,13 +301,15 @@ def test_add_monthly_notification_status_stats(): data = create_empty_monthly_notification_status_stats_dict(2018) # this data won't be affected data["2018-05"][NotificationType.EMAIL][NotificationStatus.SENDING] = 32 + data["2018-05"][NotificationType.EMAIL][StatisticsType.REQUESTED] = 32 # this data will get combined with the 8 from row_data data["2018-05"][NotificationType.SMS][NotificationStatus.SENDING] = 16 + data["2018-05"][NotificationType.SMS][StatisticsType.REQUESTED] = 16 add_monthly_notification_status_stats(data, rows) # first 3 months are empty - print("********* ", data) + assert data == { "2018-01": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, "2018-02": {NotificationType.SMS: {}, NotificationType.EMAIL: {}}, @@ -326,7 +328,7 @@ def test_add_monthly_notification_status_stats(): "2018-05": { NotificationType.SMS: { NotificationStatus.SENDING: 24, - StatisticsType.REQUESTED: 8, + StatisticsType.REQUESTED: 24, }, NotificationType.EMAIL: { NotificationStatus.SENDING: 32, From e293f7e3f5a26a13cc04d787bd0f3d7af65629b2 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Fri, 14 Jun 2024 16:01:04 -0600 Subject: [PATCH 08/16] Updated all usage of datetime.utcnow() to app.utils utc_now() function. Added new endpoint /service/{{service_id}}/notifications/month --- app/service/rest.py | 31 ++++++++++++++++++- .../versions/0025_notify_service_data.py | 23 +++++++------- .../versions/0028_fix_reg_template_history.py | 4 ++- .../versions/0082_add_golive_template.py | 4 ++- .../versions/0117_international_sms_notify.py | 4 ++- .../versions/0134_add_email_2fa_template_.py | 4 ++- .../0139_migrate_sms_allowance_data.py | 3 +- .../versions/0171_add_org_invite_template.py | 4 ++- .../0265_add_confirm_edit_templates.py | 6 ++-- .../versions/0294_add_verify_reply_to_.py | 4 ++- .../versions/0330_broadcast_invite_email.py | 4 ++- .../0347_add_dvla_volumes_template.py | 4 ++- migrations/versions/0401_add_e2e_test_user.py | 7 +++-- 13 files changed, 76 insertions(+), 26 deletions(-) diff --git a/app/service/rest.py b/app/service/rest.py index 372dec7a5..29408f348 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -706,7 +706,7 @@ def get_monthly_notification_stats_by_user(service_id, user_id): statistics.add_monthly_notification_status_stats(data, stats) - now = datetime.utcnow() + now = utc_now() if end_date > now: todays_deltas = fetch_notification_status_for_service_for_day( now, service_id=service_id @@ -749,6 +749,35 @@ def get_single_month_notification_stats_by_user(service_id, user_id): return jsonify(stats) +@service_blueprint.route("//notifications/month", methods=["GET"]) +def get_single_month_notification_stats_for_service(service_id): + # check service_id validity + dao_fetch_service_by_id(service_id) + + try: + month = int(request.args.get("month", "NaN")) + year = int(request.args.get("year", "NaN")) + except ValueError: + raise InvalidRequest( + "Both a month and year are required as numbers", status_code=400 + ) + + month_year = datetime(year, month, 10, 00, 00, 00) + days = get_number_of_days_for_month(year, month) + start_date, end_date = get_month_start_and_end_date_in_utc(month_year) + + stats = {} + for d in range(days): + new_date = start_date + timedelta(days=d) + if new_date <= end_date: + key = new_date.strftime("%Y-%m-%d") + stats[key] = statistics.format_statistics( + dao_fetch_stats_for_service_from_day(service_id, new_date) + ) + + return jsonify(stats) + + def get_detailed_service(service_id, today_only=False): service = dao_fetch_service_by_id(service_id) diff --git a/migrations/versions/0025_notify_service_data.py b/migrations/versions/0025_notify_service_data.py index 0683e7dd2..e90d01aad 100644 --- a/migrations/versions/0025_notify_service_data.py +++ b/migrations/versions/0025_notify_service_data.py @@ -15,6 +15,7 @@ from alembic import op from sqlalchemy import text from app.hashing import hashpw +from app.utils import utc_now revision = "0025_notify_service_data" down_revision = "0024_add_research_mode_defaults" @@ -32,7 +33,7 @@ def upgrade(): """ conn.execute( text(user_insert), - {"user_id": user_id, "time_now": datetime.utcnow(), "password": password}, + {"user_id": user_id, "time_now": utc_now(), "password": password}, ) service_history_insert = """INSERT INTO services_history (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com', @@ -41,7 +42,7 @@ def upgrade(): """ conn.execute( text(service_history_insert), - {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id}, + {"service_id": service_id, "time_now": utc_now(), "user_id": user_id}, ) service_insert = """INSERT INTO services (id, name, created_at, active, message_limit, restricted, research_mode, email_from, created_by_id, reply_to_email_address, version) VALUES (:service_id, 'Notify service', :time_now, True, 1000, False, False, 'testsender@dispostable.com', @@ -49,7 +50,7 @@ def upgrade(): """ conn.execute( text(service_insert), - {"service_id": service_id, "time_now": datetime.utcnow(), "user_id": user_id}, + {"service_id": service_id, "time_now": utc_now(), "user_id": user_id}, ) user_to_service_insert = """INSERT INTO user_to_service (user_id, service_id) VALUES (:user_id, :service_id)""" conn.execute( @@ -74,7 +75,7 @@ def upgrade(): "template_id": uuid.uuid4(), "template_name": "Notify email verification code", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_verification_content, "service_id": service_id, "subject": "Confirm GOV.UK Notify registration", @@ -87,7 +88,7 @@ def upgrade(): "template_id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27", "template_name": "Notify email verification code", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_verification_content, "service_id": service_id, "subject": "Confirm GOV.UK Notify registration", @@ -107,7 +108,7 @@ def upgrade(): "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc", "template_name": "Notify invitation email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": invitation_content, "service_id": service_id, "subject": invitation_subject, @@ -120,7 +121,7 @@ def upgrade(): "template_id": "4f46df42-f795-4cc4-83bb-65ca312f49cc", "template_name": "Notify invitation email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": invitation_content, "service_id": service_id, "subject": invitation_subject, @@ -135,7 +136,7 @@ def upgrade(): "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246", "template_name": "Notify SMS verify code", "template_type": "sms", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": sms_code_content, "service_id": service_id, "subject": None, @@ -149,7 +150,7 @@ def upgrade(): "template_id": "36fb0730-6259-4da1-8a80-c8de22ad4246", "template_name": "Notify SMS verify code", "template_type": "sms", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": sms_code_content, "service_id": service_id, "subject": None, @@ -172,7 +173,7 @@ def upgrade(): "template_id": "474e9242-823b-4f99-813d-ed392e7f1201", "template_name": "Notify password reset email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": password_reset_content, "service_id": service_id, "subject": "Reset your GOV.UK Notify password", @@ -186,7 +187,7 @@ def upgrade(): "template_id": "474e9242-823b-4f99-813d-ed392e7f1201", "template_name": "Notify password reset email", "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": password_reset_content, "service_id": service_id, "subject": "Reset your GOV.UK Notify password", diff --git a/migrations/versions/0028_fix_reg_template_history.py b/migrations/versions/0028_fix_reg_template_history.py index 8bf11fe59..fcbffc51a 100644 --- a/migrations/versions/0028_fix_reg_template_history.py +++ b/migrations/versions/0028_fix_reg_template_history.py @@ -11,6 +11,8 @@ from datetime import datetime from sqlalchemy import text +from app.utils import utc_now + revision = "0028_fix_reg_template_history" down_revision = "0026_rename_notify_service" @@ -38,7 +40,7 @@ def upgrade(): "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27", "name": "Notify email verification code", "type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_verification_content, "service_id": service_id, "subject": "Confirm GOV.UK Notify registration", diff --git a/migrations/versions/0082_add_golive_template.py b/migrations/versions/0082_add_golive_template.py index 55cdc3e04..97ff5b97e 100644 --- a/migrations/versions/0082_add_golive_template.py +++ b/migrations/versions/0082_add_golive_template.py @@ -14,6 +14,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0082_add_go_live_template" down_revision = "0081_noti_status_as_enum" @@ -89,7 +91,7 @@ GOV.UK Notify team "template_id": template_id, "template_name": template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": template_subject, diff --git a/migrations/versions/0117_international_sms_notify.py b/migrations/versions/0117_international_sms_notify.py index ebdbbddef..c33750af4 100644 --- a/migrations/versions/0117_international_sms_notify.py +++ b/migrations/versions/0117_international_sms_notify.py @@ -9,6 +9,8 @@ Create Date: 2017-08-29 14:09:41.042061 # revision identifiers, used by Alembic. from sqlalchemy import text +from app.utils import utc_now + revision = "0117_international_sms_notify" down_revision = "0115_add_inbound_numbers" @@ -22,7 +24,7 @@ NOTIFY_SERVICE_ID = "d6aa2c68-a2d9-4437-ab19-3ae8eb202553" def upgrade(): input_params = { "notify_service_id": NOTIFY_SERVICE_ID, - "datetime_now": datetime.utcnow(), + "datetime_now": utc_now(), } conn = op.get_bind() conn.execute( diff --git a/migrations/versions/0134_add_email_2fa_template_.py b/migrations/versions/0134_add_email_2fa_template_.py index 492281175..57809e1bc 100644 --- a/migrations/versions/0134_add_email_2fa_template_.py +++ b/migrations/versions/0134_add_email_2fa_template_.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0134_add_email_2fa_template" down_revision = "0133_set_services_sms_prefix" @@ -44,7 +46,7 @@ def upgrade(): "template_id": template_id, "template_name": template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": template_subject, diff --git a/migrations/versions/0139_migrate_sms_allowance_data.py b/migrations/versions/0139_migrate_sms_allowance_data.py index 8e7536bfb..0203f5563 100644 --- a/migrations/versions/0139_migrate_sms_allowance_data.py +++ b/migrations/versions/0139_migrate_sms_allowance_data.py @@ -13,6 +13,7 @@ from alembic import op from sqlalchemy import text from app.dao.date_util import get_current_calendar_year_start_year +from app.utils import utc_now revision = "0139_migrate_sms_allowance_data" down_revision = "0138_sms_sender_nullable" @@ -34,7 +35,7 @@ def upgrade(): input_params = { "current_year": current_year, "default_limit": default_limit, - "time_now": datetime.utcnow(), + "time_now": utc_now(), } insert_row_if_not_exist = """ INSERT INTO annual_billing diff --git a/migrations/versions/0171_add_org_invite_template.py b/migrations/versions/0171_add_org_invite_template.py index 5ec1925da..7c0b9df09 100644 --- a/migrations/versions/0171_add_org_invite_template.py +++ b/migrations/versions/0171_add_org_invite_template.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0171_add_org_invite_template" down_revision = "0170_hidden_non_nullable" @@ -53,7 +55,7 @@ def upgrade(): "template_id": template_id, "template_name": template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": template_subject, diff --git a/migrations/versions/0265_add_confirm_edit_templates.py b/migrations/versions/0265_add_confirm_edit_templates.py index 378891313..ad8d2470e 100644 --- a/migrations/versions/0265_add_confirm_edit_templates.py +++ b/migrations/versions/0265_add_confirm_edit_templates.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0265_add_confirm_edit_templates" down_revision = "0264_add_folder_permissions_perm" @@ -57,7 +59,7 @@ def upgrade(): "template_id": email_template_id, "template_name": email_template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": email_template_subject, @@ -78,7 +80,7 @@ def upgrade(): "template_id": mobile_template_id, "template_name": mobile_template_name, "template_type": "sms", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": mobile_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": None, diff --git a/migrations/versions/0294_add_verify_reply_to_.py b/migrations/versions/0294_add_verify_reply_to_.py index 305c52997..d37f75ea0 100644 --- a/migrations/versions/0294_add_verify_reply_to_.py +++ b/migrations/versions/0294_add_verify_reply_to_.py @@ -12,6 +12,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0294_add_verify_reply_to" down_revision = "0293_drop_complaint_fk" @@ -58,7 +60,7 @@ def upgrade(): "template_id": email_template_id, "template_name": email_template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": email_template_subject, diff --git a/migrations/versions/0330_broadcast_invite_email.py b/migrations/versions/0330_broadcast_invite_email.py index 24dc60c68..bcd865d46 100644 --- a/migrations/versions/0330_broadcast_invite_email.py +++ b/migrations/versions/0330_broadcast_invite_email.py @@ -12,6 +12,8 @@ from datetime import datetime from alembic import op from sqlalchemy import text +from app.utils import utc_now + revision = "0330_broadcast_invite_email" down_revision = "0329_purge_broadcast_data" @@ -60,7 +62,7 @@ def upgrade(): input_params = { "template_id": template_id, "template_name": broadcast_invitation_template_name, - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": broadcast_invitation_content, "service_id": service_id, "subject": broadcast_invitation_subject, diff --git a/migrations/versions/0347_add_dvla_volumes_template.py b/migrations/versions/0347_add_dvla_volumes_template.py index 6b610c16c..f32821b79 100644 --- a/migrations/versions/0347_add_dvla_volumes_template.py +++ b/migrations/versions/0347_add_dvla_volumes_template.py @@ -13,6 +13,8 @@ from alembic import op from flask import current_app from sqlalchemy import text +from app.utils import utc_now + revision = "0347_add_dvla_volumes_template" down_revision = "0346_notify_number_sms_sender" @@ -57,7 +59,7 @@ def upgrade(): "template_id": email_template_id, "template_name": email_template_name, "template_type": "email", - "time_now": datetime.utcnow(), + "time_now": utc_now(), "content": email_template_content, "notify_service_id": current_app.config["NOTIFY_SERVICE_ID"], "subject": email_template_subject, diff --git a/migrations/versions/0401_add_e2e_test_user.py b/migrations/versions/0401_add_e2e_test_user.py index d3d83afad..e99a6af8a 100644 --- a/migrations/versions/0401_add_e2e_test_user.py +++ b/migrations/versions/0401_add_e2e_test_user.py @@ -16,6 +16,7 @@ from alembic import op from app import db from app.dao.users_dao import get_user_by_email from app.models import User +from app.utils import utc_now revision = "0401_add_e2e_test_user" down_revision = "0400_add_total_message_limit" @@ -32,11 +33,11 @@ def upgrade(): "password": password, "mobile_number": "+12025555555", "state": "active", - "created_at": datetime.datetime.utcnow(), - "password_changed_at": datetime.datetime.utcnow(), + "created_at": utc_now(), + "password_changed_at": utc_now(), "failed_login_count": 0, "platform_admin": "f", - "email_access_validated_at": datetime.datetime.utcnow(), + "email_access_validated_at": utc_now(), } conn = op.get_bind() insert_sql = """ From fd37923294a4103af28d851890fb75228e6a6ed3 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 20 Jun 2024 01:04:31 -0600 Subject: [PATCH 09/16] Updated SQLAlchemy queries and API endpoints for single database queries to improve application performance. --- app/dao/services_dao.py | 24 +++--- app/service/rest.py | 167 +++++++++++++++++++++++++++------------- 2 files changed, 125 insertions(+), 66 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index e2082a1ba..7b16aa8b3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -427,16 +427,15 @@ def dao_fetch_todays_stats_for_service(service_id): ) -def dao_fetch_stats_for_service_from_day(service_id, day): - # today = datetime.now(timezone.utc).date() - # 2024-05-20 - # day_date = datetime.strptime(day, '%Y-%m-%d').date() - start_date = get_midnight_in_utc(day) - end_date = get_midnight_in_utc(day + timedelta(days=1)) +def dao_fetch_stats_for_service_from_days(service_id, start, days): + start_date = get_midnight_in_utc(start) + end_date = get_midnight_in_utc(start + timedelta(days=days)) + return ( db.session.query( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), func.count(NotificationAllTimeView.id).label("count"), ) .filter( @@ -448,21 +447,21 @@ def dao_fetch_stats_for_service_from_day(service_id, day): .group_by( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at), ) .all() ) -def dao_fetch_stats_for_service_from_day_for_user(service_id, day, user_id): - # today = datetime.now(timezone.utc).date() - # 2024-05-20 - # day_date = datetime.strptime(day, '%Y-%m-%d').date() - start_date = get_midnight_in_utc(day) - end_date = get_midnight_in_utc(day + timedelta(days=1)) +def dao_fetch_stats_for_service_from_days_for_user(service_id, start, days, user_id): + start_date = get_midnight_in_utc(start) + end_date = get_midnight_in_utc(start + timedelta(days=days)) + return ( db.session.query( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), func.count(NotificationAllTimeView.id).label("count"), ) .filter( @@ -475,6 +474,7 @@ def dao_fetch_stats_for_service_from_day_for_user(service_id, day, user_id): .group_by( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, + func.date_trunc("day", NotificationAllTimeView.created_at), ) .all() ) diff --git a/app/service/rest.py b/app/service/rest.py index 29408f348..10cd84b46 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,4 +1,5 @@ import itertools +from collections import defaultdict from datetime import datetime, timedelta from flask import Blueprint, current_app, jsonify, request @@ -17,11 +18,7 @@ from app.dao.api_key_dao import ( save_model_api_key, ) from app.dao.dao_utils import dao_rollback, transaction -from app.dao.date_util import ( - get_calendar_year, - get_month_start_and_end_date_in_utc, - get_number_of_days_for_month, -) +from app.dao.date_util import get_calendar_year, get_month_start_and_end_date_in_utc from app.dao.fact_notification_status_dao import ( fetch_monthly_template_usage_for_service, fetch_notification_status_for_service_by_month, @@ -67,8 +64,8 @@ from app.dao.services_dao import ( dao_fetch_all_services_by_user, dao_fetch_live_services_data, dao_fetch_service_by_id, - dao_fetch_stats_for_service_from_day, - dao_fetch_stats_for_service_from_day_for_user, + dao_fetch_stats_for_service_from_days, + dao_fetch_stats_for_service_from_days_for_user, dao_fetch_todays_stats_for_all_services, dao_fetch_todays_stats_for_service, dao_remove_user_from_service, @@ -227,19 +224,33 @@ def get_service_notification_statistics_by_day(service_id, start, days): def get_service_statistics_for_specific_days(service_id, start, days=1): start_date = datetime.strptime(start, "%Y-%m-%d").date() - if days == 1: - stats = {} - stats[start] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day(service_id, start_date) - ) - else: - stats = {} - for d in range(days): - new_date = start_date - timedelta(days=d) - key = new_date.strftime("%Y-%m-%d") - stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day(service_id, new_date) - ) + def generate_date_range(start_date, days): + current_date = start_date + end_date = start_date - timedelta(days=days) + while current_date > end_date: + try: + valid_date = datetime( + current_date.year, current_date.month, current_date.day + ) + yield valid_date.date() + except ValueError: + pass + current_date -= timedelta(days=1) + + results = dao_fetch_stats_for_service_from_days(service_id, start_date, days) + + grouped_results = defaultdict(list) + for row in results: + notification_type, status, day, count = row + grouped_results[day.date()].append(row) + + for date in generate_date_range(start_date, days): + if date not in grouped_results: + grouped_results[date] = [] + + stats = {} + for day, rows in grouped_results.items(): + stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) return stats @@ -262,23 +273,35 @@ def get_service_statistics_for_specific_days_by_user( ): start_date = datetime.strptime(start, "%Y-%m-%d").date() - if days == 1: - stats = {} - stats[start] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user( - service_id, start_date, user_id - ) - ) - else: - stats = {} - for d in range(days): - new_date = start_date - timedelta(days=d) - key = new_date.strftime("%Y-%m-%d") - stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user( - service_id, new_date, user_id + def generate_date_range(start_date, days): + current_date = start_date + end_date = start_date - timedelta(days=days) + while current_date > end_date: + try: + valid_date = datetime( + current_date.year, current_date.month, current_date.day ) - ) + yield valid_date.date() + except ValueError: + pass + current_date -= timedelta(days=1) + + results = dao_fetch_stats_for_service_from_days_for_user( + service_id, start_date, days, user_id + ) + + grouped_results = defaultdict(list) + for row in results: + notification_type, status, day, count = row + grouped_results[day.date()].append(row) + + for date in generate_date_range(start_date, days): + if date not in grouped_results: + grouped_results[date] = [] + + stats = {} + for day, rows in grouped_results.items(): + stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) return stats @@ -732,19 +755,37 @@ def get_single_month_notification_stats_by_user(service_id, user_id): ) month_year = datetime(year, month, 10, 00, 00, 00) - days = get_number_of_days_for_month(year, month) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) - stats = {} - for d in range(days): - new_date = start_date + timedelta(days=d) - if new_date <= end_date: - key = new_date.strftime("%Y-%m-%d") - stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day_for_user( - service_id, new_date, user_id + def generate_date_range(start_date, end_date): + current_date = start_date + end_date = end_date + while current_date < end_date: + try: + valid_date = datetime( + current_date.year, current_date.month, current_date.day ) - ) + yield valid_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + + results = dao_fetch_stats_for_service_from_days_for_user( + service_id, start_date, user_id + ) + + grouped_results = defaultdict(list) + for row in results: + notification_type, status, day, count = row + grouped_results[day.date()].append(row) + + for date in generate_date_range(start_date, end_date): + if date not in grouped_results: + grouped_results[date] = [] + + stats = {} + for day, rows in grouped_results.items(): + stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) return jsonify(stats) @@ -763,17 +804,35 @@ def get_single_month_notification_stats_for_service(service_id): ) month_year = datetime(year, month, 10, 00, 00, 00) - days = get_number_of_days_for_month(year, month) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) + def generate_date_range(start_date, end_date): + current_date = start_date + end_date = end_date + while current_date < end_date: + try: + valid_date = datetime( + current_date.year, current_date.month, current_date.day + ) + yield valid_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + + results = dao_fetch_stats_for_service_from_days(service_id, start_date) + + grouped_results = defaultdict(list) + for row in results: + notification_type, status, day, count = row + grouped_results[day.date()].append(row) + + for date in generate_date_range(start_date, end_date): + if date not in grouped_results: + grouped_results[date] = [] + stats = {} - for d in range(days): - new_date = start_date + timedelta(days=d) - if new_date <= end_date: - key = new_date.strftime("%Y-%m-%d") - stats[key] = statistics.format_statistics( - dao_fetch_stats_for_service_from_day(service_id, new_date) - ) + for day, rows in grouped_results.items(): + stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) return jsonify(stats) From 966f9b405059d95cd229fa05af90c5cd4ecde7e9 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Thu, 20 Jun 2024 23:12:47 -0600 Subject: [PATCH 10/16] moved generate_date_range to date_util standardized the SQLAlchemy calls refactored the endpoints in service/rest.py --- app/dao/date_util.py | 22 +++++++++++ app/dao/services_dao.py | 14 ++++--- app/service/rest.py | 84 +++++++++++------------------------------ 3 files changed, 53 insertions(+), 67 deletions(-) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 7acc587aa..cac09dee2 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -71,3 +71,25 @@ def get_calendar_year_for_datetime(start_date): def get_number_of_days_for_month(year, month): return calendar.monthrange(year, month)[1] + + +def generate_date_range(start_date, end_date=None, days=0): + if end_date: + current_date = start_date + while current_date <= end_date: + try: + yield current_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + elif days > 0: + end_date = start_date + timedelta(days=days) + current_date = start_date + while current_date < end_date: + try: + yield current_date.date() + except ValueError: + pass + current_date += timedelta(days=1) + else: + return "A start_date or number of days must be specified" diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 7b16aa8b3..6aa0d42f3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -427,9 +427,9 @@ def dao_fetch_todays_stats_for_service(service_id): ) -def dao_fetch_stats_for_service_from_days(service_id, start, days): - start_date = get_midnight_in_utc(start) - end_date = get_midnight_in_utc(start + timedelta(days=days)) +def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): + start_date = get_midnight_in_utc(start_date) + end_date = get_midnight_in_utc(end_date) return ( db.session.query( @@ -453,9 +453,11 @@ def dao_fetch_stats_for_service_from_days(service_id, start, days): ) -def dao_fetch_stats_for_service_from_days_for_user(service_id, start, days, user_id): - start_date = get_midnight_in_utc(start) - end_date = get_midnight_in_utc(start + timedelta(days=days)) +def dao_fetch_stats_for_service_from_days_for_user( + service_id, start_date, end_date, user_id +): + start_date = get_midnight_in_utc(start_date) + end_date = get_midnight_in_utc(end_date) return ( db.session.query( diff --git a/app/service/rest.py b/app/service/rest.py index 204da0c80..ea59387d8 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -18,7 +18,11 @@ from app.dao.api_key_dao import ( save_model_api_key, ) from app.dao.dao_utils import dao_rollback, transaction -from app.dao.date_util import get_calendar_year, get_month_start_and_end_date_in_utc +from app.dao.date_util import ( + generate_date_range, + get_calendar_year, + get_month_start_and_end_date_in_utc, +) from app.dao.fact_notification_status_dao import ( fetch_monthly_template_usage_for_service, fetch_notification_status_for_service_by_month, @@ -222,29 +226,20 @@ def get_service_notification_statistics_by_day(service_id, start, days): def get_service_statistics_for_specific_days(service_id, start, days=1): - start_date = datetime.strptime(start, "%Y-%m-%d").date() + # start and end dates needs to be reversed because + # the end date is today and the start is x days in the past + # a day needs to be substracted to allow for today + end_date = datetime.strptime(start, "%Y-%m-%d") + start_date = end_date - timedelta(days=days - 1) - def generate_date_range(start_date, days): - current_date = start_date - end_date = start_date - timedelta(days=days) - while current_date > end_date: - try: - valid_date = datetime( - current_date.year, current_date.month, current_date.day - ) - yield valid_date.date() - except ValueError: - pass - current_date -= timedelta(days=1) - - results = dao_fetch_stats_for_service_from_days(service_id, start_date, days) + results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) grouped_results = defaultdict(list) for row in results: notification_type, status, day, count = row grouped_results[day.date()].append(row) - for date in generate_date_range(start_date, days): + for date in generate_date_range(start_date, days=days): if date not in grouped_results: grouped_results[date] = [] @@ -271,23 +266,14 @@ def get_service_notification_statistics_by_day_by_user( def get_service_statistics_for_specific_days_by_user( service_id, user_id, start, days=1 ): - start_date = datetime.strptime(start, "%Y-%m-%d").date() - - def generate_date_range(start_date, days): - current_date = start_date - end_date = start_date - timedelta(days=days) - while current_date > end_date: - try: - valid_date = datetime( - current_date.year, current_date.month, current_date.day - ) - yield valid_date.date() - except ValueError: - pass - current_date -= timedelta(days=1) + # start and end dates needs to be reversed because + # the end date is today and the start is x days in the past + # a day needs to be substracted to allow for today + end_date = datetime.strptime(start, "%Y-%m-%d") + start_date = end_date - timedelta(days=days - 1) results = dao_fetch_stats_for_service_from_days_for_user( - service_id, start_date, days, user_id + service_id, start_date, end_date, user_id ) grouped_results = defaultdict(list) @@ -295,7 +281,9 @@ def get_service_statistics_for_specific_days_by_user( notification_type, status, day, count = row grouped_results[day.date()].append(row) - for date in generate_date_range(start_date, days): + print(grouped_results) + + for date in generate_date_range(start_date, days=days): if date not in grouped_results: grouped_results[date] = [] @@ -752,21 +740,8 @@ def get_single_month_notification_stats_by_user(service_id, user_id): month_year = datetime(year, month, 10, 00, 00, 00) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) - def generate_date_range(start_date, end_date): - current_date = start_date - end_date = end_date - while current_date < end_date: - try: - valid_date = datetime( - current_date.year, current_date.month, current_date.day - ) - yield valid_date.date() - except ValueError: - pass - current_date += timedelta(days=1) - results = dao_fetch_stats_for_service_from_days_for_user( - service_id, start_date, user_id + service_id, start_date, end_date, user_id ) grouped_results = defaultdict(list) @@ -801,20 +776,7 @@ def get_single_month_notification_stats_for_service(service_id): month_year = datetime(year, month, 10, 00, 00, 00) start_date, end_date = get_month_start_and_end_date_in_utc(month_year) - def generate_date_range(start_date, end_date): - current_date = start_date - end_date = end_date - while current_date < end_date: - try: - valid_date = datetime( - current_date.year, current_date.month, current_date.day - ) - yield valid_date.date() - except ValueError: - pass - current_date += timedelta(days=1) - - results = dao_fetch_stats_for_service_from_days(service_id, start_date) + results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) grouped_results = defaultdict(list) for row in results: From d3d2610578b94e05407bbe1d3c1673c26bdb8930 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Fri, 21 Jun 2024 14:38:54 -0600 Subject: [PATCH 11/16] correct timedelta for midnight that offest calculation for today --- app/dao/services_dao.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 6aa0d42f3..dcf2536eb 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -429,7 +429,7 @@ def dao_fetch_todays_stats_for_service(service_id): def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): start_date = get_midnight_in_utc(start_date) - end_date = get_midnight_in_utc(end_date) + end_date = get_midnight_in_utc(end_date + timedelta(days=1)) return ( db.session.query( @@ -457,7 +457,7 @@ def dao_fetch_stats_for_service_from_days_for_user( service_id, start_date, end_date, user_id ): start_date = get_midnight_in_utc(start_date) - end_date = get_midnight_in_utc(end_date) + end_date = get_midnight_in_utc(end_date + timedelta(days=1)) return ( db.session.query( From 47c89647665b4cb464136c11f55b0e11814512f6 Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Fri, 21 Jun 2024 16:58:09 -0600 Subject: [PATCH 12/16] fetch_notification_status_for_service_by_month altered to use NotificationAllTimeView like the other endpoints --- app/dao/fact_notification_status_dao.py | 22 +++++++++++----------- app/dao/services_dao.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index 22c87fe83..f2769d2cd 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -84,21 +84,21 @@ def update_fact_notification_status(process_day, notification_type, service_id): def fetch_notification_status_for_service_by_month(start_date, end_date, service_id): return ( db.session.query( - func.date_trunc("month", FactNotificationStatus.local_date).label("month"), - FactNotificationStatus.notification_type, - FactNotificationStatus.notification_status, - func.sum(FactNotificationStatus.notification_count).label("count"), + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status.label('notification_status'), + func.count(NotificationAllTimeView.id).label("count"), ) .filter( - FactNotificationStatus.service_id == service_id, - FactNotificationStatus.local_date >= start_date, - FactNotificationStatus.local_date < end_date, - FactNotificationStatus.key_type != KeyType.TEST, + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + NotificationAllTimeView.key_type != KeyType.TEST, ) .group_by( - func.date_trunc("month", FactNotificationStatus.local_date).label("month"), - FactNotificationStatus.notification_type, - FactNotificationStatus.notification_status, + func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, ) .all() ) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index dcf2536eb..74f8094a8 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -442,7 +442,7 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, NotificationAllTimeView.created_at >= start_date, - NotificationAllTimeView.created_at <= end_date, + NotificationAllTimeView.created_at < end_date, ) .group_by( NotificationAllTimeView.notification_type, @@ -470,7 +470,7 @@ def dao_fetch_stats_for_service_from_days_for_user( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type != KeyType.TEST, NotificationAllTimeView.created_at >= start_date, - NotificationAllTimeView.created_at <= end_date, + NotificationAllTimeView.created_at < end_date, NotificationAllTimeView.created_by_id == user_id, ) .group_by( From b2e5522d09ec61a38262efac6770099363930cee Mon Sep 17 00:00:00 2001 From: Anastasia Gradova Date: Mon, 24 Jun 2024 17:58:37 -0600 Subject: [PATCH 13/16] Corrected test cases for new stats endpoints --- app/dao/fact_notification_status_dao.py | 2 +- .../dao/test_fact_notification_status_dao.py | 59 +++-- tests/app/service/test_statistics_rest.py | 220 ++++++++++-------- 3 files changed, 161 insertions(+), 120 deletions(-) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index f2769d2cd..a810ff0db 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -86,7 +86,7 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service db.session.query( func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), NotificationAllTimeView.notification_type, - NotificationAllTimeView.status.label('notification_status'), + NotificationAllTimeView.status.label("notification_status"), func.count(NotificationAllTimeView.id).label("count"), ) .filter( diff --git a/tests/app/dao/test_fact_notification_status_dao.py b/tests/app/dao/test_fact_notification_status_dao.py index 4c7030b2e..dc46de45d 100644 --- a/tests/app/dao/test_fact_notification_status_dao.py +++ b/tests/app/dao/test_fact_notification_status_dao.py @@ -33,31 +33,44 @@ def test_fetch_notification_status_for_service_by_month(notify_db_session): service_1 = create_service(service_name="service_1") service_2 = create_service(service_name="service_2") - create_ft_notification_status( - date(2018, 1, 1), NotificationType.SMS, service_1, count=4 - ) - create_ft_notification_status( - date(2018, 1, 2), NotificationType.SMS, service_1, count=10 - ) - create_ft_notification_status( - date(2018, 1, 2), - NotificationType.SMS, - service_1, - notification_status=NotificationStatus.CREATED, - ) - create_ft_notification_status(date(2018, 1, 3), NotificationType.EMAIL, service_1) + create_template(service=service_1) + create_template(service=service_1, template_type=TemplateType.EMAIL) + # not the service being tested + create_template(service=service_2) - create_ft_notification_status(date(2018, 2, 2), NotificationType.SMS, service_1) + # loop messages for the month + for x in range(0, 14): + create_notification( + service_1.templates[0], + created_at=datetime(2018, 1, 1, 1, x, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + service_1.templates[0], created_at=datetime(2018, 1, 1, 1, 1, 0) + ) + create_notification( + service_1.templates[1], + created_at=datetime(2018, 1, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + service_1.templates[0], + created_at=datetime(2018, 2, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) - # not included - too early - create_ft_notification_status(date(2017, 12, 31), NotificationType.SMS, service_1) - # not included - too late - create_ft_notification_status(date(2017, 3, 1), NotificationType.SMS, service_1) - # not included - wrong service - create_ft_notification_status(date(2018, 1, 3), NotificationType.SMS, service_2) - # not included - test keys - create_ft_notification_status( - date(2018, 1, 3), NotificationType.SMS, service_1, key_type=KeyType.TEST + # not the right month + create_notification( + service_1.templates[0], + created_at=datetime(2018, 4, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + + # not the right service + create_notification( + service_2.templates[0], + created_at=datetime(2018, 2, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, ) results = sorted( diff --git a/tests/app/service/test_statistics_rest.py b/tests/app/service/test_statistics_rest.py index 2163f8f36..9769a678e 100644 --- a/tests/app/service/test_statistics_rest.py +++ b/tests/app/service/test_statistics_rest.py @@ -234,17 +234,36 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv sms_t2 = create_template(sample_service) email_template = create_template(sample_service, template_type=TemplateType.EMAIL) - create_ft_notification_status(datetime(2016, 6, 1), template=sms_t1) - create_ft_notification_status(datetime(2016, 6, 2), template=sms_t1) - - create_ft_notification_status(datetime(2016, 7, 1), template=sms_t1) - create_ft_notification_status(datetime(2016, 7, 1), template=sms_t2) - create_ft_notification_status( - datetime(2016, 7, 1), - template=sms_t1, - notification_status=NotificationStatus.CREATED, + create_notification( + sms_t1, + created_at=datetime(2016, 6, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t1, + created_at=datetime(2016, 6, 2, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t1, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t2, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, + ) + create_notification( + sms_t1, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.CREATED, + ) + create_notification( + email_template, + created_at=datetime(2016, 7, 1, 1, 1, 0), + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status(datetime(2016, 7, 1), template=email_template) response = admin_request.get( "service.get_monthly_notification_stats", @@ -275,85 +294,94 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv } -@freeze_time("2016-06-05 12:00:00") -def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( - admin_request, sample_template -): - create_ft_notification_status( - datetime(2016, 5, 1, 12), - template=sample_template, - count=1, - ) - create_ft_notification_status( - datetime(2016, 6, 1, 12), - template=sample_template, - notification_status=NotificationStatus.CREATED, - count=2, - ) # noqa +# Test removed because new endpoint uses the view which combines this data +# @freeze_time("2016-06-05 12:00:00") +# def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( +# admin_request, sample_template +# ): +# create_ft_notification_status( +# datetime(2016, 5, 1, 12), +# template=sample_template, +# count=1, +# ) +# create_ft_notification_status( +# datetime(2016, 6, 1, 12), +# template=sample_template, +# notification_status=NotificationStatus.CREATED, +# count=2, +# ) # noqa - create_notification( - sample_template, - created_at=datetime(2016, 6, 5, 12), - status=NotificationStatus.CREATED, - ) - create_notification( - sample_template, - created_at=datetime(2016, 6, 5, 12), - status=NotificationStatus.DELIVERED, - ) +# create_notification( +# sample_template, +# created_at=datetime(2016, 6, 5, 12), +# status=NotificationStatus.CREATED, +# ) +# create_notification( +# sample_template, +# created_at=datetime(2016, 6, 5, 12), +# status=NotificationStatus.DELIVERED, +# ) - # this doesn't get returned in the stats because it is old - it should be in ft_notification_status by now - create_notification( - sample_template, - created_at=datetime(2016, 6, 4, 12), - status=NotificationStatus.SENDING, - ) +# # this doesn't get returned in the stats because it is old - it should be in ft_notification_status by now +# create_notification( +# sample_template, +# created_at=datetime(2016, 6, 4, 12), +# status=NotificationStatus.SENDING, +# ) - response = admin_request.get( - "service.get_monthly_notification_stats", - service_id=sample_template.service_id, - year=2016, - ) +# response = admin_request.get( +# "service.get_monthly_notification_stats", +# service_id=sample_template.service_id, +# year=2016, +# ) - assert len(response["data"]) == 6 # January to June - assert response["data"]["2016-05"] == { - NotificationType.SMS: { - NotificationStatus.DELIVERED: 1, - StatisticsType.REQUESTED: 1, - }, - NotificationType.EMAIL: {}, - } - assert response["data"]["2016-06"] == { - NotificationType.SMS: { - # combines the stats from the historic ft_notification_status and the current notifications - NotificationStatus.CREATED: 3, - NotificationStatus.DELIVERED: 1, - StatisticsType.REQUESTED: 4, - }, - NotificationType.EMAIL: {}, - } +# assert len(response["data"]) == 6 # January to June +# assert response["data"]["2016-05"] == { +# NotificationType.SMS: { +# NotificationStatus.DELIVERED: 1, +# StatisticsType.REQUESTED: 1, +# }, +# NotificationType.EMAIL: {}, +# } +# assert response["data"]["2016-06"] == { +# NotificationType.SMS: { +# # combines the stats from the historic ft_notification_status and the current notifications +# NotificationStatus.CREATED: 3, +# NotificationStatus.DELIVERED: 1, +# StatisticsType.REQUESTED: 4, +# }, +# NotificationType.EMAIL: {}, +# } def test_get_monthly_notification_stats_ignores_test_keys( admin_request, sample_service ): - create_ft_notification_status( - datetime(2016, 6, 1), - service=sample_service, + create_template(service=sample_service) + + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 1, 1, 1, 0), key_type=KeyType.NORMAL, - count=1, + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status( - datetime(2016, 6, 1), - service=sample_service, + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 2, 1, 1, 0), + key_type=KeyType.NORMAL, + status=NotificationStatus.DELIVERED, + ) + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 1, 1, 1, 0), key_type=KeyType.TEAM, - count=2, + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status( - datetime(2016, 6, 1), - service=sample_service, + create_notification( + sample_service.templates[0], + created_at=datetime(2016, 6, 1, 1, 1, 0), key_type=KeyType.TEST, - count=4, + status=NotificationStatus.DELIVERED, ) response = admin_request.get( @@ -370,21 +398,21 @@ def test_get_monthly_notification_stats_ignores_test_keys( def test_get_monthly_notification_stats_checks_dates(admin_request, sample_service): t = create_template(sample_service) - # create_ft_notification_status(datetime(2016, 3, 31), template=t, notification_status='created') - create_ft_notification_status( - datetime(2016, 4, 2), - template=t, - notification_status=NotificationStatus.SENDING, + + create_notification( + t, + created_at=datetime(2016, 4, 2), + status=NotificationStatus.SENDING, ) - create_ft_notification_status( - datetime(2017, 3, 31), - template=t, - notification_status=NotificationStatus.DELIVERED, + create_notification( + t, + created_at=datetime(2017, 3, 31), + status=NotificationStatus.DELIVERED, ) - create_ft_notification_status( - datetime(2017, 4, 11), - template=t, - notification_status=NotificationStatus.PERMANENT_FAILURE, + create_notification( + t, + created_at=datetime(2017, 4, 11), + status=NotificationStatus.PERMANENT_FAILURE, ) response = admin_request.get( @@ -411,15 +439,15 @@ def test_get_monthly_notification_stats_only_gets_for_one_service( templates = [create_template(services[0]), create_template(services[1])] - create_ft_notification_status( - datetime(2016, 6, 1), - template=templates[0], - notification_status=NotificationStatus.CREATED, + create_notification( + templates[0], + created_at=datetime(2016, 6, 1), + status=NotificationStatus.CREATED, ) - create_ft_notification_status( - datetime(2016, 6, 1), - template=templates[1], - notification_status=NotificationStatus.DELIVERED, + create_notification( + templates[1], + created_at=datetime(2016, 6, 1), + status=NotificationStatus.DELIVERED, ) response = admin_request.get( From f95d3e0b99c930f9047405f5dc7da2a71d20c40b Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Tue, 2 Jul 2024 15:38:22 -0400 Subject: [PATCH 14/16] Made changes I requested to be done. Signed-off-by: Cliff Hill --- app/dao/date_util.py | 2 +- app/dao/services_dao.py | 29 ++++++++++- app/service/rest.py | 57 ++------------------- tests/app/service/test_statistics_rest.py | 60 ----------------------- 4 files changed, 34 insertions(+), 114 deletions(-) diff --git a/app/dao/date_util.py b/app/dao/date_util.py index cac09dee2..d93986b45 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -92,4 +92,4 @@ def generate_date_range(start_date, end_date=None, days=0): pass current_date += timedelta(days=1) else: - return "A start_date or number of days must be specified" + return "An end_date or number of days must be specified" diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 74f8094a8..a7c41a28e 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,3 +1,4 @@ +from re import I import uuid from datetime import timedelta @@ -8,7 +9,7 @@ from sqlalchemy.sql.expression import and_, asc, case, func from app import db from app.dao.dao_utils import VersionOptions, autocommit, version_class -from app.dao.date_util import get_current_calendar_year +from app.dao.date_util import generate_date_range, get_current_calendar_year from app.dao.organization_dao import dao_get_organization_by_email_address from app.dao.service_sms_sender_dao import insert_service_sms_sender from app.dao.service_user_dao import dao_get_service_user @@ -41,6 +42,7 @@ from app.models import ( User, VerifyCode, ) +from app.service import statistics from app.utils import ( escape_special_characters, get_archived_db_column_value, @@ -689,3 +691,28 @@ def fetch_notification_stats_for_service_by_month_by_user( ) .all() ) + + +def get_specific_days_stats(results, start_date, days=None, end_date=None): + if days is not None and end_date is not None: + raise ValueError("Only set days OR set end_date, not both.") + elif days is not None: + gen_range = generate_date_range(start_date, days=days) + elif end_date is not None: + gen_range = generate_date_range(start_date, end_date) + else: + raise ValueError("Either days or end_date must be set.") + + grouped_results = { + date: [] for date in gen_range + } | { + day.date(): [notification_type, status, day, count] + for notification_type, status, day, count in results + } + + stats = { + day.strftime("%Y-%m-%d"): statistics.format_statistics(rows) + for day, rows in grouped_results.items() + } + + return stats diff --git a/app/service/rest.py b/app/service/rest.py index ea59387d8..516a22c3e 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -78,6 +78,7 @@ from app.dao.services_dao import ( dao_update_service, fetch_notification_stats_for_service_by_month_by_user, get_services_by_partial_name, + get_specific_days_stats, ) from app.dao.templates_dao import dao_get_template_by_id from app.dao.users_dao import get_user_by_id @@ -234,18 +235,7 @@ def get_service_statistics_for_specific_days(service_id, start, days=1): results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) - grouped_results = defaultdict(list) - for row in results: - notification_type, status, day, count = row - grouped_results[day.date()].append(row) - - for date in generate_date_range(start_date, days=days): - if date not in grouped_results: - grouped_results[date] = [] - - stats = {} - for day, rows in grouped_results.items(): - stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) + stats = get_specific_days_stats(results, start_date, days=days) return stats @@ -276,20 +266,7 @@ def get_service_statistics_for_specific_days_by_user( service_id, start_date, end_date, user_id ) - grouped_results = defaultdict(list) - for row in results: - notification_type, status, day, count = row - grouped_results[day.date()].append(row) - - print(grouped_results) - - for date in generate_date_range(start_date, days=days): - if date not in grouped_results: - grouped_results[date] = [] - - stats = {} - for day, rows in grouped_results.items(): - stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) + stats = get_specific_days_stats(results, start_date, days=days) return stats @@ -744,19 +721,7 @@ def get_single_month_notification_stats_by_user(service_id, user_id): service_id, start_date, end_date, user_id ) - grouped_results = defaultdict(list) - for row in results: - notification_type, status, day, count = row - grouped_results[day.date()].append(row) - - for date in generate_date_range(start_date, end_date): - if date not in grouped_results: - grouped_results[date] = [] - - stats = {} - for day, rows in grouped_results.items(): - stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) - + stats = get_specific_days_stats(results, start_date, end_date=end_date) return jsonify(stats) @@ -778,19 +743,7 @@ def get_single_month_notification_stats_for_service(service_id): results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date) - grouped_results = defaultdict(list) - for row in results: - notification_type, status, day, count = row - grouped_results[day.date()].append(row) - - for date in generate_date_range(start_date, end_date): - if date not in grouped_results: - grouped_results[date] = [] - - stats = {} - for day, rows in grouped_results.items(): - stats[day.strftime("%Y-%m-%d")] = statistics.format_statistics(rows) - + stats = get_specific_days_stats(results, start_date, end_date=end_date) return jsonify(stats) diff --git a/tests/app/service/test_statistics_rest.py b/tests/app/service/test_statistics_rest.py index 9769a678e..6d20cacc3 100644 --- a/tests/app/service/test_statistics_rest.py +++ b/tests/app/service/test_statistics_rest.py @@ -294,66 +294,6 @@ def test_get_monthly_notification_stats_returns_stats(admin_request, sample_serv } -# Test removed because new endpoint uses the view which combines this data -# @freeze_time("2016-06-05 12:00:00") -# def test_get_monthly_notification_stats_combines_todays_data_and_historic_stats( -# admin_request, sample_template -# ): -# create_ft_notification_status( -# datetime(2016, 5, 1, 12), -# template=sample_template, -# count=1, -# ) -# create_ft_notification_status( -# datetime(2016, 6, 1, 12), -# template=sample_template, -# notification_status=NotificationStatus.CREATED, -# count=2, -# ) # noqa - -# create_notification( -# sample_template, -# created_at=datetime(2016, 6, 5, 12), -# status=NotificationStatus.CREATED, -# ) -# create_notification( -# sample_template, -# created_at=datetime(2016, 6, 5, 12), -# status=NotificationStatus.DELIVERED, -# ) - -# # this doesn't get returned in the stats because it is old - it should be in ft_notification_status by now -# create_notification( -# sample_template, -# created_at=datetime(2016, 6, 4, 12), -# status=NotificationStatus.SENDING, -# ) - -# response = admin_request.get( -# "service.get_monthly_notification_stats", -# service_id=sample_template.service_id, -# year=2016, -# ) - -# assert len(response["data"]) == 6 # January to June -# assert response["data"]["2016-05"] == { -# NotificationType.SMS: { -# NotificationStatus.DELIVERED: 1, -# StatisticsType.REQUESTED: 1, -# }, -# NotificationType.EMAIL: {}, -# } -# assert response["data"]["2016-06"] == { -# NotificationType.SMS: { -# # combines the stats from the historic ft_notification_status and the current notifications -# NotificationStatus.CREATED: 3, -# NotificationStatus.DELIVERED: 1, -# StatisticsType.REQUESTED: 4, -# }, -# NotificationType.EMAIL: {}, -# } - - def test_get_monthly_notification_stats_ignores_test_keys( admin_request, sample_service ): From 8708d8ddce33976f2d6fd0f9bd8470b3727d252b Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Tue, 2 Jul 2024 15:57:14 -0400 Subject: [PATCH 15/16] Fixin' imports. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 1 - app/service/rest.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index a7c41a28e..1f889c6b3 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,4 +1,3 @@ -from re import I import uuid from datetime import timedelta diff --git a/app/service/rest.py b/app/service/rest.py index 516a22c3e..bef4cc896 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,5 +1,4 @@ import itertools -from collections import defaultdict from datetime import datetime, timedelta from flask import Blueprint, current_app, jsonify, request @@ -19,7 +18,6 @@ from app.dao.api_key_dao import ( ) from app.dao.dao_utils import dao_rollback, transaction from app.dao.date_util import ( - generate_date_range, get_calendar_year, get_month_start_and_end_date_in_utc, ) From c368d3d3f2b204c5036e00aded0784e7779b16d9 Mon Sep 17 00:00:00 2001 From: Cliff Hill Date: Tue, 2 Jul 2024 16:00:58 -0400 Subject: [PATCH 16/16] More import adjustments. Signed-off-by: Cliff Hill --- app/dao/services_dao.py | 4 +--- app/service/rest.py | 5 +---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 1f889c6b3..6fa663341 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -702,9 +702,7 @@ def get_specific_days_stats(results, start_date, days=None, end_date=None): else: raise ValueError("Either days or end_date must be set.") - grouped_results = { - date: [] for date in gen_range - } | { + grouped_results = {date: [] for date in gen_range} | { day.date(): [notification_type, status, day, count] for notification_type, status, day, count in results } diff --git a/app/service/rest.py b/app/service/rest.py index bef4cc896..71faab7a1 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -17,10 +17,7 @@ from app.dao.api_key_dao import ( save_model_api_key, ) from app.dao.dao_utils import dao_rollback, transaction -from app.dao.date_util import ( - get_calendar_year, - get_month_start_and_end_date_in_utc, -) +from app.dao.date_util import get_calendar_year, get_month_start_and_end_date_in_utc from app.dao.fact_notification_status_dao import ( fetch_monthly_template_usage_for_service, fetch_notification_status_for_service_by_month,