merge from main

This commit is contained in:
Kenneth Kehl
2025-01-29 12:15:29 -08:00
8 changed files with 236 additions and 31 deletions
+80 -8
View File
@@ -469,6 +469,35 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date):
start_date = get_midnight_in_utc(start_date)
end_date = get_midnight_in_utc(end_date + timedelta(days=1))
total_substmt = (
select(
func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
Job.notification_count.label("notification_count"),
)
.join(Job, NotificationAllTimeView.job_id == Job.id)
.where(
NotificationAllTimeView.service_id == service_id,
NotificationAllTimeView.key_type != KeyType.TEST,
NotificationAllTimeView.created_at >= start_date,
NotificationAllTimeView.created_at < end_date,
)
.group_by(
Job.id,
Job.notification_count,
func.date_trunc("day", NotificationAllTimeView.created_at),
)
.subquery()
)
total_stmt = select(
total_substmt.c.day,
func.sum(total_substmt.c.notification_count).label("total_notifications"),
).group_by(total_substmt.c.day)
total_notifications = {
row.day: row.total_notifications for row in db.session.execute(total_stmt).all()
}
stmt = (
select(
NotificationAllTimeView.notification_type,
@@ -488,7 +517,10 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date):
func.date_trunc("day", NotificationAllTimeView.created_at),
)
)
return db.session.execute(stmt).all()
data = db.session.execute(stmt).all()
return total_notifications, data
def dao_fetch_stats_for_service_from_days_for_user(
@@ -497,6 +529,36 @@ def dao_fetch_stats_for_service_from_days_for_user(
start_date = get_midnight_in_utc(start_date)
end_date = get_midnight_in_utc(end_date + timedelta(days=1))
total_substmt = (
select(
func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
Job.notification_count.label("notification_count"),
)
.join(Job, NotificationAllTimeView.job_id == Job.id)
.where(
NotificationAllTimeView.service_id == service_id,
NotificationAllTimeView.key_type != KeyType.TEST,
NotificationAllTimeView.created_at >= start_date,
NotificationAllTimeView.created_at < end_date,
NotificationAllTimeView.created_by_id == user_id,
)
.group_by(
Job.id,
Job.notification_count,
func.date_trunc("day", NotificationAllTimeView.created_at),
)
.subquery()
)
total_stmt = select(
total_substmt.c.day,
func.sum(total_substmt.c.notification_count).label("total_notifications"),
).group_by(total_substmt.c.day)
total_notifications = {
row.day: row.total_notifications for row in db.session.execute(total_stmt).all()
}
stmt = (
select(
NotificationAllTimeView.notification_type,
@@ -504,7 +566,6 @@ def dao_fetch_stats_for_service_from_days_for_user(
func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
func.count(NotificationAllTimeView.id).label("count"),
)
.select_from(NotificationAllTimeView)
.where(
NotificationAllTimeView.service_id == service_id,
NotificationAllTimeView.key_type != KeyType.TEST,
@@ -518,7 +579,10 @@ def dao_fetch_stats_for_service_from_days_for_user(
func.date_trunc("day", NotificationAllTimeView.created_at),
)
)
return db.session.execute(stmt).scalars().all()
data = db.session.execute(stmt).all()
return total_notifications, data
def dao_fetch_todays_stats_for_all_services(
@@ -734,7 +798,9 @@ def fetch_notification_stats_for_service_by_month_by_user(
return db.session.execute(stmt).all()
def get_specific_days_stats(data, start_date, days=None, end_date=None):
def get_specific_days_stats(
data, start_date, days=None, end_date=None, total_notifications=None
):
if days is not None and end_date is not None:
raise ValueError("Only set days OR set end_date, not both.")
elif days is not None:
@@ -745,13 +811,19 @@ def get_specific_days_stats(data, start_date, days=None, end_date=None):
raise ValueError("Either days or end_date must be set.")
grouped_data = {date: [] for date in gen_range} | {
day: [row for row in data if row.day.date() == day]
for day in {item.day.date() for item in data}
day: [row for row in data if row.day == day]
for day in {item.day for item in data}
}
stats = {
day.strftime("%Y-%m-%d"): statistics.format_statistics(rows)
day.strftime("%Y-%m-%d"): statistics.format_statistics(
rows,
total_notifications=(
total_notifications.get(day, 0)
if total_notifications is not None
else None
),
)
for day, rows in grouped_data.items()
}
return stats
+1
View File
@@ -211,3 +211,4 @@ class StatisticsType(StrEnum):
REQUESTED = "requested"
DELIVERED = "delivered"
FAILURE = "failure"
PENDING = "pending"
+23 -8
View File
@@ -233,9 +233,18 @@ def get_service_statistics_for_specific_days(service_id, start, days=1):
end_date = datetime.strptime(start, "%Y-%m-%d")
start_date = end_date - timedelta(days=days - 1)
results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date)
total_notifications, results = dao_fetch_stats_for_service_from_days(
service_id,
start_date,
end_date,
)
stats = get_specific_days_stats(results, start_date, days=days)
stats = get_specific_days_stats(
results,
start_date,
days=days,
total_notifications=total_notifications,
)
return stats
@@ -262,12 +271,16 @@ def get_service_statistics_for_specific_days_by_user(
end_date = datetime.strptime(start, "%Y-%m-%d")
start_date = end_date - timedelta(days=days - 1)
results = dao_fetch_stats_for_service_from_days_for_user(
total_notifications, results = dao_fetch_stats_for_service_from_days_for_user(
service_id, start_date, end_date, user_id
)
stats = get_specific_days_stats(results, start_date, days=days)
stats = get_specific_days_stats(
results,
start_date,
days=days,
total_notifications=total_notifications,
)
return stats
@@ -677,11 +690,11 @@ def get_single_month_notification_stats_by_user(service_id, user_id):
month_year = datetime(year, month, 10, 00, 00, 00)
start_date, end_date = get_month_start_and_end_date_in_utc(month_year)
results = dao_fetch_stats_for_service_from_days_for_user(
total_notifications, results = dao_fetch_stats_for_service_from_days_for_user(
service_id, start_date, end_date, user_id
)
stats = get_specific_days_stats(results, start_date, end_date=end_date)
stats = get_specific_days_stats(results, start_date, end_date=end_date, total_notifications=total_notifications,)
return jsonify(stats)
@@ -701,7 +714,9 @@ def get_single_month_notification_stats_for_service(service_id):
month_year = datetime(year, month, 10, 00, 00, 00)
start_date, end_date = get_month_start_and_end_date_in_utc(month_year)
results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date)
__, results = dao_fetch_stats_for_service_from_days(
service_id, start_date, end_date
)
stats = get_specific_days_stats(results, start_date, end_date=end_date)
return jsonify(stats)
+25 -3
View File
@@ -2,10 +2,16 @@ from collections import defaultdict
from datetime import datetime
from app.dao.date_util import get_months_for_financial_year
from app.enums import KeyType, NotificationStatus, StatisticsType, TemplateType
from app.enums import (
KeyType,
NotificationStatus,
NotificationType,
StatisticsType,
TemplateType,
)
def format_statistics(statistics):
def format_statistics(statistics, total_notifications=None):
# statistics come in a named tuple with uniqueness from 'notification_type', 'status' - however missing
# statuses/notification types won't be represented and the status types need to be simplified/summed up
# so we can return emails/sms * created, sent, and failed
@@ -14,11 +20,27 @@ def format_statistics(statistics):
# any row could be null, if the service either has no notifications in the notifications table,
# or no historical data in the ft_notification_status table.
if row.notification_type:
_update_statuses_from_row(counts[row.notification_type], row)
_update_statuses_from_row(
counts[row.notification_type],
row,
)
if NotificationType.SMS in counts and total_notifications is not None:
sms_dict = counts[NotificationType.SMS]
delivered_count = sms_dict[StatisticsType.DELIVERED]
failed_count = sms_dict[StatisticsType.FAILURE]
sms_dict[StatisticsType.PENDING] = calculate_pending_stats(
delivered_count, failed_count, total_notifications
)
return counts
def calculate_pending_stats(delivered_count, failed_count, total_notifications):
pending_count = total_notifications - (delivered_count + failed_count)
return max(0, pending_count)
def format_admin_stats(statistics):
counts = create_stats_dict()
+37 -1
View File
@@ -1652,11 +1652,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 2,
StatisticsType.PENDING: 2,
},
},
(_this_date.date() + timedelta(days=1)).strftime("%Y-%m-%d"): {
@@ -1664,11 +1666,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 1,
StatisticsType.PENDING: 0,
},
},
(_this_date.date() + timedelta(days=2)).strftime("%Y-%m-%d"): {
@@ -1676,11 +1680,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 1,
StatisticsType.PENDING: 0,
},
},
(_this_date.date() + timedelta(days=3)).strftime("%Y-%m-%d"): {
@@ -1688,11 +1694,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
},
(_this_date.date() + timedelta(days=4)).strftime("%Y-%m-%d"): {
@@ -1700,11 +1708,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 1,
StatisticsType.PENDING: 0,
},
},
},
@@ -1727,11 +1737,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 2,
StatisticsType.PENDING: 2,
},
},
(_this_date.date() + timedelta(days=1)).strftime("%Y-%m-%d"): {
@@ -1739,11 +1751,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 1,
StatisticsType.PENDING: 0,
},
},
(_this_date.date() + timedelta(days=2)).strftime("%Y-%m-%d"): {
@@ -1751,11 +1765,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 1,
StatisticsType.PENDING: 0,
},
},
(_this_date.date() + timedelta(days=3)).strftime("%Y-%m-%d"): {
@@ -1763,11 +1779,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
},
(_this_date.date() + timedelta(days=4)).strftime("%Y-%m-%d"): {
@@ -1775,11 +1793,13 @@ _this_date = utc_now() - timedelta(days=4)
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 0,
StatisticsType.PENDING: 0,
},
TemplateType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.REQUESTED: 1,
StatisticsType.PENDING: 0,
},
},
},
@@ -1800,5 +1820,21 @@ def test_get_specific_days(data, start_date, days, end_date, expected, is_error)
new_line.count = 1
new_line.something = line["something"]
new_data.append(new_line)
results = get_specific_days_stats(new_data, start_date, days, end_date)
total_notifications = None
date_key = _this_date.date().strftime("%Y-%m-%d")
if expected and date_key in expected:
sms_stats = expected[date_key].get(TemplateType.SMS, {})
requested = sms_stats.get(StatisticsType.REQUESTED, 0)
if requested > 0:
total_notifications = {_this_date: requested}
results = get_specific_days_stats(
new_data,
start_date,
days,
end_date,
total_notifications=total_notifications,
)
assert results == expected
+18
View File
@@ -2222,6 +2222,7 @@ def test_set_sms_prefixing_for_service_cant_be_none(
StatisticsType.REQUESTED: 2,
StatisticsType.DELIVERED: 1,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
},
),
(
@@ -2230,6 +2231,7 @@ def test_set_sms_prefixing_for_service_cant_be_none(
StatisticsType.REQUESTED: 1,
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
},
),
],
@@ -2278,11 +2280,13 @@ def test_get_services_with_detailed_flag(client, sample_template):
NotificationType.EMAIL: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 0,
},
NotificationType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 3,
},
}
@@ -2309,11 +2313,13 @@ def test_get_services_with_detailed_flag_excluding_from_test_key(
NotificationType.EMAIL: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 0,
},
NotificationType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 2,
},
}
@@ -2385,11 +2391,13 @@ def test_get_detailed_services_groups_by_service(notify_db_session):
NotificationType.EMAIL: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 0,
},
NotificationType.SMS: {
StatisticsType.DELIVERED: 1,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 3,
},
}
@@ -2398,11 +2406,13 @@ def test_get_detailed_services_groups_by_service(notify_db_session):
NotificationType.EMAIL: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 0,
},
NotificationType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 1,
},
}
@@ -2428,11 +2438,13 @@ def test_get_detailed_services_includes_services_with_no_notifications(
NotificationType.EMAIL: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 0,
},
NotificationType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 1,
},
}
@@ -2441,11 +2453,13 @@ def test_get_detailed_services_includes_services_with_no_notifications(
NotificationType.EMAIL: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 0,
},
NotificationType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 0,
},
}
@@ -2470,11 +2484,13 @@ def test_get_detailed_services_only_includes_todays_notifications(sample_templat
NotificationType.EMAIL: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 0,
},
NotificationType.SMS: {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 3,
},
}
@@ -2523,11 +2539,13 @@ def test_get_detailed_services_for_date_range(
assert data[0]["statistics"][NotificationType.EMAIL] == {
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 0,
}
assert data[0]["statistics"][NotificationType.SMS] == {
StatisticsType.DELIVERED: 2,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
StatisticsType.REQUESTED: 2,
}
+48 -11
View File
@@ -9,6 +9,7 @@ from freezegun import freeze_time
from app.enums import KeyType, NotificationStatus, NotificationType, StatisticsType
from app.service.statistics import (
add_monthly_notification_status_stats,
calculate_pending_stats,
create_empty_monthly_notification_status_stats_dict,
create_stats_dict,
create_zeroed_stats_dicts,
@@ -27,22 +28,22 @@ NewStatsRow = collections.namedtuple(
@pytest.mark.idparametrize(
"stats, email_counts, sms_counts",
{
"empty": ([], [0, 0, 0], [0, 0, 0]),
"empty": ([], [0, 0, 0, 0], [0, 0, 0, 0]),
"always_increment_requested": (
[
StatsRow(NotificationType.EMAIL, NotificationStatus.DELIVERED, 1),
StatsRow(NotificationType.EMAIL, NotificationStatus.FAILED, 1),
],
[2, 1, 1],
[0, 0, 0],
[2, 1, 1, 0],
[0, 0, 0, 0],
),
"dont_mix_template_types": (
[
StatsRow(NotificationType.EMAIL, NotificationStatus.DELIVERED, 1),
StatsRow(NotificationType.SMS, NotificationStatus.DELIVERED, 1),
],
[1, 1, 0],
[1, 1, 0],
[1, 1, 0, 0],
[1, 1, 0, 0],
),
"convert_fail_statuses_to_failed": (
[
@@ -57,8 +58,8 @@ NewStatsRow = collections.namedtuple(
NotificationType.EMAIL, NotificationStatus.PERMANENT_FAILURE, 1
),
],
[4, 0, 4],
[0, 0, 0],
[4, 0, 4, 0],
[0, 0, 0, 0],
),
"convert_sent_to_delivered": (
[
@@ -66,16 +67,16 @@ NewStatsRow = collections.namedtuple(
StatsRow(NotificationType.SMS, NotificationStatus.DELIVERED, 1),
StatsRow(NotificationType.SMS, NotificationStatus.SENT, 1),
],
[0, 0, 0],
[3, 2, 0],
[0, 0, 0, 0],
[3, 2, 0, 0],
),
"handles_none_rows": (
[
StatsRow(NotificationType.SMS, NotificationStatus.SENDING, 1),
StatsRow(None, None, None),
],
[0, 0, 0],
[1, 0, 0],
[0, 0, 0, 0],
[1, 0, 0, 0],
),
},
)
@@ -89,6 +90,7 @@ def test_format_statistics(stats, email_counts, sms_counts):
StatisticsType.REQUESTED,
StatisticsType.DELIVERED,
StatisticsType.FAILURE,
StatisticsType.PENDING,
],
email_counts,
)
@@ -101,23 +103,58 @@ def test_format_statistics(stats, email_counts, sms_counts):
StatisticsType.REQUESTED,
StatisticsType.DELIVERED,
StatisticsType.FAILURE,
StatisticsType.PENDING,
],
sms_counts,
)
}
def test_format_statistics_with_pending():
stats = [
StatsRow(NotificationType.SMS, NotificationStatus.DELIVERED, 10),
StatsRow(NotificationType.SMS, NotificationStatus.FAILED, 2),
]
total_notifications_for_sms = 20
result = format_statistics(stats, total_notifications=total_notifications_for_sms)
expected_sms_counts = {
StatisticsType.REQUESTED: 12,
StatisticsType.DELIVERED: 10,
StatisticsType.FAILURE: 2,
StatisticsType.PENDING: 8,
}
assert result[NotificationType.SMS] == expected_sms_counts
@pytest.mark.parametrize(
"delivered, failed, total, expected",
[
(10, 2, 20, 8),
(10, 10, 20, 0),
(15, 10, 20, 0),
],
)
def test_calculate_pending(delivered, failed, total, expected):
assert calculate_pending_stats(delivered, failed, total) == expected
def test_create_zeroed_stats_dicts():
assert create_zeroed_stats_dicts() == {
NotificationType.SMS: {
StatisticsType.REQUESTED: 0,
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
},
NotificationType.EMAIL: {
StatisticsType.REQUESTED: 0,
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
},
}
@@ -119,6 +119,7 @@ def test_get_template_usage_by_month_returns_two_templates(
StatisticsType.REQUESTED: 2,
StatisticsType.DELIVERED: 1,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
},
),
(
@@ -127,6 +128,7 @@ def test_get_template_usage_by_month_returns_two_templates(
StatisticsType.REQUESTED: 1,
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
},
),
],
@@ -163,11 +165,13 @@ def test_get_service_notification_statistics_with_unknown_service(admin_request)
StatisticsType.REQUESTED: 0,
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
},
NotificationType.EMAIL: {
StatisticsType.REQUESTED: 0,
StatisticsType.DELIVERED: 0,
StatisticsType.FAILURE: 0,
StatisticsType.PENDING: 0,
},
}