from datetime import timedelta from sqlalchemy import Date, case, cast, func, select, union_all from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import aliased from sqlalchemy.sql.expression import extract, literal from sqlalchemy.types import DateTime, Integer, Text from app import db from app.dao.dao_utils import autocommit from app.enums import KeyType, NotificationStatus, NotificationType from app.models import ( FactNotificationStatus, Notification, NotificationAllTimeView, Service, Template, TemplateFolder, User, template_folder_map, ) from app.utils import ( get_midnight_in_utc, get_month_from_utc_column, midnight_n_days_ago, utc_now, ) @autocommit def update_fact_notification_status(process_day, notification_type, service_id): start_date = get_midnight_in_utc(process_day) end_date = get_midnight_in_utc(process_day + timedelta(days=1)) # delete any existing rows in case some no longer exist e.g. if all messages are sent FactNotificationStatus.query.filter( FactNotificationStatus.local_date == process_day, FactNotificationStatus.notification_type == notification_type, FactNotificationStatus.service_id == service_id, ).delete() query = ( db.session.query( literal(process_day).label("process_day"), NotificationAllTimeView.template_id, literal(service_id).label("service_id"), func.coalesce( NotificationAllTimeView.job_id, "00000000-0000-0000-0000-000000000000" ).label("job_id"), literal(notification_type).label("notification_type"), NotificationAllTimeView.key_type, NotificationAllTimeView.status, func.count().label("notification_count"), ) .filter( NotificationAllTimeView.created_at >= start_date, NotificationAllTimeView.created_at < end_date, NotificationAllTimeView.notification_type == notification_type, NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.key_type.in_((KeyType.NORMAL, KeyType.TEAM)), ) .group_by( NotificationAllTimeView.template_id, NotificationAllTimeView.template_id, "job_id", NotificationAllTimeView.key_type, NotificationAllTimeView.status, ) ) db.session.connection().execute( insert(FactNotificationStatus.__table__).from_select( [ FactNotificationStatus.local_date, FactNotificationStatus.template_id, FactNotificationStatus.service_id, FactNotificationStatus.job_id, FactNotificationStatus.notification_type, FactNotificationStatus.key_type, FactNotificationStatus.notification_status, FactNotificationStatus.notification_count, ], query, ) ) def fetch_notification_status_for_service_by_month(start_date, end_date, service_id): return ( db.session.query( func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), NotificationAllTimeView.notification_type, NotificationAllTimeView.status.label("notification_status"), func.count(NotificationAllTimeView.id).label("count"), ) .filter( NotificationAllTimeView.service_id == service_id, NotificationAllTimeView.created_at >= start_date, NotificationAllTimeView.created_at < end_date, NotificationAllTimeView.key_type != KeyType.TEST, ) .group_by( func.date_trunc("month", NotificationAllTimeView.created_at).label("month"), NotificationAllTimeView.notification_type, NotificationAllTimeView.status, ) .all() ) def fetch_notification_status_for_service_for_day(fetch_day, service_id): return ( db.session.query( # return current month as a datetime so the data has the same shape as the ft_notification_status query literal(fetch_day.replace(day=1), type_=DateTime).label("month"), Notification.notification_type, Notification.status.label("notification_status"), func.count().label("count"), ) .filter( Notification.created_at >= get_midnight_in_utc(fetch_day), Notification.created_at < get_midnight_in_utc(fetch_day + timedelta(days=1)), Notification.service_id == service_id, Notification.key_type != KeyType.TEST, ) .group_by(Notification.notification_type, Notification.status) .all() ) def fetch_notification_status_for_service_for_today_and_7_previous_days( service_id: str, by_template: bool = False, limit_days: int = 7 ) -> list[dict | None]: start_date = midnight_n_days_ago(limit_days) now = get_midnight_in_utc(utc_now()) # Query for the last 7 days stats_for_7_days = select( cast(FactNotificationStatus.notification_type, Text).label("notification_type"), cast(FactNotificationStatus.notification_status, Text).label("status"), *( [ FactNotificationStatus.template_id.label("template_id"), FactNotificationStatus.local_date.label("date_used"), ] if by_template else [] ), FactNotificationStatus.notification_count.label("count"), ).where( FactNotificationStatus.service_id == service_id, FactNotificationStatus.local_date >= start_date, FactNotificationStatus.key_type != KeyType.TEST, ) # Query for today's stats stats_for_today = ( select( cast(Notification.notification_type, Text), cast(Notification.status, Text), *( [ Notification.template_id, literal(now).label("date_used"), ] if by_template else [] ), func.count().label("count"), ) .where( Notification.created_at >= now, Notification.service_id == service_id, Notification.key_type != KeyType.TEST, ) .group_by( Notification.notification_type, *([Notification.template_id] if by_template else []), Notification.status, ) ) # Combine the queries using union_all all_stats_union = union_all(stats_for_7_days, stats_for_today).subquery() all_stats_alias = aliased(all_stats_union, name="all_stats") # Final query with optional template joins query = select( *( [ TemplateFolder.name.label("folder"), Template.name.label("template_name"), False, # TODO: Handle `is_precompiled_letter` template_folder_map.c.template_folder_id, all_stats_alias.c.template_id, User.name.label("created_by"), Template.created_by_id, func.max(all_stats_alias.c.date_used).label( "last_used" ), # Get the most recent date ] if by_template else [] ), all_stats_alias.c.notification_type, all_stats_alias.c.status, cast(func.sum(all_stats_alias.c.count), Integer).label("count"), ) if by_template: query = ( query.join(Template, all_stats_alias.c.template_id == Template.id) .join(User, Template.created_by_id == User.id) .outerjoin( template_folder_map, Template.id == template_folder_map.c.template_id ) .outerjoin( TemplateFolder, TemplateFolder.id == template_folder_map.c.template_folder_id, ) ) # Group by all necessary fields except date_used query = query.group_by( *( [ TemplateFolder.name, Template.name, all_stats_alias.c.template_id, User.name, template_folder_map.c.template_folder_id, Template.created_by_id, ] if by_template else [] ), all_stats_alias.c.notification_type, all_stats_alias.c.status, ) # Execute the query using Flask-SQLAlchemy's session result = db.session.execute(query) return result.mappings().all() def fetch_notification_status_totals_for_all_services(start_date, end_date): stats = ( db.session.query( FactNotificationStatus.notification_type.cast(db.Text).label( "notification_type" ), FactNotificationStatus.notification_status.cast(db.Text).label("status"), FactNotificationStatus.key_type.cast(db.Text).label("key_type"), func.sum(FactNotificationStatus.notification_count).label("count"), ) .filter( FactNotificationStatus.local_date >= start_date, FactNotificationStatus.local_date <= end_date, ) .group_by( FactNotificationStatus.notification_type, FactNotificationStatus.notification_status, FactNotificationStatus.key_type, ) ) today = get_midnight_in_utc(utc_now()) if start_date <= utc_now().date() <= end_date: stats_for_today = ( db.session.query( Notification.notification_type.cast(db.Text).label("notification_type"), Notification.status.cast(db.Text), Notification.key_type.cast(db.Text), func.count().label("count"), ) .filter(Notification.created_at >= today) .group_by( Notification.notification_type, Notification.status, Notification.key_type, ) ) all_stats_table = stats.union_all(stats_for_today).subquery() query = ( db.session.query( all_stats_table.c.notification_type, all_stats_table.c.status, all_stats_table.c.key_type, func.cast(func.sum(all_stats_table.c.count), Integer).label("count"), ) .group_by( all_stats_table.c.notification_type, all_stats_table.c.status, all_stats_table.c.key_type, ) .order_by(all_stats_table.c.notification_type) ) else: query = stats.order_by(FactNotificationStatus.notification_type) return query.all() def fetch_notification_statuses_for_job(job_id): return ( db.session.query( FactNotificationStatus.notification_status.label("status"), func.sum(FactNotificationStatus.notification_count).label("count"), ) .filter( FactNotificationStatus.job_id == job_id, ) .group_by(FactNotificationStatus.notification_status) .all() ) def fetch_stats_for_all_services_by_date_range( start_date, end_date, include_from_test_key=True ): stats = ( db.session.query( FactNotificationStatus.service_id.label("service_id"), Service.name.label("name"), Service.restricted.label("restricted"), Service.active.label("active"), Service.created_at.label("created_at"), FactNotificationStatus.notification_type.cast(db.Text).label( "notification_type" ), FactNotificationStatus.notification_status.cast(db.Text).label("status"), func.sum(FactNotificationStatus.notification_count).label("count"), ) .filter( FactNotificationStatus.local_date >= start_date, FactNotificationStatus.local_date <= end_date, FactNotificationStatus.service_id == Service.id, ) .group_by( FactNotificationStatus.service_id.label("service_id"), Service.name, Service.restricted, Service.active, Service.created_at, FactNotificationStatus.notification_type, FactNotificationStatus.notification_status, ) .order_by( FactNotificationStatus.service_id, FactNotificationStatus.notification_type ) ) if not include_from_test_key: stats = stats.filter(FactNotificationStatus.key_type != KeyType.TEST) if start_date <= utc_now().date() <= end_date: today = get_midnight_in_utc(utc_now()) subquery = ( db.session.query( Notification.notification_type.label("notification_type"), Notification.status.label("status"), Notification.service_id.label("service_id"), func.count(Notification.id).label("count"), ) .filter(Notification.created_at >= today) .group_by( Notification.notification_type, Notification.status, Notification.service_id, ) ) if not include_from_test_key: subquery = subquery.filter(Notification.key_type != KeyType.TEST) subquery = subquery.subquery() stats_for_today = db.session.query( Service.id.label("service_id"), Service.name.label("name"), Service.restricted.label("restricted"), Service.active.label("active"), Service.created_at.label("created_at"), subquery.c.notification_type.cast(db.Text).label("notification_type"), subquery.c.status.cast(db.Text).label("status"), subquery.c.count.label("count"), ).outerjoin(subquery, subquery.c.service_id == Service.id) all_stats_table = stats.union_all(stats_for_today).subquery() query = ( db.session.query( all_stats_table.c.service_id, all_stats_table.c.name, all_stats_table.c.restricted, all_stats_table.c.active, all_stats_table.c.created_at, all_stats_table.c.notification_type, all_stats_table.c.status, func.cast(func.sum(all_stats_table.c.count), Integer).label("count"), ) .group_by( all_stats_table.c.service_id, all_stats_table.c.name, all_stats_table.c.restricted, all_stats_table.c.active, all_stats_table.c.created_at, all_stats_table.c.notification_type, all_stats_table.c.status, ) .order_by( all_stats_table.c.name, all_stats_table.c.notification_type, all_stats_table.c.status, ) ) else: query = stats return query.all() def fetch_monthly_template_usage_for_service(start_date, end_date, service_id): # services_dao.replaces dao_fetch_monthly_historical_usage_by_template_for_service stats = ( db.session.query( FactNotificationStatus.template_id.label("template_id"), Template.name.label("name"), Template.template_type.label("template_type"), extract("month", FactNotificationStatus.local_date).label("month"), extract("year", FactNotificationStatus.local_date).label("year"), func.sum(FactNotificationStatus.notification_count).label("count"), ) .join(Template, FactNotificationStatus.template_id == Template.id) .filter( FactNotificationStatus.service_id == service_id, FactNotificationStatus.local_date >= start_date, FactNotificationStatus.local_date <= end_date, FactNotificationStatus.key_type != KeyType.TEST, FactNotificationStatus.notification_status != NotificationStatus.CANCELLED, ) .group_by( FactNotificationStatus.template_id, Template.name, Template.template_type, extract("month", FactNotificationStatus.local_date).label("month"), extract("year", FactNotificationStatus.local_date).label("year"), ) .order_by( extract("year", FactNotificationStatus.local_date), extract("month", FactNotificationStatus.local_date), Template.name, ) ) if start_date <= utc_now() <= end_date: today = get_midnight_in_utc(utc_now()) month = get_month_from_utc_column(Notification.created_at) stats_for_today = ( db.session.query( Notification.template_id.label("template_id"), Template.name.label("name"), Template.template_type.label("template_type"), extract("month", month).label("month"), extract("year", month).label("year"), func.count().label("count"), ) .join( Template, Notification.template_id == Template.id, ) .filter( Notification.created_at >= today, Notification.service_id == service_id, Notification.key_type != KeyType.TEST, Notification.status != NotificationStatus.CANCELLED, ) .group_by( Notification.template_id, Template.hidden, Template.name, Template.template_type, month, ) ) all_stats_table = stats.union_all(stats_for_today).subquery() query = ( db.session.query( all_stats_table.c.template_id, all_stats_table.c.name, all_stats_table.c.template_type, func.cast(all_stats_table.c.month, Integer).label("month"), func.cast(all_stats_table.c.year, Integer).label("year"), func.cast(func.sum(all_stats_table.c.count), Integer).label("count"), ) .group_by( all_stats_table.c.template_id, all_stats_table.c.name, all_stats_table.c.template_type, all_stats_table.c.month, all_stats_table.c.year, ) .order_by( all_stats_table.c.year, all_stats_table.c.month, all_stats_table.c.name ) ) else: query = stats return query.all() def get_total_notifications_for_date_range(start_date, end_date): query = ( db.session.query( FactNotificationStatus.local_date.label("local_date"), func.sum( case( ( FactNotificationStatus.notification_type == NotificationType.EMAIL, FactNotificationStatus.notification_count, ), else_=0, ) ).label("emails"), func.sum( case( ( FactNotificationStatus.notification_type == NotificationType.SMS, FactNotificationStatus.notification_count, ), else_=0, ) ).label("sms"), ) .filter( FactNotificationStatus.key_type != KeyType.TEST, ) .group_by(FactNotificationStatus.local_date) .order_by(FactNotificationStatus.local_date) ) if start_date and end_date: query = query.filter( FactNotificationStatus.local_date >= start_date, FactNotificationStatus.local_date <= end_date, ) return query.all() def fetch_monthly_notification_statuses_per_service(start_date, end_date): return ( db.session.query( func.date_trunc("month", FactNotificationStatus.local_date) .cast(Date) .label("date_created"), Service.id.label("service_id"), Service.name.label("service_name"), FactNotificationStatus.notification_type, func.sum( case( ( FactNotificationStatus.notification_status.in_( [NotificationStatus.SENDING, NotificationStatus.PENDING] ), FactNotificationStatus.notification_count, ), else_=0, ) ).label("count_sending"), func.sum( case( ( FactNotificationStatus.notification_status == NotificationStatus.DELIVERED, FactNotificationStatus.notification_count, ), else_=0, ) ).label("count_delivered"), func.sum( case( ( FactNotificationStatus.notification_status.in_( [ NotificationStatus.TECHNICAL_FAILURE, NotificationStatus.FAILED, ] ), FactNotificationStatus.notification_count, ), else_=0, ) ).label("count_technical_failure"), func.sum( case( ( FactNotificationStatus.notification_status == NotificationStatus.TEMPORARY_FAILURE, FactNotificationStatus.notification_count, ), else_=0, ) ).label("count_temporary_failure"), func.sum( case( ( FactNotificationStatus.notification_status == NotificationStatus.PERMANENT_FAILURE, FactNotificationStatus.notification_count, ), else_=0, ) ).label("count_permanent_failure"), func.sum( case( ( FactNotificationStatus.notification_status == NotificationStatus.SENT, FactNotificationStatus.notification_count, ), else_=0, ) ).label("count_sent"), ) .join(Service, FactNotificationStatus.service_id == Service.id) .filter( FactNotificationStatus.notification_status != NotificationStatus.CREATED, Service.active.is_(True), FactNotificationStatus.key_type != KeyType.TEST, Service.restricted.is_(False), FactNotificationStatus.local_date >= start_date, FactNotificationStatus.local_date <= end_date, ) .group_by( Service.id, Service.name, func.date_trunc("month", FactNotificationStatus.local_date).cast(Date), FactNotificationStatus.notification_type, ) .order_by( func.date_trunc("month", FactNotificationStatus.local_date).cast(Date), Service.id, FactNotificationStatus.notification_type, ) .all() )