mirror of
https://github.com/GSA/notifications-api.git
synced 2025-12-20 15:31:15 -05:00
913cf5e12d
previously we checked notifications table, and if the results were zero, checked the notification history table to see if there's data in there. When we know that data isn't in notifications, we're still checking. These queries take half a second per service, and we're doing at least ten for each of the five thousand services we have in notify. Most of these services have no data in either table for any given day, and we can reduce the amount of queries we do by only checking one table. Check the data retention for a service, and then if the date is older than the retention, get from history table. NOTE: This requires that the delete tasks haven't run yet for the day! If your retention is three days, this will look in the Notification table for data from three days ago - expecting that shortly after the task finishes, we'll delete that data.
513 lines
20 KiB
Python
513 lines
20 KiB
Python
from datetime import datetime, timedelta, time
|
|
|
|
from flask import current_app
|
|
from notifications_utils.timezones import convert_bst_to_utc
|
|
from sqlalchemy import case, func, Date
|
|
from sqlalchemy.dialects.postgresql import insert
|
|
from sqlalchemy.sql.expression import literal, extract
|
|
from sqlalchemy.types import DateTime, Integer
|
|
|
|
from app import db
|
|
from app.models import (
|
|
EMAIL_TYPE,
|
|
FactNotificationStatus,
|
|
KEY_TYPE_TEST,
|
|
LETTER_TYPE,
|
|
Notification,
|
|
NOTIFICATION_CANCELLED,
|
|
NOTIFICATION_CREATED,
|
|
NOTIFICATION_DELIVERED,
|
|
NOTIFICATION_FAILED,
|
|
NOTIFICATION_SENDING,
|
|
NOTIFICATION_SENT,
|
|
NOTIFICATION_TECHNICAL_FAILURE,
|
|
NOTIFICATION_TEMPORARY_FAILURE,
|
|
NOTIFICATION_PERMANENT_FAILURE,
|
|
Service,
|
|
SMS_TYPE,
|
|
Template,
|
|
)
|
|
from app.utils import (
|
|
get_london_midnight_in_utc,
|
|
midnight_n_days_ago,
|
|
get_london_month_from_utc_column,
|
|
get_notification_table_to_use,
|
|
)
|
|
|
|
|
|
def fetch_notification_status_for_day(process_day):
|
|
start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
|
|
end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
|
|
|
|
current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))
|
|
|
|
all_data_for_process_day = []
|
|
services = Service.query.all()
|
|
# for each service
|
|
# for each notification type
|
|
# query notifications for day
|
|
# if no rows try notificationHistory
|
|
for service in services:
|
|
for notification_type in [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE]:
|
|
table = get_notification_table_to_use(service, notification_type, process_day)
|
|
|
|
data_for_service_and_type = query_for_fact_status_data(
|
|
table=table,
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
notification_type=notification_type,
|
|
service_id=service.id
|
|
)
|
|
|
|
all_data_for_process_day += data_for_service_and_type
|
|
|
|
return all_data_for_process_day
|
|
|
|
|
|
def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id):
|
|
query = db.session.query(
|
|
table.template_id,
|
|
table.service_id,
|
|
func.coalesce(table.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
|
|
table.notification_type,
|
|
table.key_type,
|
|
table.status,
|
|
func.count().label('notification_count')
|
|
).filter(
|
|
table.created_at >= start_date,
|
|
table.created_at < end_date,
|
|
table.notification_type == notification_type,
|
|
table.service_id == service_id,
|
|
table.key_type != KEY_TYPE_TEST
|
|
).group_by(
|
|
table.template_id,
|
|
table.service_id,
|
|
'job_id',
|
|
table.notification_type,
|
|
table.key_type,
|
|
table.status
|
|
)
|
|
return query.all()
|
|
|
|
|
|
def update_fact_notification_status(data, process_day):
|
|
table = FactNotificationStatus.__table__
|
|
FactNotificationStatus.query.filter(
|
|
FactNotificationStatus.bst_date == process_day
|
|
).delete()
|
|
|
|
for row in data:
|
|
stmt = insert(table).values(
|
|
bst_date=process_day,
|
|
template_id=row.template_id,
|
|
service_id=row.service_id,
|
|
job_id=row.job_id,
|
|
notification_type=row.notification_type,
|
|
key_type=row.key_type,
|
|
notification_status=row.status,
|
|
notification_count=row.notification_count,
|
|
)
|
|
db.session.connection().execute(stmt)
|
|
db.session.commit()
|
|
|
|
|
|
def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
|
|
return db.session.query(
|
|
func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
|
|
FactNotificationStatus.notification_type,
|
|
FactNotificationStatus.notification_status,
|
|
func.sum(FactNotificationStatus.notification_count).label('count')
|
|
).filter(
|
|
FactNotificationStatus.service_id == service_id,
|
|
FactNotificationStatus.bst_date >= start_date,
|
|
FactNotificationStatus.bst_date < end_date,
|
|
FactNotificationStatus.key_type != KEY_TYPE_TEST
|
|
).group_by(
|
|
func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
|
|
FactNotificationStatus.notification_type,
|
|
FactNotificationStatus.notification_status
|
|
).all()
|
|
|
|
|
|
def fetch_notification_status_for_service_for_day(bst_day, service_id):
|
|
return db.session.query(
|
|
# return current month as a datetime so the data has the same shape as the ft_notification_status query
|
|
literal(bst_day.replace(day=1), type_=DateTime).label('month'),
|
|
Notification.notification_type,
|
|
Notification.status.label('notification_status'),
|
|
func.count().label('count')
|
|
).filter(
|
|
Notification.created_at >= get_london_midnight_in_utc(bst_day),
|
|
Notification.created_at < get_london_midnight_in_utc(bst_day + timedelta(days=1)),
|
|
Notification.service_id == service_id,
|
|
Notification.key_type != KEY_TYPE_TEST
|
|
).group_by(
|
|
Notification.notification_type,
|
|
Notification.status
|
|
).all()
|
|
|
|
|
|
def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7):
|
|
start_date = midnight_n_days_ago(limit_days)
|
|
now = datetime.utcnow()
|
|
stats_for_7_days = db.session.query(
|
|
FactNotificationStatus.notification_type.label('notification_type'),
|
|
FactNotificationStatus.notification_status.label('status'),
|
|
*([FactNotificationStatus.template_id.label('template_id')] if by_template else []),
|
|
FactNotificationStatus.notification_count.label('count')
|
|
).filter(
|
|
FactNotificationStatus.service_id == service_id,
|
|
FactNotificationStatus.bst_date >= start_date,
|
|
FactNotificationStatus.key_type != KEY_TYPE_TEST
|
|
)
|
|
|
|
stats_for_today = db.session.query(
|
|
Notification.notification_type.cast(db.Text),
|
|
Notification.status,
|
|
*([Notification.template_id] if by_template else []),
|
|
func.count().label('count')
|
|
).filter(
|
|
Notification.created_at >= get_london_midnight_in_utc(now),
|
|
Notification.service_id == service_id,
|
|
Notification.key_type != KEY_TYPE_TEST
|
|
).group_by(
|
|
Notification.notification_type,
|
|
*([Notification.template_id] if by_template else []),
|
|
Notification.status
|
|
)
|
|
|
|
all_stats_table = stats_for_7_days.union_all(stats_for_today).subquery()
|
|
|
|
query = db.session.query(
|
|
*([
|
|
Template.name.label("template_name"),
|
|
Template.is_precompiled_letter,
|
|
all_stats_table.c.template_id
|
|
] if by_template else []),
|
|
all_stats_table.c.notification_type,
|
|
all_stats_table.c.status,
|
|
func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
|
|
)
|
|
|
|
if by_template:
|
|
query = query.filter(all_stats_table.c.template_id == Template.id)
|
|
|
|
return query.group_by(
|
|
*([Template.name, Template.is_precompiled_letter, all_stats_table.c.template_id] if by_template else []),
|
|
all_stats_table.c.notification_type,
|
|
all_stats_table.c.status,
|
|
).all()
|
|
|
|
|
|
def fetch_notification_status_totals_for_all_services(start_date, end_date):
|
|
stats = db.session.query(
|
|
FactNotificationStatus.notification_type.label('notification_type'),
|
|
FactNotificationStatus.notification_status.label('status'),
|
|
FactNotificationStatus.key_type.label('key_type'),
|
|
func.sum(FactNotificationStatus.notification_count).label('count')
|
|
).filter(
|
|
FactNotificationStatus.bst_date >= start_date,
|
|
FactNotificationStatus.bst_date <= end_date
|
|
).group_by(
|
|
FactNotificationStatus.notification_type,
|
|
FactNotificationStatus.notification_status,
|
|
FactNotificationStatus.key_type,
|
|
)
|
|
today = get_london_midnight_in_utc(datetime.utcnow())
|
|
if start_date <= datetime.utcnow().date() <= end_date:
|
|
stats_for_today = db.session.query(
|
|
Notification.notification_type.cast(db.Text).label('notification_type'),
|
|
Notification.status,
|
|
Notification.key_type,
|
|
func.count().label('count')
|
|
).filter(
|
|
Notification.created_at >= today
|
|
).group_by(
|
|
Notification.notification_type.cast(db.Text),
|
|
Notification.status,
|
|
Notification.key_type,
|
|
)
|
|
all_stats_table = stats.union_all(stats_for_today).subquery()
|
|
query = db.session.query(
|
|
all_stats_table.c.notification_type,
|
|
all_stats_table.c.status,
|
|
all_stats_table.c.key_type,
|
|
func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
|
|
).group_by(
|
|
all_stats_table.c.notification_type,
|
|
all_stats_table.c.status,
|
|
all_stats_table.c.key_type,
|
|
).order_by(
|
|
all_stats_table.c.notification_type
|
|
)
|
|
else:
|
|
query = stats.order_by(
|
|
FactNotificationStatus.notification_type
|
|
)
|
|
return query.all()
|
|
|
|
|
|
def fetch_notification_statuses_for_job(job_id):
|
|
return db.session.query(
|
|
FactNotificationStatus.notification_status.label('status'),
|
|
func.sum(FactNotificationStatus.notification_count).label('count'),
|
|
).filter(
|
|
FactNotificationStatus.job_id == job_id,
|
|
).group_by(
|
|
FactNotificationStatus.notification_status
|
|
).all()
|
|
|
|
|
|
def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_from_test_key=True):
|
|
stats = db.session.query(
|
|
FactNotificationStatus.service_id.label('service_id'),
|
|
Service.name.label('name'),
|
|
Service.restricted.label('restricted'),
|
|
Service.research_mode.label('research_mode'),
|
|
Service.active.label('active'),
|
|
Service.created_at.label('created_at'),
|
|
FactNotificationStatus.notification_type.label('notification_type'),
|
|
FactNotificationStatus.notification_status.label('status'),
|
|
func.sum(FactNotificationStatus.notification_count).label('count')
|
|
).filter(
|
|
FactNotificationStatus.bst_date >= start_date,
|
|
FactNotificationStatus.bst_date <= end_date,
|
|
FactNotificationStatus.service_id == Service.id,
|
|
).group_by(
|
|
FactNotificationStatus.service_id.label('service_id'),
|
|
Service.name,
|
|
Service.restricted,
|
|
Service.research_mode,
|
|
Service.active,
|
|
Service.created_at,
|
|
FactNotificationStatus.notification_type,
|
|
FactNotificationStatus.notification_status,
|
|
).order_by(
|
|
FactNotificationStatus.service_id,
|
|
FactNotificationStatus.notification_type
|
|
)
|
|
if not include_from_test_key:
|
|
stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST)
|
|
|
|
if start_date <= datetime.utcnow().date() <= end_date:
|
|
today = get_london_midnight_in_utc(datetime.utcnow())
|
|
subquery = db.session.query(
|
|
Notification.notification_type.cast(db.Text).label('notification_type'),
|
|
Notification.status.label('status'),
|
|
Notification.service_id.label('service_id'),
|
|
func.count(Notification.id).label('count')
|
|
).filter(
|
|
Notification.created_at >= today
|
|
).group_by(
|
|
Notification.notification_type,
|
|
Notification.status,
|
|
Notification.service_id
|
|
)
|
|
if not include_from_test_key:
|
|
subquery = subquery.filter(Notification.key_type != KEY_TYPE_TEST)
|
|
subquery = subquery.subquery()
|
|
|
|
stats_for_today = db.session.query(
|
|
Service.id.label('service_id'),
|
|
Service.name.label('name'),
|
|
Service.restricted.label('restricted'),
|
|
Service.research_mode.label('research_mode'),
|
|
Service.active.label('active'),
|
|
Service.created_at.label('created_at'),
|
|
subquery.c.notification_type.label('notification_type'),
|
|
subquery.c.status.label('status'),
|
|
subquery.c.count.label('count')
|
|
).outerjoin(
|
|
subquery,
|
|
subquery.c.service_id == Service.id
|
|
)
|
|
|
|
all_stats_table = stats.union_all(stats_for_today).subquery()
|
|
query = db.session.query(
|
|
all_stats_table.c.service_id,
|
|
all_stats_table.c.name,
|
|
all_stats_table.c.restricted,
|
|
all_stats_table.c.research_mode,
|
|
all_stats_table.c.active,
|
|
all_stats_table.c.created_at,
|
|
all_stats_table.c.notification_type,
|
|
all_stats_table.c.status,
|
|
func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
|
|
).group_by(
|
|
all_stats_table.c.service_id,
|
|
all_stats_table.c.name,
|
|
all_stats_table.c.restricted,
|
|
all_stats_table.c.research_mode,
|
|
all_stats_table.c.active,
|
|
all_stats_table.c.created_at,
|
|
all_stats_table.c.notification_type,
|
|
all_stats_table.c.status,
|
|
).order_by(
|
|
all_stats_table.c.name,
|
|
all_stats_table.c.notification_type,
|
|
all_stats_table.c.status
|
|
)
|
|
else:
|
|
query = stats
|
|
return query.all()
|
|
|
|
|
|
def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
|
|
# services_dao.replaces dao_fetch_monthly_historical_usage_by_template_for_service
|
|
stats = db.session.query(
|
|
FactNotificationStatus.template_id.label('template_id'),
|
|
Template.name.label('name'),
|
|
Template.template_type.label('template_type'),
|
|
Template.is_precompiled_letter.label('is_precompiled_letter'),
|
|
extract('month', FactNotificationStatus.bst_date).label('month'),
|
|
extract('year', FactNotificationStatus.bst_date).label('year'),
|
|
func.sum(FactNotificationStatus.notification_count).label('count')
|
|
).join(
|
|
Template, FactNotificationStatus.template_id == Template.id
|
|
).filter(
|
|
FactNotificationStatus.service_id == service_id,
|
|
FactNotificationStatus.bst_date >= start_date,
|
|
FactNotificationStatus.bst_date <= end_date,
|
|
FactNotificationStatus.key_type != KEY_TYPE_TEST,
|
|
FactNotificationStatus.notification_status != NOTIFICATION_CANCELLED,
|
|
).group_by(
|
|
FactNotificationStatus.template_id,
|
|
Template.name,
|
|
Template.template_type,
|
|
Template.is_precompiled_letter,
|
|
extract('month', FactNotificationStatus.bst_date).label('month'),
|
|
extract('year', FactNotificationStatus.bst_date).label('year'),
|
|
).order_by(
|
|
extract('year', FactNotificationStatus.bst_date),
|
|
extract('month', FactNotificationStatus.bst_date),
|
|
Template.name
|
|
)
|
|
|
|
if start_date <= datetime.utcnow() <= end_date:
|
|
today = get_london_midnight_in_utc(datetime.utcnow())
|
|
month = get_london_month_from_utc_column(Notification.created_at)
|
|
|
|
stats_for_today = db.session.query(
|
|
Notification.template_id.label('template_id'),
|
|
Template.name.label('name'),
|
|
Template.template_type.label('template_type'),
|
|
Template.is_precompiled_letter.label('is_precompiled_letter'),
|
|
extract('month', month).label('month'),
|
|
extract('year', month).label('year'),
|
|
func.count().label('count')
|
|
).join(
|
|
Template, Notification.template_id == Template.id,
|
|
).filter(
|
|
Notification.created_at >= today,
|
|
Notification.service_id == service_id,
|
|
Notification.key_type != KEY_TYPE_TEST,
|
|
Notification.status != NOTIFICATION_CANCELLED
|
|
).group_by(
|
|
Notification.template_id,
|
|
Template.hidden,
|
|
Template.name,
|
|
Template.template_type,
|
|
month
|
|
)
|
|
|
|
all_stats_table = stats.union_all(stats_for_today).subquery()
|
|
query = db.session.query(
|
|
all_stats_table.c.template_id,
|
|
all_stats_table.c.name,
|
|
all_stats_table.c.is_precompiled_letter,
|
|
all_stats_table.c.template_type,
|
|
func.cast(all_stats_table.c.month, Integer).label('month'),
|
|
func.cast(all_stats_table.c.year, Integer).label('year'),
|
|
func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
|
|
).group_by(
|
|
all_stats_table.c.template_id,
|
|
all_stats_table.c.name,
|
|
all_stats_table.c.is_precompiled_letter,
|
|
all_stats_table.c.template_type,
|
|
all_stats_table.c.month,
|
|
all_stats_table.c.year,
|
|
).order_by(
|
|
all_stats_table.c.year,
|
|
all_stats_table.c.month,
|
|
all_stats_table.c.name
|
|
)
|
|
else:
|
|
query = stats
|
|
return query.all()
|
|
|
|
|
|
def get_total_sent_notifications_for_day_and_type(day, notification_type):
|
|
result = db.session.query(
|
|
func.sum(FactNotificationStatus.notification_count).label('count')
|
|
).filter(
|
|
FactNotificationStatus.notification_type == notification_type,
|
|
FactNotificationStatus.key_type != KEY_TYPE_TEST,
|
|
FactNotificationStatus.bst_date == day,
|
|
).scalar()
|
|
|
|
return result or 0
|
|
|
|
|
|
def fetch_monthly_notification_statuses_per_service(start_date, end_date):
|
|
return db.session.query(
|
|
func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date).label('date_created'),
|
|
Service.id.label('service_id'),
|
|
Service.name.label('service_name'),
|
|
FactNotificationStatus.notification_type,
|
|
func.sum(case(
|
|
[
|
|
(FactNotificationStatus.notification_status == NOTIFICATION_SENDING,
|
|
FactNotificationStatus.notification_count)
|
|
],
|
|
else_=0)).label('count_sending'),
|
|
func.sum(case(
|
|
[
|
|
(FactNotificationStatus.notification_status == NOTIFICATION_DELIVERED,
|
|
FactNotificationStatus.notification_count)
|
|
],
|
|
else_=0)).label('count_delivered'),
|
|
func.sum(case(
|
|
[
|
|
(FactNotificationStatus.notification_status.in_([NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_FAILED]),
|
|
FactNotificationStatus.notification_count)
|
|
],
|
|
else_=0)).label('count_technical_failure'),
|
|
func.sum(case(
|
|
[
|
|
(FactNotificationStatus.notification_status == NOTIFICATION_TEMPORARY_FAILURE,
|
|
FactNotificationStatus.notification_count)
|
|
],
|
|
else_=0)).label('count_temporary_failure'),
|
|
func.sum(case(
|
|
[
|
|
(FactNotificationStatus.notification_status == NOTIFICATION_PERMANENT_FAILURE,
|
|
FactNotificationStatus.notification_count)
|
|
],
|
|
else_=0)).label('count_permanent_failure'),
|
|
func.sum(case(
|
|
[
|
|
(FactNotificationStatus.notification_status == NOTIFICATION_SENT,
|
|
FactNotificationStatus.notification_count)
|
|
],
|
|
else_=0)).label('count_sent'),
|
|
).join(
|
|
Service, FactNotificationStatus.service_id == Service.id
|
|
).filter(
|
|
FactNotificationStatus.notification_status != NOTIFICATION_CREATED,
|
|
Service.active.is_(True),
|
|
FactNotificationStatus.key_type != KEY_TYPE_TEST,
|
|
Service.research_mode.is_(False),
|
|
Service.restricted.is_(False),
|
|
FactNotificationStatus.bst_date >= start_date,
|
|
FactNotificationStatus.bst_date <= end_date,
|
|
).group_by(
|
|
Service.id,
|
|
Service.name,
|
|
func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date),
|
|
FactNotificationStatus.notification_type,
|
|
).order_by(
|
|
func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date),
|
|
Service.id,
|
|
FactNotificationStatus.notification_type,
|
|
).all()
|