darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-20 14:29:25 -04:00
Code Issues Packages Projects Releases Wiki Activity

merge from main

Browse Source
This commit is contained in:
Kenneth Kehl
2024-03-01 13:50:09 -08:00
parent aef797eb79 18dddf4043
commit b777249b6f
140 changed files with 8031 additions and 5017 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/dao/annual_billing_dao.py
View File

@@ -3,6 +3,7 @@ from flask import current_app
from app import db
from app.dao.dao_utils import autocommit
from app.dao.date_util import get_current_calendar_year_start_year
from app.enums import OrganizationType
from app.models import AnnualBilling
@@ -65,17 +66,17 @@ def dao_get_all_free_sms_fragment_limit(service_id):
def set_default_free_allowance_for_service(service, year_start=None):
default_free_sms_fragment_limits = {
"federal": {
OrganizationType.FEDERAL: {
2020: 250_000,
2021: 150_000,
2022: 40_000,
},
"state": {
OrganizationType.STATE: {
2020: 250_000,
2021: 150_000,
2022: 40_000,
},
"other": {
OrganizationType.OTHER: {
2020: 250_000,
2021: 150_000,
2022: 40_000,
@@ -97,7 +98,9 @@ def set_default_free_allowance_for_service(service, year_start=None):
f"no organization type for service {service.id}. Using other default of "
f"{default_free_sms_fragment_limits['other'][year_start]}"
)
free_allowance = default_free_sms_fragment_limits["other"][year_start]
free_allowance = default_free_sms_fragment_limits[OrganizationType.OTHER][
year_start
]
return dao_create_or_update_annual_billing_for_year(
service.id, free_allowance, year_start

6
app/dao/date_util.py
View File

@@ -3,11 +3,7 @@ from datetime import date, datetime, time, timedelta
def get_months_for_financial_year(year):
return [
month
for month in (
get_months_for_year(4, 13, year) + get_months_for_year(1, 4, year + 1)
)
if month < datetime.now()
month for month in (get_months_for_year(1, 13, year)) if month < datetime.now()
]

55
app/dao/fact_billing_dao.py
View File

@@ -8,13 +8,8 @@ from sqlalchemy.sql.expression import case, literal
from app import db
from app.dao.date_util import get_calendar_year_dates, get_calendar_year_for_datetime
from app.dao.organization_dao import dao_get_organization_live_services
from app.enums import KeyType, NotificationStatus, NotificationType
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
NOTIFICATION_STATUS_TYPES_BILLABLE_SMS,
NOTIFICATION_STATUS_TYPES_SENT_EMAILS,
SMS_TYPE,
AnnualBilling,
FactBilling,
NotificationAllTimeView,
@@ -53,7 +48,7 @@ def fetch_sms_free_allowance_remainder_until_date(end_date):
AnnualBilling.service_id == FactBilling.service_id,
FactBilling.local_date >= start_of_year,
FactBilling.local_date < end_date,
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
),
)
.filter(
@@ -117,7 +112,7 @@ def fetch_sms_billing_for_all_services(start_date, end_date):
.filter(
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
)
.group_by(
Organization.name,
@@ -269,7 +264,7 @@ def query_service_email_usage_for_year(service_id, year):
FactBilling.service_id == service_id,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == EMAIL_TYPE,
FactBilling.notification_type == NotificationType.EMAIL,
)
@@ -356,7 +351,7 @@ def query_service_sms_usage_for_year(service_id, year):
FactBilling.service_id == service_id,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
AnnualBilling.financial_year_start == year,
)
)
@@ -386,7 +381,7 @@ def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=F
services = [Service.query.get(service_id)]
for service in services:
for notification_type in (SMS_TYPE, EMAIL_TYPE):
for notification_type in (NotificationType.SMS, NotificationType.EMAIL):
if (not check_permissions) or service.has_permission(notification_type):
results = _query_for_billing_data(
notification_type=notification_type,
@@ -414,9 +409,9 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
)
.filter(
NotificationAllTimeView.status.in_(
NOTIFICATION_STATUS_TYPES_SENT_EMAILS
NotificationStatus.sent_email_types()
),
NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
NotificationAllTimeView.key_type.in_((KeyType.NORMAL, KeyType.TEAM)),
NotificationAllTimeView.created_at >= start_date,
NotificationAllTimeView.created_at < end_date,
NotificationAllTimeView.notification_type == notification_type,
@@ -448,9 +443,9 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
)
.filter(
NotificationAllTimeView.status.in_(
NOTIFICATION_STATUS_TYPES_BILLABLE_SMS
NotificationStatus.billable_sms_types()
),
NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
NotificationAllTimeView.key_type.in_((KeyType.NORMAL, KeyType.TEAM)),
NotificationAllTimeView.created_at >= start_date,
NotificationAllTimeView.created_at < end_date,
NotificationAllTimeView.notification_type == notification_type,
@@ -465,8 +460,8 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
)
query_funcs = {
SMS_TYPE: _sms_query,
EMAIL_TYPE: _email_query,
NotificationType.SMS: _sms_query,
NotificationType.EMAIL: _email_query,
}
query = query_funcs[notification_type]()
@@ -484,7 +479,9 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
.filter(
NotificationHistory.created_at >= start_date,
NotificationHistory.created_at <= end_date,
NotificationHistory.notification_type.in_([SMS_TYPE, EMAIL_TYPE]),
NotificationHistory.notification_type.in_(
[NotificationType.SMS, NotificationType.EMAIL]
),
NotificationHistory.billable_units != 0,
)
.distinct()
@@ -495,7 +492,7 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
def get_rate(rates, notification_type, date):
start_of_day = get_midnight_in_utc(date)
if notification_type == SMS_TYPE:
if notification_type == NotificationType.SMS:
return next(
r.rate
for r in rates
@@ -576,7 +573,7 @@ def fetch_email_usage_for_organization(organization_id, start_date, end_date):
.filter(
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == EMAIL_TYPE,
FactBilling.notification_type == NotificationType.EMAIL,
Service.organization_id == organization_id,
Service.restricted.is_(False),
)
@@ -690,7 +687,7 @@ def query_organization_sms_usage_for_year(organization_id, year):
Service.id == FactBilling.service_id,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
),
)
.filter(
@@ -784,7 +781,7 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.notifications_sent,
)
],
@@ -795,7 +792,7 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.billable_units,
)
],
@@ -806,7 +803,7 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.billable_units * FactBilling.rate_multiplier,
)
],
@@ -817,7 +814,7 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
case(
[
(
FactBilling.notification_type == EMAIL_TYPE,
FactBilling.notification_type == NotificationType.EMAIL,
FactBilling.notifications_sent,
)
],
@@ -871,7 +868,7 @@ def fetch_daily_sms_provider_volumes_for_platform(start_date, end_date):
).label("sms_cost"),
)
.filter(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
)
@@ -902,7 +899,7 @@ def fetch_volumes_by_service(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.notifications_sent,
)
],
@@ -913,7 +910,7 @@ def fetch_volumes_by_service(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.billable_units * FactBilling.rate_multiplier,
)
],
@@ -924,7 +921,7 @@ def fetch_volumes_by_service(start_date, end_date):
case(
[
(
FactBilling.notification_type == EMAIL_TYPE,
FactBilling.notification_type == NotificationType.EMAIL,
FactBilling.notifications_sent,
)
],

99
app/dao/fact_notification_status_dao.py
View File

@@ -7,20 +7,8 @@ from sqlalchemy.types import DateTime, Integer
from app import db
from app.dao.dao_utils import autocommit
from app.enums import KeyType, NotificationStatus, NotificationType
from app.models import (
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST,
NOTIFICATION_CANCELLED,
NOTIFICATION_CREATED,
NOTIFICATION_DELIVERED,
NOTIFICATION_FAILED,
NOTIFICATION_PENDING,
NOTIFICATION_PERMANENT_FAILURE,
NOTIFICATION_SENDING,
NOTIFICATION_SENT,
NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_TEMPORARY_FAILURE,
FactNotificationStatus,
Notification,
NotificationAllTimeView,
@@ -64,7 +52,7 @@ def update_fact_notification_status(process_day, notification_type, service_id):
NotificationAllTimeView.created_at < end_date,
NotificationAllTimeView.notification_type == notification_type,
NotificationAllTimeView.service_id == service_id,
NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
NotificationAllTimeView.key_type.in_((KeyType.NORMAL, KeyType.TEAM)),
)
.group_by(
NotificationAllTimeView.template_id,
@@ -104,7 +92,7 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service
FactNotificationStatus.service_id == service_id,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date < end_date,
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
)
.group_by(
func.date_trunc("month", FactNotificationStatus.local_date).label("month"),
@@ -129,7 +117,7 @@ def fetch_notification_status_for_service_for_day(fetch_day, service_id):
Notification.created_at
< get_midnight_in_utc(fetch_day + timedelta(days=1)),
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
)
.group_by(Notification.notification_type, Notification.status)
.all()
@@ -142,8 +130,10 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(
start_date = midnight_n_days_ago(limit_days)
now = datetime.utcnow()
stats_for_7_days = db.session.query(
FactNotificationStatus.notification_type.label("notification_type"),
FactNotificationStatus.notification_status.label("status"),
FactNotificationStatus.notification_type.cast(db.Text).label(
"notification_type"
),
FactNotificationStatus.notification_status.cast(db.Text).label("status"),
*(
[FactNotificationStatus.template_id.label("template_id")]
if by_template
@@ -153,20 +143,20 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(
).filter(
FactNotificationStatus.service_id == service_id,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
)
stats_for_today = (
db.session.query(
Notification.notification_type.cast(db.Text),
Notification.status,
Notification.status.cast(db.Text),
*([Notification.template_id] if by_template else []),
func.count().label("count"),
)
.filter(
Notification.created_at >= get_midnight_in_utc(now),
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
)
.group_by(
Notification.notification_type,
@@ -205,9 +195,11 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(
def fetch_notification_status_totals_for_all_services(start_date, end_date):
stats = (
db.session.query(
FactNotificationStatus.notification_type.label("notification_type"),
FactNotificationStatus.notification_status.label("status"),
FactNotificationStatus.key_type.label("key_type"),
FactNotificationStatus.notification_type.cast(db.Text).label(
"notification_type"
),
FactNotificationStatus.notification_status.cast(db.Text).label("status"),
FactNotificationStatus.key_type.cast(db.Text).label("key_type"),
func.sum(FactNotificationStatus.notification_count).label("count"),
)
.filter(
@@ -225,13 +217,13 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date):
stats_for_today = (
db.session.query(
Notification.notification_type.cast(db.Text).label("notification_type"),
Notification.status,
Notification.key_type,
Notification.status.cast(db.Text),
Notification.key_type.cast(db.Text),
func.count().label("count"),
)
.filter(Notification.created_at >= today)
.group_by(
Notification.notification_type.cast(db.Text),
Notification.notification_type,
Notification.status,
Notification.key_type,
)
@@ -280,8 +272,10 @@ def fetch_stats_for_all_services_by_date_range(
Service.restricted.label("restricted"),
Service.active.label("active"),
Service.created_at.label("created_at"),
FactNotificationStatus.notification_type.label("notification_type"),
FactNotificationStatus.notification_status.label("status"),
FactNotificationStatus.notification_type.cast(db.Text).label(
"notification_type"
),
FactNotificationStatus.notification_status.cast(db.Text).label("status"),
func.sum(FactNotificationStatus.notification_count).label("count"),
)
.filter(
@@ -303,13 +297,13 @@ def fetch_stats_for_all_services_by_date_range(
)
)
if not include_from_test_key:
stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST)
stats = stats.filter(FactNotificationStatus.key_type != KeyType.TEST)
if start_date <= datetime.utcnow().date() <= end_date:
today = get_midnight_in_utc(datetime.utcnow())
subquery = (
db.session.query(
Notification.notification_type.cast(db.Text).label("notification_type"),
Notification.notification_type.label("notification_type"),
Notification.status.label("status"),
Notification.service_id.label("service_id"),
func.count(Notification.id).label("count"),
@@ -322,7 +316,7 @@ def fetch_stats_for_all_services_by_date_range(
)
)
if not include_from_test_key:
subquery = subquery.filter(Notification.key_type != KEY_TYPE_TEST)
subquery = subquery.filter(Notification.key_type != KeyType.TEST)
subquery = subquery.subquery()
stats_for_today = db.session.query(
@@ -331,8 +325,8 @@ def fetch_stats_for_all_services_by_date_range(
Service.restricted.label("restricted"),
Service.active.label("active"),
Service.created_at.label("created_at"),
subquery.c.notification_type.label("notification_type"),
subquery.c.status.label("status"),
subquery.c.notification_type.cast(db.Text).label("notification_type"),
subquery.c.status.cast(db.Text).label("status"),
subquery.c.count.label("count"),
).outerjoin(subquery, subquery.c.service_id == Service.id)
@@ -384,8 +378,8 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
FactNotificationStatus.service_id == service_id,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date <= end_date,
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.notification_status != NOTIFICATION_CANCELLED,
FactNotificationStatus.key_type != KeyType.TEST,
FactNotificationStatus.notification_status != NotificationStatus.CANCELLED,
)
.group_by(
FactNotificationStatus.template_id,
@@ -421,8 +415,8 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
.filter(
Notification.created_at >= today,
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.status != NOTIFICATION_CANCELLED,
Notification.key_type != KeyType.TEST,
Notification.status != NotificationStatus.CANCELLED,
)
.group_by(
Notification.template_id,
@@ -462,12 +456,13 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
def get_total_notifications_for_date_range(start_date, end_date):
query = (
db.session.query(
FactNotificationStatus.local_date.cast(db.Text).label("local_date"),
FactNotificationStatus.local_date.label("local_date"),
func.sum(
case(
[
(
FactNotificationStatus.notification_type == "email",
FactNotificationStatus.notification_type
== NotificationType.EMAIL,
FactNotificationStatus.notification_count,
)
],
@@ -478,7 +473,8 @@ def get_total_notifications_for_date_range(start_date, end_date):
case(
[
(
FactNotificationStatus.notification_type == "sms",
FactNotificationStatus.notification_type
== NotificationType.SMS,
FactNotificationStatus.notification_count,
)
],
@@ -487,7 +483,7 @@ def get_total_notifications_for_date_range(start_date, end_date):
).label("sms"),
)
.filter(
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
)
.group_by(FactNotificationStatus.local_date)
.order_by(FactNotificationStatus.local_date)
@@ -514,7 +510,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status.in_(
[NOTIFICATION_SENDING, NOTIFICATION_PENDING]
[NotificationStatus.SENDING, NotificationStatus.PENDING]
),
FactNotificationStatus.notification_count,
)
@@ -527,7 +523,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status
== NOTIFICATION_DELIVERED,
== NotificationStatus.DELIVERED,
FactNotificationStatus.notification_count,
)
],
@@ -539,7 +535,10 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status.in_(
[NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_FAILED]
[
NotificationStatus.TECHNICAL_FAILURE,
NotificationStatus.FAILED,
]
),
FactNotificationStatus.notification_count,
)
@@ -552,7 +551,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status
== NOTIFICATION_TEMPORARY_FAILURE,
== NotificationStatus.TEMPORARY_FAILURE,
FactNotificationStatus.notification_count,
)
],
@@ -564,7 +563,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status
== NOTIFICATION_PERMANENT_FAILURE,
== NotificationStatus.PERMANENT_FAILURE,
FactNotificationStatus.notification_count,
)
],
@@ -576,7 +575,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status
== NOTIFICATION_SENT,
== NotificationStatus.SENT,
FactNotificationStatus.notification_count,
)
],
@@ -586,9 +585,9 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
)
.join(Service, FactNotificationStatus.service_id == Service.id)
.filter(
FactNotificationStatus.notification_status != NOTIFICATION_CREATED,
FactNotificationStatus.notification_status != NotificationStatus.CREATED,
Service.active.is_(True),
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
Service.restricted.is_(False),
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date <= end_date,

11
app/dao/inbound_sms_dao.py
View File

@@ -5,13 +5,8 @@ from sqlalchemy.orm import aliased
from app import db
from app.dao.dao_utils import autocommit
from app.models import (
SMS_TYPE,
InboundSms,
InboundSmsHistory,
Service,
ServiceDataRetention,
)
from app.enums import NotificationType
from app.models import InboundSms, InboundSmsHistory, Service, ServiceDataRetention
from app.utils import midnight_n_days_ago
@@ -130,7 +125,7 @@ def delete_inbound_sms_older_than_retention():
ServiceDataRetention.query.join(
ServiceDataRetention.service, Service.inbound_number
)
.filter(ServiceDataRetention.notification_type == SMS_TYPE)
.filter(ServiceDataRetention.notification_type == NotificationType.SMS)
.all()
)

9
app/dao/invited_user_dao.py
View File

@@ -1,7 +1,8 @@
from datetime import datetime, timedelta
from app import db
from app.models import INVITE_EXPIRED, INVITE_PENDING, InvitedUser
from app.enums import InvitedUserStatus
from app.models import InvitedUser
def save_invited_user(invited_user):
@@ -20,7 +21,7 @@ def get_expired_invite_by_service_and_id(service_id, invited_user_id):
return InvitedUser.query.filter(
InvitedUser.service_id == service_id,
InvitedUser.id == invited_user_id,
InvitedUser.status == INVITE_EXPIRED,
InvitedUser.status == InvitedUserStatus.EXPIRED,
).one()
@@ -41,9 +42,9 @@ def expire_invitations_created_more_than_two_days_ago():
db.session.query(InvitedUser)
.filter(
InvitedUser.created_at <= datetime.utcnow() - timedelta(days=2),
InvitedUser.status.in_((INVITE_PENDING,)),
InvitedUser.status.in_((InvitedUserStatus.PENDING,)),
)
.update({InvitedUser.status: INVITE_EXPIRED})
.update({InvitedUser.status: InvitedUserStatus.EXPIRED})
)
db.session.commit()
return expired

14
app/dao/jobs_dao.py
View File

@@ -5,10 +5,8 @@ from flask import current_app
from sqlalchemy import and_, asc, desc, func
from app import db
from app.enums import JobStatus
from app.models import (
JOB_STATUS_FINISHED,
JOB_STATUS_PENDING,
JOB_STATUS_SCHEDULED,
FactNotificationStatus,
Job,
Notification,
@@ -85,7 +83,7 @@ def dao_get_scheduled_job_stats(
)
.filter(
Job.service_id == service_id,
Job.job_status == JOB_STATUS_SCHEDULED,
Job.job_status == JobStatus.SCHEDULED,
)
.one()
)
@@ -111,7 +109,7 @@ def dao_set_scheduled_jobs_to_pending():
"""
jobs = (
Job.query.filter(
Job.job_status == JOB_STATUS_SCHEDULED,
Job.job_status == JobStatus.SCHEDULED,
Job.scheduled_for < datetime.utcnow(),
)
.order_by(asc(Job.scheduled_for))
@@ -120,7 +118,7 @@ def dao_set_scheduled_jobs_to_pending():
)
for job in jobs:
job.job_status = JOB_STATUS_PENDING
job.job_status = JobStatus.PENDING
db.session.add_all(jobs)
db.session.commit()
@@ -132,7 +130,7 @@ def dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id):
return Job.query.filter(
Job.service_id == service_id,
Job.id == job_id,
Job.job_status == JOB_STATUS_SCHEDULED,
Job.job_status == JobStatus.SCHEDULED,
Job.scheduled_for > datetime.utcnow(),
).one()
@@ -200,7 +198,7 @@ def find_jobs_with_missing_rows():
jobs_with_rows_missing = (
db.session.query(Job)
.filter(
Job.job_status == JOB_STATUS_FINISHED,
Job.job_status == JobStatus.FINISHED,
Job.processing_finished < ten_minutes_ago,
Job.processing_finished > yesterday,
Job.id == Notification.job_id,

78
app/dao/notifications_dao.py
View File

@@ -16,22 +16,8 @@ from werkzeug.datastructures import MultiDict
from app import create_uuid, db
from app.dao.dao_utils import autocommit
from app.models import (
EMAIL_TYPE,
KEY_TYPE_TEST,
NOTIFICATION_CREATED,
NOTIFICATION_FAILED,
NOTIFICATION_PENDING,
NOTIFICATION_PENDING_VIRUS_CHECK,
NOTIFICATION_PERMANENT_FAILURE,
NOTIFICATION_SENDING,
NOTIFICATION_SENT,
NOTIFICATION_TEMPORARY_FAILURE,
SMS_TYPE,
FactNotificationStatus,
Notification,
NotificationHistory,
)
from app.enums import KeyType, NotificationStatus, NotificationType
from app.models import FactNotificationStatus, Notification, NotificationHistory
from app.utils import (
escape_special_characters,
get_midnight_in_utc,
@@ -45,7 +31,7 @@ def dao_get_last_date_template_was_used(template_id, service_id):
.filter(
Notification.service_id == service_id,
Notification.template_id == template_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
)
.scalar()
)
@@ -57,7 +43,7 @@ def dao_get_last_date_template_was_used(template_id, service_id):
db.session.query(functions.max(FactNotificationStatus.local_date))
.filter(
FactNotificationStatus.template_id == template_id,
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
)
.scalar()
)
@@ -71,7 +57,7 @@ def dao_create_notification(notification):
# need to populate defaulted fields before we create the notification history object
notification.id = create_uuid()
if not notification.status:
notification.status = NOTIFICATION_CREATED
notification.status = NotificationStatus.CREATED
# notify-api-749 do not write to db
# if we have a verify_code we know this is the authentication notification at login time
@@ -80,6 +66,7 @@ def dao_create_notification(notification):
pass
else:
notification.personalisation = ""
# notify-api-742 remove phone numbers from db
notification.to = "1"
notification.normalised_to = "1"
@@ -94,10 +81,10 @@ def country_records_delivery(phone_prefix):
def _decide_permanent_temporary_failure(current_status, status):
# If we go from pending to delivered we need to set failure type as temporary-failure
if (
current_status == NOTIFICATION_PENDING
and status == NOTIFICATION_PERMANENT_FAILURE
current_status == NotificationStatus.PENDING
and status == NotificationStatus.PERMANENT_FAILURE
):
status = NOTIFICATION_TEMPORARY_FAILURE
status = NotificationStatus.TEMPORARY_FAILURE
return status
@@ -136,17 +123,17 @@ def update_notification_status_by_id(
return None
if notification.status not in {
NOTIFICATION_CREATED,
NOTIFICATION_SENDING,
NOTIFICATION_PENDING,
NOTIFICATION_SENT,
NOTIFICATION_PENDING_VIRUS_CHECK,
NotificationStatus.CREATED,
NotificationStatus.SENDING,
NotificationStatus.PENDING,
NotificationStatus.SENT,
NotificationStatus.PENDING_VIRUS_CHECK,
}:
_duplicate_update_warning(notification, status)
return None
if (
notification.notification_type == SMS_TYPE
notification.notification_type == NotificationType.SMS
and notification.international
and not country_records_delivery(notification.phone_prefix)
):
@@ -180,7 +167,10 @@ def update_notification_status_by_reference(reference, status):
)
return None
if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
if notification.status not in {
NotificationStatus.SENDING,
NotificationStatus.PENDING,
}:
_duplicate_update_warning(notification, status)
return None
@@ -218,7 +208,9 @@ def dao_get_notification_count_for_service(*, service_id):
def dao_get_failed_notification_count():
failed_count = Notification.query.filter_by(status=NOTIFICATION_FAILED).count()
failed_count = Notification.query.filter_by(
status=NotificationStatus.FAILED
).count()
return failed_count
@@ -286,7 +278,7 @@ def get_notifications_for_service(
if key_type is not None:
filters.append(Notification.key_type == key_type)
elif not include_from_test_key:
filters.append(Notification.key_type != KEY_TYPE_TEST)
filters.append(Notification.key_type != KeyType.TEST)
if client_reference is not None:
filters.append(Notification.client_reference == client_reference)
@@ -426,7 +418,7 @@ def move_notifications_to_notification_history(
Notification.notification_type == notification_type,
Notification.service_id == service_id,
Notification.created_at < timestamp_to_delete_backwards_from,
Notification.key_type == KEY_TYPE_TEST,
Notification.key_type == KeyType.TEST,
).delete(synchronize_session=False)
db.session.commit()
@@ -446,14 +438,16 @@ def dao_timeout_notifications(cutoff_time, limit=100000):
if they're still sending from before the specified cutoff_time.
"""
updated_at = datetime.utcnow()
current_statuses = [NOTIFICATION_SENDING, NOTIFICATION_PENDING]
new_status = NOTIFICATION_TEMPORARY_FAILURE
current_statuses = [NotificationStatus.SENDING, NotificationStatus.PENDING]
new_status = NotificationStatus.TEMPORARY_FAILURE
notifications = (
Notification.query.filter(
Notification.created_at < cutoff_time,
Notification.status.in_(current_statuses),
Notification.notification_type.in_([SMS_TYPE, EMAIL_TYPE]),
Notification.notification_type.in_(
[NotificationType.SMS, NotificationType.EMAIL]
),
)
.limit(limit)
.all()
@@ -493,7 +487,7 @@ def dao_get_notifications_by_recipient_or_reference(
page_size=None,
error_out=True,
):
if notification_type == SMS_TYPE:
if notification_type == NotificationType.SMS:
normalised = try_validate_and_format_phone_number(search_term)
for character in {"(", ")", " ", "-"}:
@@ -501,7 +495,7 @@ def dao_get_notifications_by_recipient_or_reference(
normalised = normalised.lstrip("+0")
elif notification_type == EMAIL_TYPE:
elif notification_type == NotificationType.EMAIL:
try:
normalised = validate_and_format_email_address(search_term)
except InvalidEmailError:
@@ -515,7 +509,9 @@ def dao_get_notifications_by_recipient_or_reference(
normalised = "".join(search_term.split()).lower()
else:
raise TypeError(f"Notification type must be {EMAIL_TYPE}, {SMS_TYPE}, or None")
raise TypeError(
f"Notification type must be {NotificationType.EMAIL}, {NotificationType.SMS}, or None"
)
normalised = escape_special_characters(normalised)
search_term = escape_special_characters(search_term)
@@ -526,7 +522,7 @@ def dao_get_notifications_by_recipient_or_reference(
Notification.normalised_to.like("%{}%".format(normalised)),
Notification.client_reference.ilike("%{}%".format(search_term)),
),
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
]
if statuses:
@@ -589,7 +585,7 @@ def dao_get_notifications_processing_time_stats(start_date, end_date):
Notification.created_at >= start_date,
Notification.created_at < end_date,
Notification.api_key_id.isnot(None),
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
)
.one()
)
@@ -613,7 +609,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
notifications = Notification.query.filter(
Notification.created_at <= older_than_date,
Notification.notification_type == notification_type,
Notification.status == NOTIFICATION_CREATED,
Notification.status == NotificationStatus.CREATED,
).all()
return notifications

25
app/dao/permissions_dao.py
View File

@@ -1,26 +1,7 @@
from app import db
from app.dao import DAOClass
from app.models import (
MANAGE_API_KEYS,
MANAGE_SETTINGS,
MANAGE_TEMPLATES,
MANAGE_USERS,
SEND_EMAILS,
SEND_TEXTS,
VIEW_ACTIVITY,
Permission,
)
# Default permissions for a service
default_service_permissions = [
MANAGE_USERS,
MANAGE_TEMPLATES,
MANAGE_SETTINGS,
SEND_TEXTS,
SEND_EMAILS,
MANAGE_API_KEYS,
VIEW_ACTIVITY,
]
from app.enums import PermissionType
from app.models import Permission
class PermissionDAO(DAOClass):
@@ -28,7 +9,7 @@ class PermissionDAO(DAOClass):
model = Permission
def add_default_service_permissions_for_user(self, user, service):
for name in default_service_permissions:
for name in PermissionType.defaults():
permission = Permission(permission=name, user=user, service=service)
self.create_instance(permission, _commit=False)

14
app/dao/provider_details_dao.py
View File

@@ -5,13 +5,8 @@ from sqlalchemy import asc, desc, func
from app import db
from app.dao.dao_utils import autocommit
from app.models import (
SMS_TYPE,
FactBilling,
ProviderDetails,
ProviderDetailsHistory,
User,
)
from app.enums import NotificationType
from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, User
def get_provider_details_by_id(provider_details_id):
@@ -62,7 +57,8 @@ def _get_sms_providers_for_update(time_threshold):
# get current priority of both providers
q = (
ProviderDetails.query.filter(
ProviderDetails.notification_type == "sms", ProviderDetails.active
ProviderDetails.notification_type == NotificationType.SMS,
ProviderDetails.active,
)
.with_for_update()
.all()
@@ -126,7 +122,7 @@ def dao_get_provider_stats():
),
)
.filter(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.local_date >= first_day_of_the_month,
)
.group_by(FactBilling.provider)

13
app/dao/service_callback_api_dao.py
View File

@@ -2,11 +2,8 @@ from datetime import datetime
from app import create_uuid, db
from app.dao.dao_utils import autocommit, version_class
from app.models import (
COMPLAINT_CALLBACK_TYPE,
DELIVERY_STATUS_CALLBACK_TYPE,
ServiceCallbackApi,
)
from app.enums import CallbackType
from app.models import ServiceCallbackApi
@autocommit
@@ -40,13 +37,15 @@ def get_service_callback_api(service_callback_api_id, service_id):
def get_service_delivery_status_callback_api_for_service(service_id):
return ServiceCallbackApi.query.filter_by(
service_id=service_id, callback_type=DELIVERY_STATUS_CALLBACK_TYPE
service_id=service_id,
callback_type=CallbackType.DELIVERY_STATUS,
).first()
def get_service_complaint_callback_api_for_service(service_id):
return ServiceCallbackApi.query.filter_by(
service_id=service_id, callback_type=COMPLAINT_CALLBACK_TYPE
service_id=service_id,
callback_type=CallbackType.COMPLAINT,
).first()

43
app/dao/services_dao.py
View File

@@ -13,12 +13,13 @@ from app.dao.organization_dao import dao_get_organization_by_email_address
from app.dao.service_sms_sender_dao import insert_service_sms_sender
from app.dao.service_user_dao import dao_get_service_user
from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
from app.enums import (
KeyType,
NotificationStatus,
NotificationType,
ServicePermissionType,
)
from app.models import (
EMAIL_TYPE,
INTERNATIONAL_SMS_TYPE,
KEY_TYPE_TEST,
NOTIFICATION_PERMANENT_FAILURE,
SMS_TYPE,
AnnualBilling,
ApiKey,
FactBilling,
@@ -45,12 +46,6 @@ from app.utils import (
get_midnight_in_utc,
)
DEFAULT_SERVICE_PERMISSIONS = [
SMS_TYPE,
EMAIL_TYPE,
INTERNATIONAL_SMS_TYPE,
]
def dao_fetch_all_services(only_active=False):
query = Service.query.order_by(asc(Service.created_at)).options(joinedload("users"))
@@ -107,7 +102,8 @@ def dao_fetch_live_services_data():
case(
[
(
this_year_ft_billing.c.notification_type == "email",
this_year_ft_billing.c.notification_type
== NotificationType.EMAIL,
func.sum(this_year_ft_billing.c.notifications_sent),
)
],
@@ -116,7 +112,8 @@ def dao_fetch_live_services_data():
case(
[
(
this_year_ft_billing.c.notification_type == "sms",
this_year_ft_billing.c.notification_type
== NotificationType.SMS,
func.sum(this_year_ft_billing.c.notifications_sent),
)
],
@@ -278,7 +275,7 @@ def dao_create_service(
raise ValueError("Can't create a service without a user")
if service_permissions is None:
service_permissions = DEFAULT_SERVICE_PERMISSIONS
service_permissions = ServicePermissionType.defaults()
organization = dao_get_organization_by_email_address(user.email_address)
@@ -412,7 +409,7 @@ def dao_fetch_todays_stats_for_service(service_id):
)
.filter(
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
Notification.created_at >= start_date,
)
.group_by(
@@ -446,7 +443,7 @@ def dao_fetch_todays_stats_for_all_services(
)
if not include_from_test_key:
subquery = subquery.filter(Notification.key_type != KEY_TYPE_TEST)
subquery = subquery.filter(Notification.key_type != KeyType.TEST)
subquery = subquery.subquery()
@@ -517,8 +514,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
Notification.service_id == Service.id,
Notification.created_at >= start_date,
Notification.created_at <= end_date,
Notification.key_type != KEY_TYPE_TEST,
Notification.notification_type == SMS_TYPE,
Notification.key_type != KeyType.TEST,
Notification.notification_type == NotificationType.SMS,
func.substr(Notification.normalised_to, 3, 7) == "7700900",
Service.restricted == False, # noqa
Service.active == True, # noqa
@@ -541,8 +538,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
Notification.service_id == Service.id,
Notification.created_at >= start_date,
Notification.created_at <= end_date,
Notification.key_type != KEY_TYPE_TEST,
Notification.notification_type == SMS_TYPE,
Notification.key_type != KeyType.TEST,
Notification.notification_type == NotificationType.SMS,
Service.restricted == False, # noqa
Service.active == True, # noqa
)
@@ -569,9 +566,9 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
Notification.service_id == Service.id,
Notification.created_at >= start_date,
Notification.created_at <= end_date,
Notification.key_type != KEY_TYPE_TEST,
Notification.notification_type == SMS_TYPE,
Notification.status == NOTIFICATION_PERMANENT_FAILURE,
Notification.key_type != KeyType.TEST,
Notification.notification_type == NotificationType.SMS,
Notification.status == NotificationStatus.PERMANENT_FAILURE,
Service.restricted == False, # noqa
Service.active == True, # noqa
)

18
app/dao/uploads_dao.py
View File

@@ -5,16 +5,8 @@ from flask import current_app
from sqlalchemy import String, and_, desc, func, literal, text
from app import db
from app.models import (
JOB_STATUS_CANCELLED,
JOB_STATUS_SCHEDULED,
LETTER_TYPE,
NOTIFICATION_CANCELLED,
Job,
Notification,
ServiceDataRetention,
Template,
)
from app.enums import JobStatus, NotificationStatus, NotificationType
from app.models import Job, Notification, ServiceDataRetention, Template
from app.utils import midnight_n_days_ago
@@ -53,7 +45,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
Job.service_id == service_id,
Job.original_file_name != current_app.config["TEST_MESSAGE_FILENAME"],
Job.original_file_name != current_app.config["ONE_OFF_MESSAGE_FILENAME"],
Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED]),
Job.job_status.notin_([JobStatus.CANCELLED, JobStatus.SCHEDULED]),
func.coalesce(Job.processing_started, Job.created_at)
>= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
]
@@ -90,9 +82,9 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
letters_query_filter = [
Notification.service_id == service_id,
Notification.notification_type == LETTER_TYPE,
Notification.notification_type == NotificationType.LETTER,
Notification.api_key_id == None, # noqa
Notification.status != NOTIFICATION_CANCELLED,
Notification.status != NotificationStatus.CANCELLED,
Template.hidden == True, # noqa
Notification.created_at
>= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),

12
app/dao/users_dao.py
View File

@@ -9,8 +9,9 @@ from app import db
from app.dao.dao_utils import autocommit
from app.dao.permissions_dao import permission_dao
from app.dao.service_user_dao import dao_get_service_users_by_user_id
from app.enums import AuthType, PermissionType
from app.errors import InvalidRequest
from app.models import EMAIL_AUTH_TYPE, User, VerifyCode
from app.models import User, VerifyCode
from app.utils import escape_special_characters, get_archived_db_column_value
@@ -30,7 +31,10 @@ def save_user_attribute(usr, update_dict=None):
def save_model_user(
user, update_dict=None, password=None, validated_email_access=False
user,
update_dict=None,
password=None,
validated_email_access=False,
):
if password:
user.password = password
@@ -171,7 +175,7 @@ def dao_archive_user(user):
user.organizations = []
user.auth_type = EMAIL_AUTH_TYPE
user.auth_type = AuthType.EMAIL
user.email_address = get_archived_db_column_value(user.email_address)
user.mobile_number = None
user.password = str(uuid.uuid4())
@@ -194,7 +198,7 @@ def user_can_be_archived(user):
return False
if not any(
"manage_settings" in user.get_permissions(service.id)
PermissionType.MANAGE_SETTINGS in user.get_permissions(service.id)
for user in other_active_users
):
# no-one else has manage settings