darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-11 10:28:55 -04:00
Code Issues Packages Projects Releases Wiki Activity

merge with main

Browse Source
This commit is contained in:
Aditi Anand
2024-03-13 16:56:54 -04:00
parent 385ec3583f 8fe70fe14d
commit 5d15560538
167 changed files with 7728 additions and 3961 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/dao/annual_billing_dao.py
+7 -4
View File
@@ -3,6 +3,7 @@ from flask import current_app
from app import db
from app.dao.dao_utils import autocommit
from app.dao.date_util import get_current_calendar_year_start_year
from app.enums import OrganizationType
from app.models import AnnualBilling
@@ -65,17 +66,17 @@ def dao_get_all_free_sms_fragment_limit(service_id):
def set_default_free_allowance_for_service(service, year_start=None):
default_free_sms_fragment_limits = {
"federal": {
OrganizationType.FEDERAL: {
2020: 250_000,
2021: 150_000,
2022: 40_000,
},
"state": {
OrganizationType.STATE: {
2020: 250_000,
2021: 150_000,
2022: 40_000,
},
"other": {
OrganizationType.OTHER: {
2020: 250_000,
2021: 150_000,
2022: 40_000,
@@ -97,7 +98,9 @@ def set_default_free_allowance_for_service(service, year_start=None):
f"no organization type for service {service.id}. Using other default of "
f"{default_free_sms_fragment_limits['other'][year_start]}"
)
free_allowance = default_free_sms_fragment_limits["other"][year_start]
free_allowance = default_free_sms_fragment_limits[OrganizationType.OTHER][
year_start
]
return dao_create_or_update_annual_billing_for_year(
service.id, free_allowance, year_start
app/dao/date_util.py
+1 -5
View File
@@ -3,11 +3,7 @@ from datetime import date, datetime, time, timedelta
def get_months_for_financial_year(year):
return [
month
for month in (
get_months_for_year(4, 13, year) + get_months_for_year(1, 4, year + 1)
)
if month < datetime.now()
month for month in (get_months_for_year(1, 13, year)) if month < datetime.now()
]
app/dao/fact_billing_dao.py
+26 -29
View File
@@ -8,13 +8,8 @@ from sqlalchemy.sql.expression import case, literal
from app import db
from app.dao.date_util import get_calendar_year_dates, get_calendar_year_for_datetime
from app.dao.organization_dao import dao_get_organization_live_services
from app.enums import KeyType, NotificationStatus, NotificationType
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
NOTIFICATION_STATUS_TYPES_BILLABLE_SMS,
NOTIFICATION_STATUS_TYPES_SENT_EMAILS,
SMS_TYPE,
AnnualBilling,
FactBilling,
NotificationAllTimeView,
@@ -53,7 +48,7 @@ def fetch_sms_free_allowance_remainder_until_date(end_date):
AnnualBilling.service_id == FactBilling.service_id,
FactBilling.local_date >= start_of_year,
FactBilling.local_date < end_date,
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
),
)
.filter(
@@ -117,7 +112,7 @@ def fetch_sms_billing_for_all_services(start_date, end_date):
.filter(
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
)
.group_by(
Organization.name,
@@ -269,7 +264,7 @@ def query_service_email_usage_for_year(service_id, year):
FactBilling.service_id == service_id,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == EMAIL_TYPE,
FactBilling.notification_type == NotificationType.EMAIL,
)
@@ -356,7 +351,7 @@ def query_service_sms_usage_for_year(service_id, year):
FactBilling.service_id == service_id,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
AnnualBilling.financial_year_start == year,
)
)
@@ -386,7 +381,7 @@ def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=F
services = [Service.query.get(service_id)]
for service in services:
for notification_type in (SMS_TYPE, EMAIL_TYPE):
for notification_type in (NotificationType.SMS, NotificationType.EMAIL):
if (not check_permissions) or service.has_permission(notification_type):
results = _query_for_billing_data(
notification_type=notification_type,
@@ -414,9 +409,9 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
)
.filter(
NotificationAllTimeView.status.in_(
NOTIFICATION_STATUS_TYPES_SENT_EMAILS
NotificationStatus.sent_email_types()
),
NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
NotificationAllTimeView.key_type.in_((KeyType.NORMAL, KeyType.TEAM)),
NotificationAllTimeView.created_at >= start_date,
NotificationAllTimeView.created_at < end_date,
NotificationAllTimeView.notification_type == notification_type,
@@ -448,9 +443,9 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
)
.filter(
NotificationAllTimeView.status.in_(
NOTIFICATION_STATUS_TYPES_BILLABLE_SMS
NotificationStatus.billable_sms_types()
),
NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
NotificationAllTimeView.key_type.in_((KeyType.NORMAL, KeyType.TEAM)),
NotificationAllTimeView.created_at >= start_date,
NotificationAllTimeView.created_at < end_date,
NotificationAllTimeView.notification_type == notification_type,
@@ -465,8 +460,8 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
)
query_funcs = {
SMS_TYPE: _sms_query,
EMAIL_TYPE: _email_query,
NotificationType.SMS: _sms_query,
NotificationType.EMAIL: _email_query,
}
query = query_funcs[notification_type]()
@@ -484,7 +479,9 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
.filter(
NotificationHistory.created_at >= start_date,
NotificationHistory.created_at <= end_date,
NotificationHistory.notification_type.in_([SMS_TYPE, EMAIL_TYPE]),
NotificationHistory.notification_type.in_(
[NotificationType.SMS, NotificationType.EMAIL]
),
NotificationHistory.billable_units != 0,
)
.distinct()
@@ -495,7 +492,7 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
def get_rate(rates, notification_type, date):
start_of_day = get_midnight_in_utc(date)
if notification_type == SMS_TYPE:
if notification_type == NotificationType.SMS:
return next(
r.rate
for r in rates
@@ -576,7 +573,7 @@ def fetch_email_usage_for_organization(organization_id, start_date, end_date):
.filter(
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == EMAIL_TYPE,
FactBilling.notification_type == NotificationType.EMAIL,
Service.organization_id == organization_id,
Service.restricted.is_(False),
)
@@ -690,7 +687,7 @@ def query_organization_sms_usage_for_year(organization_id, year):
Service.id == FactBilling.service_id,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
),
)
.filter(
@@ -784,7 +781,7 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.notifications_sent,
)
],
@@ -795,7 +792,7 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.billable_units,
)
],
@@ -806,7 +803,7 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.billable_units * FactBilling.rate_multiplier,
)
],
@@ -817,7 +814,7 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
case(
[
(
FactBilling.notification_type == EMAIL_TYPE,
FactBilling.notification_type == NotificationType.EMAIL,
FactBilling.notifications_sent,
)
],
@@ -871,7 +868,7 @@ def fetch_daily_sms_provider_volumes_for_platform(start_date, end_date):
).label("sms_cost"),
)
.filter(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
)
@@ -902,7 +899,7 @@ def fetch_volumes_by_service(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.notifications_sent,
)
],
@@ -913,7 +910,7 @@ def fetch_volumes_by_service(start_date, end_date):
case(
[
(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.billable_units * FactBilling.rate_multiplier,
)
],
@@ -924,7 +921,7 @@ def fetch_volumes_by_service(start_date, end_date):
case(
[
(
FactBilling.notification_type == EMAIL_TYPE,
FactBilling.notification_type == NotificationType.EMAIL,
FactBilling.notifications_sent,
)
],
app/dao/fact_notification_status_dao.py
+49 -50
View File
@@ -7,20 +7,8 @@ from sqlalchemy.types import DateTime, Integer
from app import db
from app.dao.dao_utils import autocommit
from app.enums import KeyType, NotificationStatus, NotificationType
from app.models import (
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST,
NOTIFICATION_CANCELLED,
NOTIFICATION_CREATED,
NOTIFICATION_DELIVERED,
NOTIFICATION_FAILED,
NOTIFICATION_PENDING,
NOTIFICATION_PERMANENT_FAILURE,
NOTIFICATION_SENDING,
NOTIFICATION_SENT,
NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_TEMPORARY_FAILURE,
FactNotificationStatus,
Notification,
NotificationAllTimeView,
@@ -64,7 +52,7 @@ def update_fact_notification_status(process_day, notification_type, service_id):
NotificationAllTimeView.created_at < end_date,
NotificationAllTimeView.notification_type == notification_type,
NotificationAllTimeView.service_id == service_id,
NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
NotificationAllTimeView.key_type.in_((KeyType.NORMAL, KeyType.TEAM)),
)
.group_by(
NotificationAllTimeView.template_id,
@@ -104,7 +92,7 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service
FactNotificationStatus.service_id == service_id,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date < end_date,
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
)
.group_by(
func.date_trunc("month", FactNotificationStatus.local_date).label("month"),
@@ -129,7 +117,7 @@ def fetch_notification_status_for_service_for_day(fetch_day, service_id):
Notification.created_at
< get_midnight_in_utc(fetch_day + timedelta(days=1)),
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
)
.group_by(Notification.notification_type, Notification.status)
.all()
@@ -142,8 +130,10 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(
start_date = midnight_n_days_ago(limit_days)
now = datetime.utcnow()
stats_for_7_days = db.session.query(
FactNotificationStatus.notification_type.label("notification_type"),
FactNotificationStatus.notification_status.label("status"),
FactNotificationStatus.notification_type.cast(db.Text).label(
"notification_type"
),
FactNotificationStatus.notification_status.cast(db.Text).label("status"),
*(
[FactNotificationStatus.template_id.label("template_id")]
if by_template
@@ -153,20 +143,20 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(
).filter(
FactNotificationStatus.service_id == service_id,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
)
stats_for_today = (
db.session.query(
Notification.notification_type.cast(db.Text),
Notification.status,
Notification.status.cast(db.Text),
*([Notification.template_id] if by_template else []),
func.count().label("count"),
)
.filter(
Notification.created_at >= get_midnight_in_utc(now),
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
)
.group_by(
Notification.notification_type,
@@ -205,9 +195,11 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(
def fetch_notification_status_totals_for_all_services(start_date, end_date):
stats = (
db.session.query(
FactNotificationStatus.notification_type.label("notification_type"),
FactNotificationStatus.notification_status.label("status"),
FactNotificationStatus.key_type.label("key_type"),
FactNotificationStatus.notification_type.cast(db.Text).label(
"notification_type"
),
FactNotificationStatus.notification_status.cast(db.Text).label("status"),
FactNotificationStatus.key_type.cast(db.Text).label("key_type"),
func.sum(FactNotificationStatus.notification_count).label("count"),
)
.filter(
@@ -225,13 +217,13 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date):
stats_for_today = (
db.session.query(
Notification.notification_type.cast(db.Text).label("notification_type"),
Notification.status,
Notification.key_type,
Notification.status.cast(db.Text),
Notification.key_type.cast(db.Text),
func.count().label("count"),
)
.filter(Notification.created_at >= today)
.group_by(
Notification.notification_type.cast(db.Text),
Notification.notification_type,
Notification.status,
Notification.key_type,
)
@@ -280,8 +272,10 @@ def fetch_stats_for_all_services_by_date_range(
Service.restricted.label("restricted"),
Service.active.label("active"),
Service.created_at.label("created_at"),
FactNotificationStatus.notification_type.label("notification_type"),
FactNotificationStatus.notification_status.label("status"),
FactNotificationStatus.notification_type.cast(db.Text).label(
"notification_type"
),
FactNotificationStatus.notification_status.cast(db.Text).label("status"),
func.sum(FactNotificationStatus.notification_count).label("count"),
)
.filter(
@@ -303,13 +297,13 @@ def fetch_stats_for_all_services_by_date_range(
)
)
if not include_from_test_key:
stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST)
stats = stats.filter(FactNotificationStatus.key_type != KeyType.TEST)
if start_date <= datetime.utcnow().date() <= end_date:
today = get_midnight_in_utc(datetime.utcnow())
subquery = (
db.session.query(
Notification.notification_type.cast(db.Text).label("notification_type"),
Notification.notification_type.label("notification_type"),
Notification.status.label("status"),
Notification.service_id.label("service_id"),
func.count(Notification.id).label("count"),
@@ -322,7 +316,7 @@ def fetch_stats_for_all_services_by_date_range(
)
)
if not include_from_test_key:
subquery = subquery.filter(Notification.key_type != KEY_TYPE_TEST)
subquery = subquery.filter(Notification.key_type != KeyType.TEST)
subquery = subquery.subquery()
stats_for_today = db.session.query(
@@ -331,8 +325,8 @@ def fetch_stats_for_all_services_by_date_range(
Service.restricted.label("restricted"),
Service.active.label("active"),
Service.created_at.label("created_at"),
subquery.c.notification_type.label("notification_type"),
subquery.c.status.label("status"),
subquery.c.notification_type.cast(db.Text).label("notification_type"),
subquery.c.status.cast(db.Text).label("status"),
subquery.c.count.label("count"),
).outerjoin(subquery, subquery.c.service_id == Service.id)
@@ -384,8 +378,8 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
FactNotificationStatus.service_id == service_id,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date <= end_date,
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.notification_status != NOTIFICATION_CANCELLED,
FactNotificationStatus.key_type != KeyType.TEST,
FactNotificationStatus.notification_status != NotificationStatus.CANCELLED,
)
.group_by(
FactNotificationStatus.template_id,
@@ -421,8 +415,8 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
.filter(
Notification.created_at >= today,
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.status != NOTIFICATION_CANCELLED,
Notification.key_type != KeyType.TEST,
Notification.status != NotificationStatus.CANCELLED,
)
.group_by(
Notification.template_id,
@@ -462,12 +456,13 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
def get_total_notifications_for_date_range(start_date, end_date):
query = (
db.session.query(
FactNotificationStatus.local_date.cast(db.Text).label("local_date"),
FactNotificationStatus.local_date.label("local_date"),
func.sum(
case(
[
(
FactNotificationStatus.notification_type == "email",
FactNotificationStatus.notification_type
== NotificationType.EMAIL,
FactNotificationStatus.notification_count,
)
],
@@ -478,7 +473,8 @@ def get_total_notifications_for_date_range(start_date, end_date):
case(
[
(
FactNotificationStatus.notification_type == "sms",
FactNotificationStatus.notification_type
== NotificationType.SMS,
FactNotificationStatus.notification_count,
)
],
@@ -487,7 +483,7 @@ def get_total_notifications_for_date_range(start_date, end_date):
).label("sms"),
)
.filter(
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
)
.group_by(FactNotificationStatus.local_date)
.order_by(FactNotificationStatus.local_date)
@@ -514,7 +510,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status.in_(
[NOTIFICATION_SENDING, NOTIFICATION_PENDING]
[NotificationStatus.SENDING, NotificationStatus.PENDING]
),
FactNotificationStatus.notification_count,
)
@@ -527,7 +523,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status
== NOTIFICATION_DELIVERED,
== NotificationStatus.DELIVERED,
FactNotificationStatus.notification_count,
)
],
@@ -539,7 +535,10 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status.in_(
[NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_FAILED]
[
NotificationStatus.TECHNICAL_FAILURE,
NotificationStatus.FAILED,
]
),
FactNotificationStatus.notification_count,
)
@@ -552,7 +551,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status
== NOTIFICATION_TEMPORARY_FAILURE,
== NotificationStatus.TEMPORARY_FAILURE,
FactNotificationStatus.notification_count,
)
],
@@ -564,7 +563,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status
== NOTIFICATION_PERMANENT_FAILURE,
== NotificationStatus.PERMANENT_FAILURE,
FactNotificationStatus.notification_count,
)
],
@@ -576,7 +575,7 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
[
(
FactNotificationStatus.notification_status
== NOTIFICATION_SENT,
== NotificationStatus.SENT,
FactNotificationStatus.notification_count,
)
],
@@ -586,9 +585,9 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
)
.join(Service, FactNotificationStatus.service_id == Service.id)
.filter(
FactNotificationStatus.notification_status != NOTIFICATION_CREATED,
FactNotificationStatus.notification_status != NotificationStatus.CREATED,
Service.active.is_(True),
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
Service.restricted.is_(False),
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date <= end_date,
app/dao/inbound_sms_dao.py
+3 -8
View File
@@ -5,13 +5,8 @@ from sqlalchemy.orm import aliased
from app import db
from app.dao.dao_utils import autocommit
from app.models import (
SMS_TYPE,
InboundSms,
InboundSmsHistory,
Service,
ServiceDataRetention,
)
from app.enums import NotificationType
from app.models import InboundSms, InboundSmsHistory, Service, ServiceDataRetention
from app.utils import midnight_n_days_ago
@@ -130,7 +125,7 @@ def delete_inbound_sms_older_than_retention():
ServiceDataRetention.query.join(
ServiceDataRetention.service, Service.inbound_number
)
.filter(ServiceDataRetention.notification_type == SMS_TYPE)
.filter(ServiceDataRetention.notification_type == NotificationType.SMS)
.all()
)
app/dao/invited_user_dao.py
+5 -4
View File
@@ -1,7 +1,8 @@
from datetime import datetime, timedelta
from app import db
from app.models import INVITE_EXPIRED, INVITE_PENDING, InvitedUser
from app.enums import InvitedUserStatus
from app.models import InvitedUser
def save_invited_user(invited_user):
@@ -20,7 +21,7 @@ def get_expired_invite_by_service_and_id(service_id, invited_user_id):
return InvitedUser.query.filter(
InvitedUser.service_id == service_id,
InvitedUser.id == invited_user_id,
InvitedUser.status == INVITE_EXPIRED,
InvitedUser.status == InvitedUserStatus.EXPIRED,
).one()
@@ -41,9 +42,9 @@ def expire_invitations_created_more_than_two_days_ago():
db.session.query(InvitedUser)
.filter(
InvitedUser.created_at <= datetime.utcnow() - timedelta(days=2),
InvitedUser.status.in_((INVITE_PENDING,)),
InvitedUser.status.in_((InvitedUserStatus.PENDING,)),
)
.update({InvitedUser.status: INVITE_EXPIRED})
.update({InvitedUser.status: InvitedUserStatus.EXPIRED})
)
db.session.commit()
return expired
app/dao/jobs_dao.py
+6 -8
View File
@@ -5,10 +5,8 @@ from flask import current_app
from sqlalchemy import and_, asc, desc, func
from app import db
from app.enums import JobStatus
from app.models import (
JOB_STATUS_FINISHED,
JOB_STATUS_PENDING,
JOB_STATUS_SCHEDULED,
FactNotificationStatus,
Job,
Notification,
@@ -85,7 +83,7 @@ def dao_get_scheduled_job_stats(
)
.filter(
Job.service_id == service_id,
Job.job_status == JOB_STATUS_SCHEDULED,
Job.job_status == JobStatus.SCHEDULED,
)
.one()
)
@@ -111,7 +109,7 @@ def dao_set_scheduled_jobs_to_pending():
"""
jobs = (
Job.query.filter(
Job.job_status == JOB_STATUS_SCHEDULED,
Job.job_status == JobStatus.SCHEDULED,
Job.scheduled_for < datetime.utcnow(),
)
.order_by(asc(Job.scheduled_for))
@@ -120,7 +118,7 @@ def dao_set_scheduled_jobs_to_pending():
)
for job in jobs:
job.job_status = JOB_STATUS_PENDING
job.job_status = JobStatus.PENDING
db.session.add_all(jobs)
db.session.commit()
@@ -132,7 +130,7 @@ def dao_get_future_scheduled_job_by_id_and_service_id(job_id, service_id):
return Job.query.filter(
Job.service_id == service_id,
Job.id == job_id,
Job.job_status == JOB_STATUS_SCHEDULED,
Job.job_status == JobStatus.SCHEDULED,
Job.scheduled_for > datetime.utcnow(),
).one()
@@ -200,7 +198,7 @@ def find_jobs_with_missing_rows():
jobs_with_rows_missing = (
db.session.query(Job)
.filter(
Job.job_status == JOB_STATUS_FINISHED,
Job.job_status == JobStatus.FINISHED,
Job.processing_finished < ten_minutes_ago,
Job.processing_finished > yesterday,
Job.id == Notification.job_id,
app/dao/notifications_dao.py
+45 -41
View File
@@ -16,22 +16,8 @@ from werkzeug.datastructures import MultiDict
from app import create_uuid, db
from app.dao.dao_utils import autocommit
from app.models import (
EMAIL_TYPE,
KEY_TYPE_TEST,
NOTIFICATION_CREATED,
NOTIFICATION_FAILED,
NOTIFICATION_PENDING,
NOTIFICATION_PENDING_VIRUS_CHECK,
NOTIFICATION_PERMANENT_FAILURE,
NOTIFICATION_SENDING,
NOTIFICATION_SENT,
NOTIFICATION_TEMPORARY_FAILURE,
SMS_TYPE,
FactNotificationStatus,
Notification,
NotificationHistory,
)
from app.enums import KeyType, NotificationStatus, NotificationType
from app.models import FactNotificationStatus, Notification, NotificationHistory
from app.utils import (
escape_special_characters,
get_midnight_in_utc,
@@ -45,7 +31,7 @@ def dao_get_last_date_template_was_used(template_id, service_id):
.filter(
Notification.service_id == service_id,
Notification.template_id == template_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
)
.scalar()
)
@@ -57,7 +43,7 @@ def dao_get_last_date_template_was_used(template_id, service_id):
db.session.query(functions.max(FactNotificationStatus.local_date))
.filter(
FactNotificationStatus.template_id == template_id,
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.key_type != KeyType.TEST,
)
.scalar()
)
@@ -71,7 +57,16 @@ def dao_create_notification(notification):
# need to populate defaulted fields before we create the notification history object
notification.id = create_uuid()
if not notification.status:
notification.status = NOTIFICATION_CREATED
notification.status = NotificationStatus.CREATED
# notify-api-749 do not write to db
# if we have a verify_code we know this is the authentication notification at login time
# and not csv (containing PII) provided by the user, so allow verify_code to continue to exist
if "verify_code" in str(notification.personalisation):
pass
else:
notification.personalisation = ""
# notify-api-742 remove phone numbers from db
notification.to = "1"
notification.normalised_to = "1"
@@ -86,10 +81,10 @@ def country_records_delivery(phone_prefix):
def _decide_permanent_temporary_failure(current_status, status):
# If we go from pending to delivered we need to set failure type as temporary-failure
if (
current_status == NOTIFICATION_PENDING
and status == NOTIFICATION_PERMANENT_FAILURE
current_status == NotificationStatus.PENDING
and status == NotificationStatus.PERMANENT_FAILURE
):
status = NOTIFICATION_TEMPORARY_FAILURE
status = NotificationStatus.TEMPORARY_FAILURE
return status
@@ -128,17 +123,17 @@ def update_notification_status_by_id(
return None
if notification.status not in {
NOTIFICATION_CREATED,
NOTIFICATION_SENDING,
NOTIFICATION_PENDING,
NOTIFICATION_SENT,
NOTIFICATION_PENDING_VIRUS_CHECK,
NotificationStatus.CREATED,
NotificationStatus.SENDING,
NotificationStatus.PENDING,
NotificationStatus.SENT,
NotificationStatus.PENDING_VIRUS_CHECK,
}:
_duplicate_update_warning(notification, status)
return None
if (
notification.notification_type == SMS_TYPE
notification.notification_type == NotificationType.SMS
and notification.international
and not country_records_delivery(notification.phone_prefix)
):
@@ -172,7 +167,10 @@ def update_notification_status_by_reference(reference, status):
)
return None
if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
if notification.status not in {
NotificationStatus.SENDING,
NotificationStatus.PENDING,
}:
_duplicate_update_warning(notification, status)
return None
@@ -210,7 +208,9 @@ def dao_get_notification_count_for_service(*, service_id):
def dao_get_failed_notification_count():
failed_count = Notification.query.filter_by(status=NOTIFICATION_FAILED).count()
failed_count = Notification.query.filter_by(
status=NotificationStatus.FAILED
).count()
return failed_count
@@ -278,7 +278,7 @@ def get_notifications_for_service(
if key_type is not None:
filters.append(Notification.key_type == key_type)
elif not include_from_test_key:
filters.append(Notification.key_type != KEY_TYPE_TEST)
filters.append(Notification.key_type != KeyType.TEST)
if client_reference is not None:
filters.append(Notification.client_reference == client_reference)
@@ -418,7 +418,7 @@ def move_notifications_to_notification_history(
Notification.notification_type == notification_type,
Notification.service_id == service_id,
Notification.created_at < timestamp_to_delete_backwards_from,
Notification.key_type == KEY_TYPE_TEST,
Notification.key_type == KeyType.TEST,
).delete(synchronize_session=False)
db.session.commit()
@@ -438,14 +438,16 @@ def dao_timeout_notifications(cutoff_time, limit=100000):
if they're still sending from before the specified cutoff_time.
"""
updated_at = datetime.utcnow()
current_statuses = [NOTIFICATION_SENDING, NOTIFICATION_PENDING]
new_status = NOTIFICATION_TEMPORARY_FAILURE
current_statuses = [NotificationStatus.SENDING, NotificationStatus.PENDING]
new_status = NotificationStatus.TEMPORARY_FAILURE
notifications = (
Notification.query.filter(
Notification.created_at < cutoff_time,
Notification.status.in_(current_statuses),
Notification.notification_type.in_([SMS_TYPE, EMAIL_TYPE]),
Notification.notification_type.in_(
[NotificationType.SMS, NotificationType.EMAIL]
),
)
.limit(limit)
.all()
@@ -485,7 +487,7 @@ def dao_get_notifications_by_recipient_or_reference(
page_size=None,
error_out=True,
):
if notification_type == SMS_TYPE:
if notification_type == NotificationType.SMS:
normalised = try_validate_and_format_phone_number(search_term)
for character in {"(", ")", " ", "-"}:
@@ -493,7 +495,7 @@ def dao_get_notifications_by_recipient_or_reference(
normalised = normalised.lstrip("+0")
elif notification_type == EMAIL_TYPE:
elif notification_type == NotificationType.EMAIL:
try:
normalised = validate_and_format_email_address(search_term)
except InvalidEmailError:
@@ -507,7 +509,9 @@ def dao_get_notifications_by_recipient_or_reference(
normalised = "".join(search_term.split()).lower()
else:
raise TypeError(f"Notification type must be {EMAIL_TYPE}, {SMS_TYPE}, or None")
raise TypeError(
f"Notification type must be {NotificationType.EMAIL}, {NotificationType.SMS}, or None"
)
normalised = escape_special_characters(normalised)
search_term = escape_special_characters(search_term)
@@ -518,7 +522,7 @@ def dao_get_notifications_by_recipient_or_reference(
Notification.normalised_to.like("%{}%".format(normalised)),
Notification.client_reference.ilike("%{}%".format(search_term)),
),
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
]
if statuses:
@@ -581,7 +585,7 @@ def dao_get_notifications_processing_time_stats(start_date, end_date):
Notification.created_at >= start_date,
Notification.created_at < end_date,
Notification.api_key_id.isnot(None),
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
)
.one()
)
@@ -605,7 +609,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
notifications = Notification.query.filter(
Notification.created_at <= older_than_date,
Notification.notification_type == notification_type,
Notification.status == NOTIFICATION_CREATED,
Notification.status == NotificationStatus.CREATED,
).all()
return notifications
app/dao/permissions_dao.py
+3 -22
View File
@@ -1,26 +1,7 @@
from app import db
from app.dao import DAOClass
from app.models import (
MANAGE_API_KEYS,
MANAGE_SETTINGS,
MANAGE_TEMPLATES,
MANAGE_USERS,
SEND_EMAILS,
SEND_TEXTS,
VIEW_ACTIVITY,
Permission,
)
# Default permissions for a service
default_service_permissions = [
MANAGE_USERS,
MANAGE_TEMPLATES,
MANAGE_SETTINGS,
SEND_TEXTS,
SEND_EMAILS,
MANAGE_API_KEYS,
VIEW_ACTIVITY,
]
from app.enums import PermissionType
from app.models import Permission
class PermissionDAO(DAOClass):
@@ -28,7 +9,7 @@ class PermissionDAO(DAOClass):
model = Permission
def add_default_service_permissions_for_user(self, user, service):
for name in default_service_permissions:
for name in PermissionType.defaults():
permission = Permission(permission=name, user=user, service=service)
self.create_instance(permission, _commit=False)
app/dao/provider_details_dao.py
+5 -9
View File
@@ -5,13 +5,8 @@ from sqlalchemy import asc, desc, func
from app import db
from app.dao.dao_utils import autocommit
from app.models import (
SMS_TYPE,
FactBilling,
ProviderDetails,
ProviderDetailsHistory,
User,
)
from app.enums import NotificationType
from app.models import FactBilling, ProviderDetails, ProviderDetailsHistory, User
def get_provider_details_by_id(provider_details_id):
@@ -62,7 +57,8 @@ def _get_sms_providers_for_update(time_threshold):
# get current priority of both providers
q = (
ProviderDetails.query.filter(
ProviderDetails.notification_type == "sms", ProviderDetails.active
ProviderDetails.notification_type == NotificationType.SMS,
ProviderDetails.active,
)
.with_for_update()
.all()
@@ -126,7 +122,7 @@ def dao_get_provider_stats():
),
)
.filter(
FactBilling.notification_type == SMS_TYPE,
FactBilling.notification_type == NotificationType.SMS,
FactBilling.local_date >= first_day_of_the_month,
)
.group_by(FactBilling.provider)
app/dao/service_callback_api_dao.py
+6 -7
View File
@@ -2,11 +2,8 @@ from datetime import datetime
from app import create_uuid, db
from app.dao.dao_utils import autocommit, version_class
from app.models import (
COMPLAINT_CALLBACK_TYPE,
DELIVERY_STATUS_CALLBACK_TYPE,
ServiceCallbackApi,
)
from app.enums import CallbackType
from app.models import ServiceCallbackApi
@autocommit
@@ -40,13 +37,15 @@ def get_service_callback_api(service_callback_api_id, service_id):
def get_service_delivery_status_callback_api_for_service(service_id):
return ServiceCallbackApi.query.filter_by(
service_id=service_id, callback_type=DELIVERY_STATUS_CALLBACK_TYPE
service_id=service_id,
callback_type=CallbackType.DELIVERY_STATUS,
).first()
def get_service_complaint_callback_api_for_service(service_id):
return ServiceCallbackApi.query.filter_by(
service_id=service_id, callback_type=COMPLAINT_CALLBACK_TYPE
service_id=service_id,
callback_type=CallbackType.COMPLAINT,
).first()
app/dao/services_dao.py
+20 -23
View File
@@ -13,12 +13,13 @@ from app.dao.organization_dao import dao_get_organization_by_email_address
from app.dao.service_sms_sender_dao import insert_service_sms_sender
from app.dao.service_user_dao import dao_get_service_user
from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
from app.enums import (
KeyType,
NotificationStatus,
NotificationType,
ServicePermissionType,
)
from app.models import (
EMAIL_TYPE,
INTERNATIONAL_SMS_TYPE,
KEY_TYPE_TEST,
NOTIFICATION_PERMANENT_FAILURE,
SMS_TYPE,
AnnualBilling,
ApiKey,
FactBilling,
@@ -45,12 +46,6 @@ from app.utils import (
get_midnight_in_utc,
)
DEFAULT_SERVICE_PERMISSIONS = [
SMS_TYPE,
EMAIL_TYPE,
INTERNATIONAL_SMS_TYPE,
]
def dao_fetch_all_services(only_active=False):
query = Service.query.order_by(asc(Service.created_at)).options(joinedload("users"))
@@ -107,7 +102,8 @@ def dao_fetch_live_services_data():
case(
[
(
this_year_ft_billing.c.notification_type == "email",
this_year_ft_billing.c.notification_type
== NotificationType.EMAIL,
func.sum(this_year_ft_billing.c.notifications_sent),
)
],
@@ -116,7 +112,8 @@ def dao_fetch_live_services_data():
case(
[
(
this_year_ft_billing.c.notification_type == "sms",
this_year_ft_billing.c.notification_type
== NotificationType.SMS,
func.sum(this_year_ft_billing.c.notifications_sent),
)
],
@@ -278,7 +275,7 @@ def dao_create_service(
raise ValueError("Can't create a service without a user")
if service_permissions is None:
service_permissions = DEFAULT_SERVICE_PERMISSIONS
service_permissions = ServicePermissionType.defaults()
organization = dao_get_organization_by_email_address(user.email_address)
@@ -412,7 +409,7 @@ def dao_fetch_todays_stats_for_service(service_id):
)
.filter(
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST,
Notification.key_type != KeyType.TEST,
Notification.created_at >= start_date,
)
.group_by(
@@ -446,7 +443,7 @@ def dao_fetch_todays_stats_for_all_services(
)
if not include_from_test_key:
subquery = subquery.filter(Notification.key_type != KEY_TYPE_TEST)
subquery = subquery.filter(Notification.key_type != KeyType.TEST)
subquery = subquery.subquery()
@@ -517,8 +514,8 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500)
Notification.service_id == Service.id,
Notification.created_at >= start_date,
Notification.created_at <= end_date,
Notification.key_type != KEY_TYPE_TEST,
Notification.notification_type == SMS_TYPE,
Notification.key_type != KeyType.TEST,
Notification.notification_type == NotificationType.SMS,
func.substr(Notification.normalised_to, 3, 7) == "7700900",
Service.restricted == False, # noqa
Service.active == True, # noqa
@@ -541,8 +538,8 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
Notification.service_id == Service.id,
Notification.created_at >= start_date,
Notification.created_at <= end_date,
Notification.key_type != KEY_TYPE_TEST,
Notification.notification_type == SMS_TYPE,
Notification.key_type != KeyType.TEST,
Notification.notification_type == NotificationType.SMS,
Service.restricted == False, # noqa
Service.active == True, # noqa
)
@@ -569,9 +566,9 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10
Notification.service_id == Service.id,
Notification.created_at >= start_date,
Notification.created_at <= end_date,
Notification.key_type != KEY_TYPE_TEST,
Notification.notification_type == SMS_TYPE,
Notification.status == NOTIFICATION_PERMANENT_FAILURE,
Notification.key_type != KeyType.TEST,
Notification.notification_type == NotificationType.SMS,
Notification.status == NotificationStatus.PERMANENT_FAILURE,
Service.restricted == False, # noqa
Service.active == True, # noqa
)
app/dao/uploads_dao.py
+5 -13
View File
@@ -5,16 +5,8 @@ from flask import current_app
from sqlalchemy import String, and_, desc, func, literal, text
from app import db
from app.models import (
JOB_STATUS_CANCELLED,
JOB_STATUS_SCHEDULED,
LETTER_TYPE,
NOTIFICATION_CANCELLED,
Job,
Notification,
ServiceDataRetention,
Template,
)
from app.enums import JobStatus, NotificationStatus, NotificationType
from app.models import Job, Notification, ServiceDataRetention, Template
from app.utils import midnight_n_days_ago
@@ -53,7 +45,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
Job.service_id == service_id,
Job.original_file_name != current_app.config["TEST_MESSAGE_FILENAME"],
Job.original_file_name != current_app.config["ONE_OFF_MESSAGE_FILENAME"],
Job.job_status.notin_([JOB_STATUS_CANCELLED, JOB_STATUS_SCHEDULED]),
Job.job_status.notin_([JobStatus.CANCELLED, JobStatus.SCHEDULED]),
func.coalesce(Job.processing_started, Job.created_at)
>= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
]
@@ -90,9 +82,9 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
letters_query_filter = [
Notification.service_id == service_id,
Notification.notification_type == LETTER_TYPE,
Notification.notification_type == NotificationType.LETTER,
Notification.api_key_id == None, # noqa
Notification.status != NOTIFICATION_CANCELLED,
Notification.status != NotificationStatus.CANCELLED,
Template.hidden == True, # noqa
Notification.created_at
>= today - func.coalesce(ServiceDataRetention.days_of_retention, 7),
app/dao/users_dao.py
+34 -4
View File
@@ -9,8 +9,9 @@ from app import db
from app.dao.dao_utils import autocommit
from app.dao.permissions_dao import permission_dao
from app.dao.service_user_dao import dao_get_service_users_by_user_id
from app.enums import AuthType, PermissionType
from app.errors import InvalidRequest
from app.models import EMAIL_AUTH_TYPE, User, VerifyCode
from app.models import User, VerifyCode
from app.utils import escape_special_characters, get_archived_db_column_value
@@ -24,13 +25,42 @@ def create_secret_code(length=6):
return "{:0{length}d}".format(random_number, length=length)
def get_login_gov_user(login_uuid, email_address):
"""
We want to check to see if the user is registered with login.gov
If we can find the login.gov uuid in our user table, then they are.
Also, because we originally keyed off email address we might have a few
older users who registered with login.gov but we don't know what their
login.gov uuids are. Eventually the code that checks by email address
should be removed.
"""
print(User.query.filter_by(login_uuid=login_uuid).first())
user = User.query.filter_by(login_uuid=login_uuid).first()
if user:
if user.email_address != email_address:
save_user_attribute(user, {"email_address": email_address})
return user
# Remove this 1 July 2025, all users should have login.gov uuids by now
user = User.query.filter_by(email_address=email_address).first()
if user:
save_user_attribute(user, {"login_uuid": login_uuid})
return user
return None
def save_user_attribute(usr, update_dict=None):
db.session.query(User).filter_by(id=usr.id).update(update_dict or {})
db.session.commit()
def save_model_user(
user, update_dict=None, password=None, validated_email_access=False
user,
update_dict=None,
password=None,
validated_email_access=False,
):
if password:
user.password = password
@@ -171,7 +201,7 @@ def dao_archive_user(user):
user.organizations = []
user.auth_type = EMAIL_AUTH_TYPE
user.auth_type = AuthType.EMAIL
user.email_address = get_archived_db_column_value(user.email_address)
user.mobile_number = None
user.password = str(uuid.uuid4())
@@ -194,7 +224,7 @@ def user_can_be_archived(user):
return False
if not any(
"manage_settings" in user.get_permissions(service.id)
PermissionType.MANAGE_SETTINGS in user.get_permissions(service.id)
for user in other_active_users
):
# no-one else has manage settings