darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-19 14:08:47 -04:00
Code Issues Packages Projects Releases Wiki Activity

Remove letters-related code (#175)

Browse Source 
This deletes a big ol' chunk of code related to letters. It's not everything—there are still a few things that might be tied to sms/email—but it's the the heart of letters function. SMS and email function should be untouched by this.

Areas affected:

- Things obviously about letters
- PDF tasks, used for precompiling letters
- Virus scanning, used for those PDFs
- FTP, used to send letters to the printer
- Postage stuff
This commit is contained in:
Steven Reilly
2023-03-02 20:20:31 -05:00
committed by GitHub
parent b07b95f795
commit ff4190a8eb
141 changed files with 1108 additions and 12083 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
app/dao/daily_sorted_letter_dao.py
View File

@@ -1,38 +0,0 @@
from datetime import datetime
from sqlalchemy.dialects.postgresql import insert
from app import db
from app.dao.dao_utils import autocommit
from app.models import DailySortedLetter
def dao_get_daily_sorted_letter_by_billing_day(billing_day):
return DailySortedLetter.query.filter_by(
billing_day=billing_day
).first()
@autocommit
def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
'''
This uses the Postgres upsert to avoid race conditions when two threads try and insert
at the same row. The excluded object refers to values that we tried to insert but were
rejected.
http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
'''
table = DailySortedLetter.__table__
stmt = insert(table).values(
billing_day=new_daily_sorted_letter.billing_day,
file_name=new_daily_sorted_letter.file_name,
unsorted_count=new_daily_sorted_letter.unsorted_count,
sorted_count=new_daily_sorted_letter.sorted_count)
stmt = stmt.on_conflict_do_update(
index_elements=[table.c.billing_day, table.c.file_name],
set_={
'unsorted_count': stmt.excluded.unsorted_count,
'sorted_count': stmt.excluded.sorted_count,
'updated_at': datetime.utcnow()
}
)
db.session.connection().execute(stmt)

250
app/dao/fact_billing_dao.py
View File

@@ -14,17 +14,13 @@ from app.dao.date_util import (
from app.dao.organisation_dao import dao_get_organisation_live_services
from app.models import (
EMAIL_TYPE,
INTERNATIONAL_POSTAGE_TYPES,
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
LETTER_TYPE,
NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS,
NOTIFICATION_STATUS_TYPES_BILLABLE_SMS,
NOTIFICATION_STATUS_TYPES_SENT_EMAILS,
SMS_TYPE,
AnnualBilling,
FactBilling,
LetterRate,
NotificationAllTimeView,
NotificationHistory,
Organisation,
@@ -120,84 +116,6 @@ def fetch_sms_billing_for_all_services(start_date, end_date):
return query.all()
def fetch_letter_costs_and_totals_for_all_services(start_date, end_date):
query = db.session.query(
Organisation.name.label("organisation_name"),
Organisation.id.label("organisation_id"),
Service.name.label("service_name"),
Service.id.label("service_id"),
func.sum(FactBilling.notifications_sent).label("total_letters"),
func.sum(FactBilling.notifications_sent * FactBilling.rate).label("letter_cost")
).select_from(
Service
).outerjoin(
Service.organisation
).join(
FactBilling, FactBilling.service_id == Service.id,
).filter(
FactBilling.service_id == Service.id,
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == LETTER_TYPE,
).group_by(
Organisation.name,
Organisation.id,
Service.id,
Service.name,
).order_by(
Organisation.name,
Service.name
)
return query.all()
def fetch_letter_line_items_for_all_services(start_date, end_date):
formatted_postage = case(
[(FactBilling.postage.in_(INTERNATIONAL_POSTAGE_TYPES), "international")], else_=FactBilling.postage
).label("postage")
postage_order = case(
(formatted_postage == "second", 1),
(formatted_postage == "first", 2),
(formatted_postage == "international", 3),
else_=0 # assumes never get 0 as a result
)
query = db.session.query(
Organisation.name.label("organisation_name"),
Organisation.id.label("organisation_id"),
Service.name.label("service_name"),
Service.id.label("service_id"),
FactBilling.rate.label("letter_rate"),
formatted_postage,
func.sum(FactBilling.notifications_sent).label("letters_sent"),
).select_from(
Service
).outerjoin(
Service.organisation
).join(
FactBilling, FactBilling.service_id == Service.id,
).filter(
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == LETTER_TYPE,
).group_by(
Organisation.name,
Organisation.id,
Service.id,
Service.name,
FactBilling.rate,
formatted_postage
).order_by(
Organisation.name,
Service.name,
postage_order,
FactBilling.rate,
)
return query.all()
def fetch_billing_totals_for_year(service_id, year):
"""
Returns a row for each distinct rate and notification_type from ft_billing
@@ -233,7 +151,6 @@ def fetch_billing_totals_for_year(service_id, year):
for query in [
query_service_sms_usage_for_year(service_id, year).subquery(),
query_service_email_usage_for_year(service_id, year).subquery(),
query_service_letter_usage_for_year(service_id, year).subquery(),
]
]).subquery()
).order_by(
@@ -244,20 +161,19 @@ def fetch_billing_totals_for_year(service_id, year):
def fetch_monthly_billing_for_year(service_id, year):
"""
Returns a row for each distinct rate, notification_type, postage and month
Returns a row for each distinct rate, notification_type, and month
from ft_billing over the specified financial year e.g.
(
rate=0.0165,
notification_type=sms,
postage=none,
month=2022-04-01 00:00:00,
notifications_sent=123,
...
)
The "postage" field is "none" except for letters. Each subquery takes care
of anything specific to the notification type e.g. rate multipliers for SMS.
Each subquery takes care of anything specific to the notification type e.g.
rate multipliers for SMS.
Since the data in ft_billing is only refreshed once a day for all services,
we also update the table on-the-fly if we need accurate data for this year.
@@ -276,7 +192,6 @@ def fetch_monthly_billing_for_year(service_id, year):
db.session.query(
query.c.rate.label("rate"),
query.c.notification_type.label("notification_type"),
query.c.postage.label("postage"),
func.date_trunc('month', query.c.local_date).cast(Date).label("month"),
func.sum(query.c.notifications_sent).label("notifications_sent"),
@@ -287,13 +202,11 @@ def fetch_monthly_billing_for_year(service_id, year):
).group_by(
query.c.rate,
query.c.notification_type,
query.c.postage,
'month',
)
for query in [
query_service_sms_usage_for_year(service_id, year).subquery(),
query_service_email_usage_for_year(service_id, year).subquery(),
query_service_letter_usage_for_year(service_id, year).subquery(),
]
]).subquery()
).order_by(
@@ -308,7 +221,6 @@ def query_service_email_usage_for_year(service_id, year):
return db.session.query(
FactBilling.local_date,
FactBilling.postage, # should always be "none"
FactBilling.notifications_sent,
FactBilling.billable_units.label("chargeable_units"),
FactBilling.rate,
@@ -324,30 +236,6 @@ def query_service_email_usage_for_year(service_id, year):
)
def query_service_letter_usage_for_year(service_id, year):
year_start, year_end = get_financial_year_dates(year)
return db.session.query(
FactBilling.local_date,
FactBilling.postage,
FactBilling.notifications_sent,
# We can't use billable_units here as it represents the
# sheet count for letters, which is already accounted for
# in the rate. We actually charge per letter, not sheet.
FactBilling.notifications_sent.label("chargeable_units"),
FactBilling.rate,
FactBilling.notification_type,
(FactBilling.notifications_sent * FactBilling.rate).label("cost"),
literal(0).label("free_allowance_used"),
FactBilling.notifications_sent.label("charged_units"),
).filter(
FactBilling.service_id == service_id,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == LETTER_TYPE
)
def query_service_sms_usage_for_year(service_id, year):
"""
Returns rows from the ft_billing table with some calculated values like cost,
@@ -410,7 +298,6 @@ def query_service_sms_usage_for_year(service_id, year):
return db.session.query(
FactBilling.local_date,
FactBilling.postage, # should always be "none"
FactBilling.notifications_sent,
this_rows_chargeable_units.label("chargeable_units"),
FactBilling.rate,
@@ -453,7 +340,7 @@ def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=F
services = [Service.query.get(service_id)]
for service in services:
for notification_type in (SMS_TYPE, EMAIL_TYPE, LETTER_TYPE):
for notification_type in (SMS_TYPE, EMAIL_TYPE):
if (not check_permissions) or service.has_permission(notification_type):
results = _query_for_billing_data(
notification_type=notification_type,
@@ -476,8 +363,6 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
literal('ses').label('sent_by'),
literal(0).label('rate_multiplier'),
literal(False).label('international'),
literal(None).label('letter_page_count'),
literal('none').label('postage'),
literal(0).label('billable_units'),
func.count().label('notifications_sent'),
).filter(
@@ -503,8 +388,6 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
sent_by.label('sent_by'),
rate_multiplier.label('rate_multiplier'),
international.label('international'),
literal(None).label('letter_page_count'),
literal('none').label('postage'),
func.sum(NotificationAllTimeView.billable_units).label('billable_units'),
func.count().label('notifications_sent'),
).filter(
@@ -521,40 +404,9 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
international,
)
def _letter_query():
rate_multiplier = func.coalesce(NotificationAllTimeView.rate_multiplier, 1).cast(Integer)
postage = func.coalesce(NotificationAllTimeView.postage, 'none')
return db.session.query(
NotificationAllTimeView.template_id,
literal(service.crown).label('crown'),
literal(service.id).label('service_id'),
literal(notification_type).label('notification_type'),
literal('dvla').label('sent_by'),
rate_multiplier.label('rate_multiplier'),
NotificationAllTimeView.international,
NotificationAllTimeView.billable_units.label('letter_page_count'),
postage.label('postage'),
func.sum(NotificationAllTimeView.billable_units).label('billable_units'),
func.count().label('notifications_sent'),
).filter(
NotificationAllTimeView.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS),
NotificationAllTimeView.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
NotificationAllTimeView.created_at >= start_date,
NotificationAllTimeView.created_at < end_date,
NotificationAllTimeView.notification_type == notification_type,
NotificationAllTimeView.service_id == service.id
).group_by(
NotificationAllTimeView.template_id,
rate_multiplier,
NotificationAllTimeView.billable_units,
postage,
NotificationAllTimeView.international
)
query_funcs = {
SMS_TYPE: _sms_query,
EMAIL_TYPE: _email_query,
LETTER_TYPE: _letter_query
}
query = query_funcs[notification_type]()
@@ -562,9 +414,8 @@ def _query_for_billing_data(notification_type, start_date, end_date, service):
def get_rates_for_billing():
non_letter_rates = Rate.query.order_by(desc(Rate.valid_from)).all()
letter_rates = LetterRate.query.order_by(desc(LetterRate.start_date)).all()
return non_letter_rates, letter_rates
rates = Rate.query.order_by(desc(Rate.valid_from)).all()
return rates
def get_service_ids_that_need_billing_populated(start_date, end_date):
@@ -573,34 +424,20 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
).filter(
NotificationHistory.created_at >= start_date,
NotificationHistory.created_at <= end_date,
NotificationHistory.notification_type.in_([SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]),
NotificationHistory.notification_type.in_([SMS_TYPE, EMAIL_TYPE]),
NotificationHistory.billable_units != 0
).distinct().all()
def get_rate(
non_letter_rates, letter_rates, notification_type, date, crown=None, letter_page_count=None, post_class='second'
rates, notification_type, date, crown=None
):
start_of_day = get_local_midnight_in_utc(date)
if notification_type == LETTER_TYPE:
if letter_page_count == 0:
return 0
# if crown is not set default to true, this is okay because the rates are the same for both crown and non-crown.
crown = crown or True
if notification_type == SMS_TYPE:
return next(
r.rate
for r in letter_rates if (
start_of_day >= r.start_date and
crown == r.crown and
letter_page_count == r.sheet_count and
post_class == r.post_class
)
)
elif notification_type == SMS_TYPE:
return next(
r.rate
for r in non_letter_rates if (
for r in rates if (
notification_type == r.notification_type and
start_of_day >= r.valid_from
)
@@ -610,14 +447,11 @@ def get_rate(
def update_fact_billing(data, process_day):
non_letter_rates, letter_rates = get_rates_for_billing()
rate = get_rate(non_letter_rates,
letter_rates,
rates = get_rates_for_billing()
rate = get_rate(rates,
data.notification_type,
process_day,
data.crown,
data.letter_page_count,
data.postage)
data.crown)
billing_record = create_billing_record(data, rate, process_day)
table = FactBilling.__table__
@@ -638,7 +472,6 @@ def update_fact_billing(data, process_day):
billable_units=billing_record.billable_units,
notifications_sent=billing_record.notifications_sent,
rate=billing_record.rate,
postage=billing_record.postage,
)
stmt = stmt.on_conflict_do_update(
@@ -664,36 +497,10 @@ def create_billing_record(data, rate, process_day):
billable_units=data.billable_units,
notifications_sent=data.notifications_sent,
rate=rate,
postage=data.postage,
)
return billing_record
def fetch_letter_costs_for_organisation(organisation_id, start_date, end_date):
query = db.session.query(
Service.name.label("service_name"),
Service.id.label("service_id"),
func.sum(FactBilling.notifications_sent * FactBilling.rate).label("letter_cost")
).select_from(
Service
).join(
FactBilling, FactBilling.service_id == Service.id,
).filter(
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == LETTER_TYPE,
Service.organisation_id == organisation_id,
Service.restricted.is_(False)
).group_by(
Service.id,
Service.name,
).order_by(
Service.name
)
return query.all()
def fetch_email_usage_for_organisation(organisation_id, start_date, end_date):
query = db.session.query(
Service.name.label("service_name"),
@@ -840,12 +647,10 @@ def fetch_usage_year_for_organisation(organisation_id, year):
'sms_billable_units': 0,
'chargeable_billable_sms': 0,
'sms_cost': 0.0,
'letter_cost': 0.0,
'emails_sent': 0,
'active': service.active
}
sms_usages = fetch_sms_billing_for_organisation(organisation_id, year)
letter_usages = fetch_letter_costs_for_organisation(organisation_id, year_start, year_end)
email_usages = fetch_email_usage_for_organisation(organisation_id, year_start, year_end)
for usage in sms_usages:
service_with_usage[str(usage.service_id)] = {
@@ -856,12 +661,9 @@ def fetch_usage_year_for_organisation(organisation_id, year):
'sms_billable_units': usage.sms_billable_units,
'chargeable_billable_sms': usage.chargeable_billable_sms,
'sms_cost': float(usage.sms_cost),
'letter_cost': 0.0,
'emails_sent': 0,
'active': usage.active
}
for letter_usage in letter_usages:
service_with_usage[str(letter_usage.service_id)]['letter_cost'] = float(letter_usage.letter_cost)
for email_usage in email_usages:
service_with_usage[str(email_usage.service_id)]['emails_sent'] = email_usage.emails_sent
@@ -910,16 +712,6 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
(FactBilling.notification_type == EMAIL_TYPE, FactBilling.notifications_sent)
], else_=0
)).label('email_totals'),
func.sum(case(
[
(FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent)
], else_=0
)).label('letter_totals'),
func.sum(case(
[
(FactBilling.notification_type == LETTER_TYPE, FactBilling.billable_units)
], else_=0
)).label('letter_sheet_totals')
).filter(
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date
@@ -935,8 +727,6 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
func.sum(
daily_volume_stats.c.sms_fragments_times_multiplier).label('sms_chargeable_units'),
func.sum(daily_volume_stats.c.email_totals).label('email_totals'),
func.sum(daily_volume_stats.c.letter_totals).label('letter_totals'),
func.sum(daily_volume_stats.c.letter_sheet_totals).label('letter_sheet_totals')
).group_by(
daily_volume_stats.c.local_date
).order_by(
@@ -988,17 +778,6 @@ def fetch_volumes_by_service(start_date, end_date):
func.sum(case([
(FactBilling.notification_type == EMAIL_TYPE, FactBilling.notifications_sent)
], else_=0)).label('email_totals'),
func.sum(case([
(FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent)
], else_=0)).label('letter_totals'),
func.sum(case([
(FactBilling.notification_type == LETTER_TYPE, FactBilling.notifications_sent * FactBilling.rate)
], else_=0)).label("letter_cost"),
func.sum(case(
[
(FactBilling.notification_type == LETTER_TYPE, FactBilling.billable_units)
], else_=0
)).label('letter_sheet_totals')
).filter(
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date
@@ -1029,9 +808,6 @@ def fetch_volumes_by_service(start_date, end_date):
func.coalesce(func.sum(volume_stats.c.sms_fragments_times_multiplier), 0
).label("sms_chargeable_units"),
func.coalesce(func.sum(volume_stats.c.email_totals), 0).label("email_totals"),
func.coalesce(func.sum(volume_stats.c.letter_totals), 0).label("letter_totals"),
func.coalesce(func.sum(volume_stats.c.letter_cost), 0).label("letter_cost"),
func.coalesce(func.sum(volume_stats.c.letter_sheet_totals), 0).label("letter_sheet_totals")
).select_from(
Service
).outerjoin(

14
app/dao/fact_notification_status_dao.py
View File

@@ -156,7 +156,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
query = db.session.query(
*([
Template.name.label("template_name"),
Template.is_precompiled_letter,
False, # TODO: this is related to is_precompiled_letter
all_stats_table.c.template_id
] if by_template else []),
all_stats_table.c.notification_type,
@@ -168,7 +168,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
query = query.filter(all_stats_table.c.template_id == Template.id)
return query.group_by(
*([Template.name, Template.is_precompiled_letter, all_stats_table.c.template_id] if by_template else []),
*([Template.name, all_stats_table.c.template_id] if by_template else []),
all_stats_table.c.notification_type,
all_stats_table.c.status,
).all()
@@ -333,7 +333,6 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
FactNotificationStatus.template_id.label('template_id'),
Template.name.label('name'),
Template.template_type.label('template_type'),
Template.is_precompiled_letter.label('is_precompiled_letter'),
extract('month', FactNotificationStatus.local_date).label('month'),
extract('year', FactNotificationStatus.local_date).label('year'),
func.sum(FactNotificationStatus.notification_count).label('count')
@@ -349,7 +348,6 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
FactNotificationStatus.template_id,
Template.name,
Template.template_type,
Template.is_precompiled_letter,
extract('month', FactNotificationStatus.local_date).label('month'),
extract('year', FactNotificationStatus.local_date).label('year'),
).order_by(
@@ -366,7 +364,6 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
Notification.template_id.label('template_id'),
Template.name.label('name'),
Template.template_type.label('template_type'),
Template.is_precompiled_letter.label('is_precompiled_letter'),
extract('month', month).label('month'),
extract('year', month).label('year'),
func.count().label('count')
@@ -389,7 +386,6 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
query = db.session.query(
all_stats_table.c.template_id,
all_stats_table.c.name,
all_stats_table.c.is_precompiled_letter,
all_stats_table.c.template_type,
func.cast(all_stats_table.c.month, Integer).label('month'),
func.cast(all_stats_table.c.year, Integer).label('year'),
@@ -397,7 +393,6 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
).group_by(
all_stats_table.c.template_id,
all_stats_table.c.name,
all_stats_table.c.is_precompiled_letter,
all_stats_table.c.template_type,
all_stats_table.c.month,
all_stats_table.c.year,
@@ -424,11 +419,6 @@ def get_total_notifications_for_date_range(start_date, end_date):
(FactNotificationStatus.notification_type == 'sms', FactNotificationStatus.notification_count)
],
else_=0)).label('sms'),
func.sum(case(
[
(FactNotificationStatus.notification_type == 'letter', FactNotificationStatus.notification_count)
],
else_=0)).label('letters'),
).filter(
FactNotificationStatus.key_type != KEY_TYPE_TEST,
).group_by(

44
app/dao/jobs_dao.py
View File

@@ -2,23 +2,13 @@ import uuid
from datetime import datetime, timedelta
from flask import current_app
from notifications_utils.letter_timings import (
CANCELLABLE_JOB_LETTER_STATUSES,
letter_can_be_cancelled,
)
from sqlalchemy import and_, asc, desc, func
from app import db
from app.dao.dao_utils import autocommit
from app.dao.templates_dao import dao_get_template_by_id
from app.models import (
JOB_STATUS_CANCELLED,
JOB_STATUS_FINISHED,
JOB_STATUS_PENDING,
JOB_STATUS_SCHEDULED,
LETTER_TYPE,
NOTIFICATION_CANCELLED,
NOTIFICATION_CREATED,
FactNotificationStatus,
Job,
Notification,
@@ -183,40 +173,6 @@ def dao_get_jobs_older_than_data_retention(notification_types):
return jobs
@autocommit
def dao_cancel_letter_job(job):
number_of_notifications_cancelled = Notification.query.filter(
Notification.job_id == job.id
).update({'status': NOTIFICATION_CANCELLED,
'updated_at': datetime.utcnow(),
'billable_units': 0})
job.job_status = JOB_STATUS_CANCELLED
dao_update_job(job)
return number_of_notifications_cancelled
def can_letter_job_be_cancelled(job):
template = dao_get_template_by_id(job.template_id)
if template.template_type != LETTER_TYPE:
return False, "Only letter jobs can be cancelled through this endpoint. This is not a letter job."
notifications = Notification.query.filter(
Notification.job_id == job.id
).all()
count_notifications = len(notifications)
if job.job_status != JOB_STATUS_FINISHED or count_notifications != job.notification_count:
return False, "We are still processing these letters, please try again in a minute."
count_cancellable_notifications = len([
n for n in notifications if n.status in CANCELLABLE_JOB_LETTER_STATUSES
])
if count_cancellable_notifications != job.notification_count or not letter_can_be_cancelled(
NOTIFICATION_CREATED, job.created_at
):
return False, "It’s too late to cancel sending, these letters have already been sent."
return True, None
def find_jobs_with_missing_rows():
# Jobs can be a maximum of 100,000 rows. It typically takes 10 minutes to create all those notifications.
# Using 20 minutes as a condition seems reasonable.

29
app/dao/letter_branding_dao.py
View File

@@ -1,29 +0,0 @@
from app import db
from app.dao.dao_utils import autocommit
from app.models import LetterBranding
def dao_get_letter_branding_by_id(letter_branding_id):
return LetterBranding.query.filter(LetterBranding.id == letter_branding_id).one()
def dao_get_letter_branding_by_name(letter_branding_name):
return LetterBranding.query.filter_by(name=letter_branding_name).first()
def dao_get_all_letter_branding():
return LetterBranding.query.order_by(LetterBranding.name).all()
@autocommit
def dao_create_letter_branding(letter_branding):
db.session.add(letter_branding)
@autocommit
def dao_update_letter_branding(letter_branding_id, **kwargs):
letter_branding = LetterBranding.query.get(letter_branding_id)
for key, value in kwargs.items():
setattr(letter_branding, key, value or None)
db.session.add(letter_branding)
return letter_branding

163
app/dao/notifications_dao.py
View File

@@ -1,6 +1,5 @@
from datetime import datetime, timedelta
from botocore.exceptions import ClientError
from flask import current_app
from notifications_utils.international_billing_rates import (
INTERNATIONAL_BILLING_RATES,
@@ -10,10 +9,6 @@ from notifications_utils.recipients import (
try_validate_and_format_phone_number,
validate_and_format_email_address,
)
from notifications_utils.timezones import (
convert_local_timezone_to_utc,
convert_utc_to_local_timezone,
)
from sqlalchemy import asc, desc, func, or_, union
from sqlalchemy.orm import joinedload
from sqlalchemy.orm.exc import NoResultFound
@@ -23,19 +18,15 @@ from werkzeug.datastructures import MultiDict
from app import create_uuid, db
from app.dao.dao_utils import autocommit
from app.letters.utils import LetterPDFNotFound, find_letter_pdf_in_s3
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
KEY_TYPE_TEST,
LETTER_TYPE,
NOTIFICATION_CREATED,
NOTIFICATION_PENDING,
NOTIFICATION_PENDING_VIRUS_CHECK,
NOTIFICATION_PERMANENT_FAILURE,
NOTIFICATION_SENDING,
NOTIFICATION_SENT,
NOTIFICATION_STATUS_TYPES_COMPLETED,
NOTIFICATION_TEMPORARY_FAILURE,
SMS_TYPE,
FactNotificationStatus,
@@ -140,7 +131,7 @@ def update_notification_status_by_id(notification_id, status, sent_by=None):
@autocommit
def update_notification_status_by_reference(reference, status):
# this is used to update letters and emails
# this is used to update emails
notification = Notification.query.filter(Notification.reference == reference).first()
if not notification:
@@ -304,7 +295,7 @@ def insert_notification_history_delete_notifications(
SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
client_reference, international, phone_prefix, rate_multiplier, notification_status,
created_by_id, postage, document_download_count
created_by_id, document_download_count
FROM notifications
WHERE service_id = :service_id
AND notification_type = :notification_type
@@ -312,20 +303,6 @@ def insert_notification_history_delete_notifications(
AND key_type in ('normal', 'team')
limit :qry_limit
"""
select_into_temp_table_for_letters = """
CREATE TEMP TABLE NOTIFICATION_ARCHIVE ON COMMIT DROP AS
SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
client_reference, international, phone_prefix, rate_multiplier, notification_status,
created_by_id, postage, document_download_count
FROM notifications
WHERE service_id = :service_id
AND notification_type = :notification_type
AND created_at < :timestamp_to_delete_backwards_from
AND notification_status NOT IN ('pending-virus-check', 'created', 'sending')
AND key_type in ('normal', 'team')
limit :qry_limit
"""
# Insert into NotificationHistory if the row already exists do nothing.
insert_query = """
insert into notification_history
@@ -344,8 +321,7 @@ def insert_notification_history_delete_notifications(
"qry_limit": qry_limit
}
select_to_use = select_into_temp_table_for_letters if notification_type == 'letter' else select_into_temp_table
db.session.execute(select_to_use, input_params)
db.session.execute(select_into_temp_table, input_params)
result = db.session.execute("select count(*) from NOTIFICATION_ARCHIVE").fetchone()[0]
@@ -363,10 +339,6 @@ def move_notifications_to_notification_history(
qry_limit=50000
):
deleted = 0
if notification_type == LETTER_TYPE:
_delete_letters_from_s3(
notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit
)
delete_count_per_call = 1
while delete_count_per_call > 0:
delete_count_per_call = insert_notification_history_delete_notifications(
@@ -389,32 +361,6 @@ def move_notifications_to_notification_history(
return deleted
def _delete_letters_from_s3(
notification_type, service_id, date_to_delete_from, query_limit
):
letters_to_delete_from_s3 = db.session.query(
Notification
).filter(
Notification.notification_type == notification_type,
Notification.created_at < date_to_delete_from,
Notification.service_id == service_id,
# although letters in non completed statuses do have PDFs in s3, they do not exist in the
# production-letters-pdf bucket as they never made it that far so we do not try and delete
# them from it
Notification.status.in_(NOTIFICATION_STATUS_TYPES_COMPLETED)
).limit(query_limit).all()
for letter in letters_to_delete_from_s3:
try:
letter_pdf = find_letter_pdf_in_s3(letter)
letter_pdf.delete()
except ClientError:
current_app.logger.exception(
"Error deleting S3 object for letter: {}".format(letter.id))
except LetterPDFNotFound:
current_app.logger.warning(
"No S3 object to delete for letter: {}".format(letter.id))
@autocommit
def dao_delete_notifications_by_id(notification_id):
db.session.query(Notification).filter(
@@ -493,10 +439,8 @@ def dao_get_notifications_by_recipient_or_reference(
except InvalidEmailError:
normalised = search_term.lower()
elif notification_type in {LETTER_TYPE, None}:
# For letters, we store the address without spaces, so we need
# to removes spaces from the search term to match. We also do
# this when a notification type isn’t provided (this will
elif notification_type is None:
# This happens when a notification type isn’t provided (this will
# happen if a user doesn’t have permission to see the dashboard)
# because email addresses and phone numbers will never be stored
# with spaces either.
@@ -504,7 +448,7 @@ def dao_get_notifications_by_recipient_or_reference(
else:
raise TypeError(
f'Notification type must be {EMAIL_TYPE}, {SMS_TYPE}, {LETTER_TYPE} or None'
f'Notification type must be {EMAIL_TYPE}, {SMS_TYPE}, or None'
)
normalised = escape_special_characters(normalised)
@@ -559,8 +503,7 @@ def dao_get_notifications_processing_time_stats(start_date, end_date):
created_at > 'START DATE' AND
created_at < 'END DATE' AND
api_key_id IS NOT NULL AND
key_type != 'test' AND
notification_type != 'letter';
key_type != 'test';
"""
under_10_secs = Notification.sent_at - Notification.created_at <= timedelta(seconds=10)
sum_column = functions.coalesce(functions.sum(
@@ -580,7 +523,6 @@ def dao_get_notifications_processing_time_stats(start_date, end_date):
Notification.created_at < end_date,
Notification.api_key_id.isnot(None),
Notification.key_type != KEY_TYPE_TEST,
Notification.notification_type != LETTER_TYPE
).one()
@@ -605,97 +547,6 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
return notifications
def dao_get_letters_to_be_printed(print_run_deadline, postage, query_limit=10000):
"""
Return all letters created before the print run deadline that have not yet been sent. This yields in batches of 10k
to prevent the query taking too long and eating up too much memory. As each 10k batch is yielded, the
get_key_and_size_of_letters_to_be_sent_to_print function will go and fetch the s3 data, andhese start sending off
tasks to the notify-ftp app to send them.
CAUTION! Modify this query with caution. Modifying filters etc is fine, but if we join onto another table, then
there may be undefined behaviour. Essentially we need each ORM object returned for each row to be unique,
and we should avoid modifying state of returned objects.
For more reading:
https://docs.sqlalchemy.org/en/13/orm/query.html?highlight=yield_per#sqlalchemy.orm.query.Query.yield_per
https://www.mail-archive.com/sqlalchemy@googlegroups.com/msg12443.html
"""
notifications = Notification.query.filter(
Notification.created_at < convert_local_timezone_to_utc(print_run_deadline),
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_CREATED,
Notification.key_type == KEY_TYPE_NORMAL,
Notification.postage == postage,
Notification.billable_units > 0
).order_by(
Notification.service_id,
Notification.created_at
).yield_per(query_limit)
return notifications
def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline):
notifications = db.session.query(
func.count(Notification.id).label('letters_count'),
func.sum(Notification.billable_units).label('sheets_count'),
Notification.postage
).filter(
Notification.created_at < convert_local_timezone_to_utc(print_run_deadline),
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_CREATED,
Notification.key_type == KEY_TYPE_NORMAL,
Notification.billable_units > 0
).group_by(
Notification.postage
).order_by(
Notification.postage
).all()
return notifications
def dao_old_letters_with_created_status():
yesterday_bst = convert_utc_to_local_timezone(datetime.utcnow()) - timedelta(days=1)
last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0)
notifications = Notification.query.filter(
Notification.created_at < convert_local_timezone_to_utc(last_processing_deadline),
Notification.notification_type == LETTER_TYPE,
Notification.status == NOTIFICATION_CREATED
).order_by(
Notification.created_at
).all()
return notifications
def letters_missing_from_sending_bucket(seconds_to_subtract):
older_than_date = datetime.utcnow() - timedelta(seconds=seconds_to_subtract)
# We expect letters to have a `created` status, updated_at timestamp and billable units greater than zero.
notifications = Notification.query.filter(
Notification.billable_units == 0,
Notification.updated_at == None, # noqa
Notification.status == NOTIFICATION_CREATED,
Notification.created_at <= older_than_date,
Notification.notification_type == LETTER_TYPE,
Notification.key_type == KEY_TYPE_NORMAL
).order_by(
Notification.created_at
).all()
return notifications
def dao_precompiled_letters_still_pending_virus_check():
ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400)
notifications = Notification.query.filter(
Notification.created_at < ninety_minutes_ago,
Notification.status == NOTIFICATION_PENDING_VIRUS_CHECK
).order_by(
Notification.created_at
).all()
return notifications
def _duplicate_update_warning(notification, status):
current_app.logger.info(
(

3
app/dao/organisation_dao.py
View File

@@ -89,9 +89,6 @@ def dao_update_organisation(organisation_id, **kwargs):
if 'email_branding_id' in kwargs:
_update_organisation_services(organisation, 'email_branding')
if 'letter_branding_id' in kwargs:
_update_organisation_services(organisation, 'letter_branding')
return num_updated

2
app/dao/permissions_dao.py
View File

@@ -6,7 +6,6 @@ from app.models import (
MANAGE_TEMPLATES,
MANAGE_USERS,
SEND_EMAILS,
SEND_LETTERS,
SEND_TEXTS,
VIEW_ACTIVITY,
Permission,
@@ -19,7 +18,6 @@ default_service_permissions = [
MANAGE_SETTINGS,
SEND_TEXTS,
SEND_EMAILS,
SEND_LETTERS,
MANAGE_API_KEYS,
VIEW_ACTIVITY]

118
app/dao/returned_letters_dao.py
View File

@@ -1,118 +0,0 @@
from datetime import datetime
from sqlalchemy import desc, func
from sqlalchemy.dialects.postgresql import insert
from app import db
from app.dao.dao_utils import autocommit
from app.models import (
Job,
Notification,
NotificationHistory,
ReturnedLetter,
Template,
User,
)
from app.utils import midnight_n_days_ago
def _get_notification_ids_for_references(references):
notification_ids = db.session.query(Notification.id, Notification.service_id).filter(
Notification.reference.in_(references)
).all()
notification_history_ids = db.session.query(NotificationHistory.id, NotificationHistory.service_id).filter(
NotificationHistory.reference.in_(references)
).all()
return notification_ids + notification_history_ids
@autocommit
def insert_or_update_returned_letters(references):
data = _get_notification_ids_for_references(references)
for row in data:
table = ReturnedLetter.__table__
stmt = insert(table).values(
reported_at=datetime.utcnow().date(),
service_id=row.service_id,
notification_id=row.id,
created_at=datetime.utcnow()
)
stmt = stmt.on_conflict_do_update(
index_elements=[table.c.notification_id],
set_={
'reported_at': datetime.utcnow().date(),
'updated_at': datetime.utcnow()
}
)
db.session.connection().execute(stmt)
def fetch_recent_returned_letter_count(service_id):
return db.session.query(
func.count(ReturnedLetter.notification_id).label('returned_letter_count'),
).filter(
ReturnedLetter.service_id == service_id,
ReturnedLetter.reported_at > midnight_n_days_ago(7),
).one()
def fetch_most_recent_returned_letter(service_id):
return db.session.query(
ReturnedLetter.reported_at,
).filter(
ReturnedLetter.service_id == service_id,
).order_by(
desc(ReturnedLetter.reported_at)
).first()
def fetch_returned_letter_summary(service_id):
return db.session.query(
func.count(ReturnedLetter.notification_id).label('returned_letter_count'),
ReturnedLetter.reported_at
).filter(
ReturnedLetter.service_id == service_id,
).group_by(
ReturnedLetter.reported_at
).order_by(
desc(ReturnedLetter.reported_at)
).all()
def fetch_returned_letters(service_id, report_date):
results = []
for table in [Notification, NotificationHistory]:
query = db.session.query(
ReturnedLetter.notification_id,
ReturnedLetter.reported_at,
table.client_reference,
table.created_at,
Template.name.label('template_name'),
table.template_id,
table.template_version,
Template.hidden,
table.api_key_id,
table.created_by_id,
User.name.label('user_name'),
User.email_address,
Job.original_file_name,
(table.job_row_number + 1).label('job_row_number') # row numbers start at 0
).outerjoin(
User, table.created_by_id == User.id
).outerjoin(
Job, table.job_id == Job.id
).filter(
ReturnedLetter.service_id == service_id,
ReturnedLetter.reported_at == report_date,
ReturnedLetter.notification_id == table.id,
table.template_id == Template.id
).order_by(
desc(ReturnedLetter.reported_at), desc(table.created_at)
)
results = results + query.all()
results = sorted(results, key=lambda i: i.created_at, reverse=True)
return results

105
app/dao/service_letter_contact_dao.py
View File

@@ -1,105 +0,0 @@
from sqlalchemy import desc
from app import db
from app.dao.dao_utils import autocommit
from app.models import ServiceLetterContact, Template
def dao_get_letter_contacts_by_service_id(service_id):
letter_contacts = db.session.query(
ServiceLetterContact
).filter(
ServiceLetterContact.service_id == service_id,
ServiceLetterContact.archived == False # noqa
).order_by(
desc(ServiceLetterContact.is_default),
desc(ServiceLetterContact.created_at)
).all()
return letter_contacts
def dao_get_letter_contact_by_id(service_id, letter_contact_id):
letter_contact = db.session.query(
ServiceLetterContact
).filter(
ServiceLetterContact.service_id == service_id,
ServiceLetterContact.id == letter_contact_id,
ServiceLetterContact.archived == False # noqa
).one()
return letter_contact
@autocommit
def add_letter_contact_for_service(service_id, contact_block, is_default):
old_default = _get_existing_default(service_id)
if is_default:
_reset_old_default_to_false(old_default)
new_letter_contact = ServiceLetterContact(
service_id=service_id,
contact_block=contact_block,
is_default=is_default
)
db.session.add(new_letter_contact)
return new_letter_contact
@autocommit
def update_letter_contact(service_id, letter_contact_id, contact_block, is_default):
old_default = _get_existing_default(service_id)
# if we want to make this the default, ensure there are no other existing defaults
if is_default:
_reset_old_default_to_false(old_default)
letter_contact_update = ServiceLetterContact.query.get(letter_contact_id)
letter_contact_update.contact_block = contact_block
letter_contact_update.is_default = is_default
db.session.add(letter_contact_update)
return letter_contact_update
@autocommit
def archive_letter_contact(service_id, letter_contact_id):
letter_contact_to_archive = ServiceLetterContact.query.filter_by(
id=letter_contact_id,
service_id=service_id
).one()
Template.query.filter_by(
service_letter_contact_id=letter_contact_id
).update({
'service_letter_contact_id': None
})
letter_contact_to_archive.archived = True
db.session.add(letter_contact_to_archive)
return letter_contact_to_archive
def _get_existing_default(service_id):
old_defaults = [
x for x
in dao_get_letter_contacts_by_service_id(service_id=service_id)
if x.is_default
]
if len(old_defaults) == 0:
return None
if len(old_defaults) == 1:
return old_defaults[0]
raise Exception(
"There should only be one default letter contact for each service. Service {} has {}".format(
service_id,
len(old_defaults)
)
)
def _reset_old_default_to_false(old_default):
if old_default:
old_default.is_default = False
db.session.add(old_default)

17
app/dao/services_dao.py
View File

@@ -15,13 +15,10 @@ from app.dao.service_user_dao import dao_get_service_user
from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
from app.models import (
EMAIL_TYPE,
INTERNATIONAL_LETTERS,
INTERNATIONAL_SMS_TYPE,
KEY_TYPE_TEST,
LETTER_TYPE,
NOTIFICATION_PERMANENT_FAILURE,
SMS_TYPE,
UPLOAD_LETTERS,
AnnualBilling,
ApiKey,
FactBilling,
@@ -35,7 +32,6 @@ from app.models import (
Service,
ServiceContactList,
ServiceEmailReplyTo,
ServiceLetterContact,
ServicePermission,
ServiceSmsSender,
Template,
@@ -53,10 +49,7 @@ from app.utils import (
DEFAULT_SERVICE_PERMISSIONS = [
SMS_TYPE,
EMAIL_TYPE,
LETTER_TYPE,
INTERNATIONAL_SMS_TYPE,
UPLOAD_LETTERS,
INTERNATIONAL_LETTERS,
]
@@ -113,16 +106,12 @@ def dao_fetch_live_services_data():
Service.go_live_at.label("live_date"),
Service.volume_sms.label('sms_volume_intent'),
Service.volume_email.label('email_volume_intent'),
Service.volume_letter.label('letter_volume_intent'),
case([
(this_year_ft_billing.c.notification_type == 'email', func.sum(this_year_ft_billing.c.notifications_sent))
], else_=0).label("email_totals"),
case([
(this_year_ft_billing.c.notification_type == 'sms', func.sum(this_year_ft_billing.c.notifications_sent))
], else_=0).label("sms_totals"),
case([
(this_year_ft_billing.c.notification_type == 'letter', func.sum(this_year_ft_billing.c.notifications_sent))
], else_=0).label("letter_totals"),
AnnualBilling.free_sms_fragment_limit,
).join(
Service.annual_billing
@@ -156,7 +145,6 @@ def dao_fetch_live_services_data():
Service.go_live_at,
Service.volume_sms,
Service.volume_email,
Service.volume_letter,
this_year_ft_billing.c.notification_type,
AnnualBilling.free_sms_fragment_limit,
).order_by(
@@ -169,7 +157,6 @@ def dao_fetch_live_services_data():
if existing_service is not None:
existing_service["email_totals"] += row.email_totals
existing_service["sms_totals"] += row.sms_totals
existing_service["letter_totals"] += row.letter_totals
else:
results.append(row._asdict())
return results
@@ -315,9 +302,6 @@ def dao_create_service(
if organisation.email_branding:
service.email_branding = organisation.email_branding
if organisation.letter_branding:
service.letter_branding = organisation.letter_branding
if organisation:
service.crown = organisation.crown
service.count_as_live = not user.platform_admin
@@ -378,7 +362,6 @@ def delete_service_and_all_associated_db_objects(service):
_delete_commit(ServiceSmsSender.query.filter_by(service=service))
_delete_commit(ServiceEmailReplyTo.query.filter_by(service=service))
_delete_commit(ServiceLetterContact.query.filter_by(service=service))
_delete_commit(ServiceContactList.query.filter_by(service=service))
_delete_commit(InvitedUser.query.filter_by(service=service))
_delete_commit(Permission.query.filter_by(service=service))

66
app/dao/templates_dao.py
View File

@@ -1,19 +1,11 @@
import uuid
from datetime import datetime
from flask import current_app
from sqlalchemy import asc, desc
from app import db
from app.dao.dao_utils import VersionOptions, autocommit, version_class
from app.dao.users_dao import get_user_by_id
from app.models import (
LETTER_TYPE,
SECOND_CLASS,
Template,
TemplateHistory,
TemplateRedacted,
)
from app.models import Template, TemplateHistory, TemplateRedacted
@autocommit
@@ -46,37 +38,6 @@ def dao_update_template(template):
db.session.add(template)
@autocommit
def dao_update_template_reply_to(template_id, reply_to):
Template.query.filter_by(id=template_id).update(
{"service_letter_contact_id": reply_to,
"updated_at": datetime.utcnow(),
"version": Template.version + 1,
}
)
template = Template.query.filter_by(id=template_id).one()
history = TemplateHistory(**
{
"id": template.id,
"name": template.name,
"template_type": template.template_type,
"created_at": template.created_at,
"updated_at": template.updated_at,
"content": template.content,
"service_id": template.service_id,
"subject": template.subject,
"postage": template.postage,
"created_by_id": template.created_by_id,
"version": template.version,
"archived": template.archived,
"process_type": template.process_type,
"service_letter_contact_id": template.service_letter_contact_id,
})
db.session.add(history)
return template
@autocommit
def dao_redact_template(template, user_id):
template.template_redacted.redact_personalisation = True
@@ -132,28 +93,3 @@ def dao_get_template_versions(service_id, template_id):
).order_by(
desc(TemplateHistory.version)
).all()
def get_precompiled_letter_template(service_id):
template = Template.query.filter_by(
service_id=service_id,
template_type=LETTER_TYPE,
hidden=True
).first()
if template is not None:
return template
template = Template(
name='Pre-compiled PDF',
created_by=get_user_by_id(current_app.config['NOTIFY_USER_ID']),
service_id=service_id,
template_type=LETTER_TYPE,
hidden=True,
subject='Pre-compiled PDF',
content='',
postage=SECOND_CLASS
)
dao_create_template(template)
return template

20
app/dao/uploads_dao.py
View File

@@ -129,23 +129,3 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size
).order_by(
desc("processing_started"), desc("created_at")
).paginate(page=page, per_page=page_size)
def dao_get_uploaded_letters_by_print_date(service_id, letter_print_date, page=1, page_size=50):
return db.session.query(
Notification,
).join(
Template, Notification.template_id == Template.id
).filter(
Notification.service_id == service_id,
Notification.notification_type == LETTER_TYPE,
Notification.api_key_id.is_(None),
Notification.status != NOTIFICATION_CANCELLED,
Template.hidden.is_(True),
_get_printing_day(Notification.created_at) == letter_print_date.date(),
).order_by(
desc(Notification.created_at)
).paginate(
page=page,
per_page=page_size,
)