darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-23 15:56:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

migrate bst_date to local_date

Browse Source
This commit is contained in:
stvnrlly
2022-11-21 11:49:59 -05:00
parent 99de747a36
commit 9e7ee1c0f8
39 changed files with 315 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
app/dao/fact_billing_dao.py
View File

@@ -51,8 +51,8 @@ def fetch_sms_free_allowance_remainder_until_date(end_date):
# free_sms_fragment_limit)
FactBilling, and_(
AnnualBilling.service_id == FactBilling.service_id,
FactBilling.bst_date >= start_of_year,
FactBilling.bst_date < end_date,
FactBilling.local_date >= start_of_year,
FactBilling.local_date < end_date,
FactBilling.notification_type == SMS_TYPE,
)
).filter(
@@ -101,8 +101,8 @@ def fetch_sms_billing_for_all_services(start_date, end_date):
).join(
FactBilling, FactBilling.service_id == Service.id,
).filter(
FactBilling.bst_date >= start_date,
FactBilling.bst_date <= end_date,
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == SMS_TYPE,
).group_by(
Organisation.name,
@@ -136,8 +136,8 @@ def fetch_letter_costs_and_totals_for_all_services(start_date, end_date):
FactBilling, FactBilling.service_id == Service.id,
).filter(
FactBilling.service_id == Service.id,
FactBilling.bst_date >= start_date,
FactBilling.bst_date <= end_date,
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == LETTER_TYPE,
).group_by(
Organisation.name,
@@ -179,8 +179,8 @@ def fetch_letter_line_items_for_all_services(start_date, end_date):
).join(
FactBilling, FactBilling.service_id == Service.id,
).filter(
FactBilling.bst_date >= start_date,
FactBilling.bst_date <= end_date,
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == LETTER_TYPE,
).group_by(
Organisation.name,
@@ -277,7 +277,7 @@ def fetch_monthly_billing_for_year(service_id, year):
query.c.rate.label("rate"),
query.c.notification_type.label("notification_type"),
query.c.postage.label("postage"),
func.date_trunc('month', query.c.bst_date).cast(Date).label("month"),
func.date_trunc('month', query.c.local_date).cast(Date).label("month"),
func.sum(query.c.notifications_sent).label("notifications_sent"),
func.sum(query.c.chargeable_units).label("chargeable_units"),
@@ -307,7 +307,7 @@ def query_service_email_usage_for_year(service_id, year):
year_start, year_end = get_financial_year_dates(year)
return db.session.query(
FactBilling.bst_date,
FactBilling.local_date,
FactBilling.postage, # should always be "none"
FactBilling.notifications_sent,
FactBilling.billable_units.label("chargeable_units"),
@@ -318,8 +318,8 @@ def query_service_email_usage_for_year(service_id, year):
FactBilling.billable_units.label("charged_units"),
).filter(
FactBilling.service_id == service_id,
FactBilling.bst_date >= year_start,
FactBilling.bst_date <= year_end,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == EMAIL_TYPE
)
@@ -328,7 +328,7 @@ def query_service_letter_usage_for_year(service_id, year):
year_start, year_end = get_financial_year_dates(year)
return db.session.query(
FactBilling.bst_date,
FactBilling.local_date,
FactBilling.postage,
FactBilling.notifications_sent,
# We can't use billable_units here as it represents the
@@ -342,8 +342,8 @@ def query_service_letter_usage_for_year(service_id, year):
FactBilling.notifications_sent.label("charged_units"),
).filter(
FactBilling.service_id == service_id,
FactBilling.bst_date >= year_start,
FactBilling.bst_date <= year_end,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == LETTER_TYPE
)
@@ -354,7 +354,7 @@ def query_service_sms_usage_for_year(service_id, year):
incorporating the SMS free allowance e.g.
(
bst_date=2022-04-27,
local_date=2022-04-27,
notifications_sent=12,
chargeable_units=12,
rate=0.0165,
@@ -365,7 +365,7 @@ def query_service_sms_usage_for_year(service_id, year):
)
In order to calculate how much free allowance is left, we need to work out
how much was used for previous bst_dates - cumulative_chargeable_units -
how much was used for previous local_dates - cumulative_chargeable_units -
which we then subtract from the free allowance for the year.
cumulative_chargeable_units is calculated using a "window" clause, which has
@@ -375,7 +375,7 @@ def query_service_sms_usage_for_year(service_id, year):
https://www.postgresql.org/docs/current/tutorial-window.html
ASSUMPTION: rates always change at midnight i.e. there can only be one rate
on a given bst_date. This means we don't need to worry about how to assign
on a given local_date. This means we don't need to worry about how to assign
free allowance if it happens to run out when a rate changes.
"""
year_start, year_end = get_financial_year_dates(year)
@@ -388,7 +388,7 @@ def query_service_sms_usage_for_year(service_id, year):
chargeable_units_used_before_this_row = func.coalesce(
func.sum(this_rows_chargeable_units).over(
# order is "ASC" by default
order_by=[FactBilling.bst_date],
order_by=[FactBilling.local_date],
# first row to previous row
rows=(None, -1)
).cast(Integer),
@@ -409,7 +409,7 @@ def query_service_sms_usage_for_year(service_id, year):
free_allowance_used = func.least(remaining_free_allowance_before_this_row, this_rows_chargeable_units)
return db.session.query(
FactBilling.bst_date,
FactBilling.local_date,
FactBilling.postage, # should always be "none"
FactBilling.notifications_sent,
this_rows_chargeable_units.label("chargeable_units"),
@@ -423,8 +423,8 @@ def query_service_sms_usage_for_year(service_id, year):
AnnualBilling.service_id == service_id
).filter(
FactBilling.service_id == service_id,
FactBilling.bst_date >= year_start,
FactBilling.bst_date <= year_end,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == SMS_TYPE,
AnnualBilling.financial_year_start == year,
)
@@ -432,12 +432,12 @@ def query_service_sms_usage_for_year(service_id, year):
def delete_billing_data_for_service_for_day(process_day, service_id):
"""
Delete all ft_billing data for a given service on a given bst_date
Delete all ft_billing data for a given service on a given local_date
Returns how many rows were deleted
"""
return FactBilling.query.filter(
FactBilling.bst_date == process_day,
FactBilling.local_date == process_day,
FactBilling.service_id == service_id
).delete()
@@ -628,7 +628,7 @@ def update_fact_billing(data, process_day):
http://docs.sqlalchemy.org/en/latest/dialects/postgresql.html#insert-on-conflict-upsert
'''
stmt = insert(table).values(
bst_date=billing_record.bst_date,
local_date=billing_record.local_date,
template_id=billing_record.template_id,
service_id=billing_record.service_id,
provider=billing_record.provider,
@@ -654,7 +654,7 @@ def update_fact_billing(data, process_day):
def create_billing_record(data, rate, process_day):
billing_record = FactBilling(
bst_date=process_day,
local_date=process_day,
template_id=data.template_id,
service_id=data.service_id,
notification_type=data.notification_type,
@@ -679,8 +679,8 @@ def fetch_letter_costs_for_organisation(organisation_id, start_date, end_date):
).join(
FactBilling, FactBilling.service_id == Service.id,
).filter(
FactBilling.bst_date >= start_date,
FactBilling.bst_date <= end_date,
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == LETTER_TYPE,
Service.organisation_id == organisation_id,
Service.restricted.is_(False)
@@ -704,8 +704,8 @@ def fetch_email_usage_for_organisation(organisation_id, start_date, end_date):
).join(
FactBilling, FactBilling.service_id == Service.id,
).filter(
FactBilling.bst_date >= start_date,
FactBilling.bst_date <= end_date,
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
FactBilling.notification_type == EMAIL_TYPE,
Service.organisation_id == organisation_id,
Service.restricted.is_(False)
@@ -773,7 +773,7 @@ def query_organisation_sms_usage_for_year(organisation_id, year):
chargeable_units_used_before_this_row = func.coalesce(
func.sum(this_rows_chargeable_units).over(
# order is "ASC" by default
order_by=[FactBilling.bst_date],
order_by=[FactBilling.local_date],
# partition by service id
partition_by=FactBilling.service_id,
@@ -797,7 +797,7 @@ def query_organisation_sms_usage_for_year(organisation_id, year):
return db.session.query(
Service.id.label('service_id'),
FactBilling.bst_date,
FactBilling.local_date,
this_rows_chargeable_units.label("chargeable_units"),
(charged_units * FactBilling.rate).label("cost"),
charged_units.label("charged_units"),
@@ -808,8 +808,8 @@ def query_organisation_sms_usage_for_year(organisation_id, year):
FactBilling,
and_(
Service.id == FactBilling.service_id,
FactBilling.bst_date >= year_start,
FactBilling.bst_date <= year_end,
FactBilling.local_date >= year_start,
FactBilling.local_date <= year_end,
FactBilling.notification_type == SMS_TYPE,
)
).filter(
@@ -889,7 +889,7 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
# query to return the total notifications sent per day for each channel. NB start and end dates are inclusive
daily_volume_stats = db.session.query(
FactBilling.bst_date,
FactBilling.local_date,
func.sum(case(
[
(FactBilling.notification_type == SMS_TYPE, FactBilling.notifications_sent)
@@ -921,15 +921,15 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
], else_=0
)).label('letter_sheet_totals')
).filter(
FactBilling.bst_date >= start_date,
FactBilling.bst_date <= end_date
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date
).group_by(
FactBilling.bst_date,
FactBilling.local_date,
FactBilling.notification_type
).subquery()
aggregated_totals = db.session.query(
daily_volume_stats.c.bst_date.cast(db.Text).label('bst_date'),
daily_volume_stats.c.local_date.cast(db.Text).label('local_date'),
func.sum(daily_volume_stats.c.sms_totals).label('sms_totals'),
func.sum(daily_volume_stats.c.sms_fragment_totals).label('sms_fragment_totals'),
func.sum(
@@ -938,9 +938,9 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
func.sum(daily_volume_stats.c.letter_totals).label('letter_totals'),
func.sum(daily_volume_stats.c.letter_sheet_totals).label('letter_sheet_totals')
).group_by(
daily_volume_stats.c.bst_date
daily_volume_stats.c.local_date
).order_by(
daily_volume_stats.c.bst_date
daily_volume_stats.c.local_date
).all()
return aggregated_totals
@@ -950,7 +950,7 @@ def fetch_daily_sms_provider_volumes_for_platform(start_date, end_date):
# query to return the total notifications sent per day for each channel. NB start and end dates are inclusive
daily_volume_stats = db.session.query(
FactBilling.bst_date,
FactBilling.local_date,
FactBilling.provider,
func.sum(FactBilling.notifications_sent).label('sms_totals'),
func.sum(FactBilling.billable_units).label('sms_fragment_totals'),
@@ -958,13 +958,13 @@ def fetch_daily_sms_provider_volumes_for_platform(start_date, end_date):
func.sum(FactBilling.billable_units * FactBilling.rate_multiplier * FactBilling.rate).label('sms_cost'),
).filter(
FactBilling.notification_type == SMS_TYPE,
FactBilling.bst_date >= start_date,
FactBilling.bst_date <= end_date,
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date,
).group_by(
FactBilling.bst_date,
FactBilling.local_date,
FactBilling.provider,
).order_by(
FactBilling.bst_date,
FactBilling.local_date,
FactBilling.provider,
).all()
@@ -977,7 +977,7 @@ def fetch_volumes_by_service(start_date, end_date):
year_end_date = int(end_date.strftime('%Y'))
volume_stats = db.session.query(
FactBilling.bst_date,
FactBilling.local_date,
FactBilling.service_id,
func.sum(case([
(FactBilling.notification_type == SMS_TYPE, FactBilling.notifications_sent)
@@ -1000,10 +1000,10 @@ def fetch_volumes_by_service(start_date, end_date):
], else_=0
)).label('letter_sheet_totals')
).filter(
FactBilling.bst_date >= start_date,
FactBilling.bst_date <= end_date
FactBilling.local_date >= start_date,
FactBilling.local_date <= end_date
).group_by(
FactBilling.bst_date,
FactBilling.local_date,
FactBilling.service_id,
FactBilling.notification_type
).subquery()

58
app/dao/fact_notification_status_dao.py
View File

@@ -41,7 +41,7 @@ def update_fact_notification_status(process_day, notification_type, service_id):
# delete any existing rows in case some no longer exist e.g. if all messages are sent
FactNotificationStatus.query.filter(
FactNotificationStatus.bst_date == process_day,
FactNotificationStatus.local_date == process_day,
FactNotificationStatus.notification_type == notification_type,
FactNotificationStatus.service_id == service_id,
).delete()
@@ -72,7 +72,7 @@ def update_fact_notification_status(process_day, notification_type, service_id):
db.session.connection().execute(
insert(FactNotificationStatus.__table__).from_select(
[
FactNotificationStatus.bst_date,
FactNotificationStatus.local_date,
FactNotificationStatus.template_id,
FactNotificationStatus.service_id,
FactNotificationStatus.job_id,
@@ -88,17 +88,17 @@ def update_fact_notification_status(process_day, notification_type, service_id):
def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
return db.session.query(
func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
func.date_trunc('month', FactNotificationStatus.local_date).label('month'),
FactNotificationStatus.notification_type,
FactNotificationStatus.notification_status,
func.sum(FactNotificationStatus.notification_count).label('count')
).filter(
FactNotificationStatus.service_id == service_id,
FactNotificationStatus.bst_date >= start_date,
FactNotificationStatus.bst_date < end_date,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date < end_date,
FactNotificationStatus.key_type != KEY_TYPE_TEST
).group_by(
func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
func.date_trunc('month', FactNotificationStatus.local_date).label('month'),
FactNotificationStatus.notification_type,
FactNotificationStatus.notification_status
).all()
@@ -132,7 +132,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
FactNotificationStatus.notification_count.label('count')
).filter(
FactNotificationStatus.service_id == service_id,
FactNotificationStatus.bst_date >= start_date,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.key_type != KEY_TYPE_TEST
)
@@ -181,8 +181,8 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date):
FactNotificationStatus.key_type.label('key_type'),
func.sum(FactNotificationStatus.notification_count).label('count')
).filter(
FactNotificationStatus.bst_date >= start_date,
FactNotificationStatus.bst_date <= end_date
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date <= end_date
).group_by(
FactNotificationStatus.notification_type,
FactNotificationStatus.notification_status,
@@ -245,8 +245,8 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro
FactNotificationStatus.notification_status.label('status'),
func.sum(FactNotificationStatus.notification_count).label('count')
).filter(
FactNotificationStatus.bst_date >= start_date,
FactNotificationStatus.bst_date <= end_date,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date <= end_date,
FactNotificationStatus.service_id == Service.id,
).group_by(
FactNotificationStatus.service_id.label('service_id'),
@@ -334,15 +334,15 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
Template.name.label('name'),
Template.template_type.label('template_type'),
Template.is_precompiled_letter.label('is_precompiled_letter'),
extract('month', FactNotificationStatus.bst_date).label('month'),
extract('year', FactNotificationStatus.bst_date).label('year'),
extract('month', FactNotificationStatus.local_date).label('month'),
extract('year', FactNotificationStatus.local_date).label('year'),
func.sum(FactNotificationStatus.notification_count).label('count')
).join(
Template, FactNotificationStatus.template_id == Template.id
).filter(
FactNotificationStatus.service_id == service_id,
FactNotificationStatus.bst_date >= start_date,
FactNotificationStatus.bst_date <= end_date,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date <= end_date,
FactNotificationStatus.key_type != KEY_TYPE_TEST,
FactNotificationStatus.notification_status != NOTIFICATION_CANCELLED,
).group_by(
@@ -350,11 +350,11 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
Template.name,
Template.template_type,
Template.is_precompiled_letter,
extract('month', FactNotificationStatus.bst_date).label('month'),
extract('year', FactNotificationStatus.bst_date).label('year'),
extract('month', FactNotificationStatus.local_date).label('month'),
extract('year', FactNotificationStatus.local_date).label('year'),
).order_by(
extract('year', FactNotificationStatus.bst_date),
extract('month', FactNotificationStatus.bst_date),
extract('year', FactNotificationStatus.local_date),
extract('month', FactNotificationStatus.local_date),
Template.name
)
@@ -413,7 +413,7 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id):
def get_total_notifications_for_date_range(start_date, end_date):
query = db.session.query(
FactNotificationStatus.bst_date.cast(db.Text).label("bst_date"),
FactNotificationStatus.local_date.cast(db.Text).label("local_date"),
func.sum(case(
[
(FactNotificationStatus.notification_type == 'email', FactNotificationStatus.notification_count)
@@ -432,21 +432,21 @@ def get_total_notifications_for_date_range(start_date, end_date):
).filter(
FactNotificationStatus.key_type != KEY_TYPE_TEST,
).group_by(
FactNotificationStatus.bst_date
FactNotificationStatus.local_date
).order_by(
FactNotificationStatus.bst_date
FactNotificationStatus.local_date
)
if start_date and end_date:
query = query.filter(
FactNotificationStatus.bst_date >= start_date,
FactNotificationStatus.bst_date <= end_date
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date <= end_date
)
return query.all()
def fetch_monthly_notification_statuses_per_service(start_date, end_date):
return db.session.query(
func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date).label('date_created'),
func.date_trunc('month', FactNotificationStatus.local_date).cast(Date).label('date_created'),
Service.id.label('service_id'),
Service.name.label('service_name'),
FactNotificationStatus.notification_type,
@@ -494,15 +494,15 @@ def fetch_monthly_notification_statuses_per_service(start_date, end_date):
FactNotificationStatus.key_type != KEY_TYPE_TEST,
Service.research_mode.is_(False),
Service.restricted.is_(False),
FactNotificationStatus.bst_date >= start_date,
FactNotificationStatus.bst_date <= end_date,
FactNotificationStatus.local_date >= start_date,
FactNotificationStatus.local_date <= end_date,
).group_by(
Service.id,
Service.name,
func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date),
func.date_trunc('month', FactNotificationStatus.local_date).cast(Date),
FactNotificationStatus.notification_type,
).order_by(
func.date_trunc('month', FactNotificationStatus.bst_date).cast(Date),
func.date_trunc('month', FactNotificationStatus.local_date).cast(Date),
Service.id,
FactNotificationStatus.notification_type,
).all()

12
app/dao/fact_processing_time_dao.py
View File

@@ -18,12 +18,12 @@ def insert_update_processing_time(processing_time):
'''
table = FactProcessingTime.__table__
stmt = insert(table).values(
bst_date=processing_time.bst_date,
local_date=processing_time.local_date,
messages_total=processing_time.messages_total,
messages_within_10_secs=processing_time.messages_within_10_secs
)
stmt = stmt.on_conflict_do_update(
index_elements=[table.c.bst_date],
index_elements=[table.c.local_date],
set_={
'messages_total': stmt.excluded.messages_total,
'messages_within_10_secs': stmt.excluded.messages_within_10_secs,
@@ -35,7 +35,7 @@ def insert_update_processing_time(processing_time):
def get_processing_time_percentage_for_date_range(start_date, end_date):
query = db.session.query(
FactProcessingTime.bst_date.cast(db.Text).label("date"),
FactProcessingTime.local_date.cast(db.Text).label("date"),
FactProcessingTime.messages_total,
FactProcessingTime.messages_within_10_secs,
case([
@@ -46,8 +46,8 @@ def get_processing_time_percentage_for_date_range(start_date, end_date):
(FactProcessingTime.messages_total == 0, 100.0)
]).label("percentage")
).filter(
FactProcessingTime.bst_date >= start_date,
FactProcessingTime.bst_date <= end_date
).order_by(FactProcessingTime.bst_date)
FactProcessingTime.local_date >= start_date,
FactProcessingTime.local_date <= end_date
).order_by(FactProcessingTime.local_date)
return query.all()

4
app/dao/notifications_dao.py
View File

@@ -66,7 +66,7 @@ def dao_get_last_date_template_was_used(template_id, service_id):
return last_date_from_notifications
last_date = db.session.query(
functions.max(FactNotificationStatus.bst_date)
functions.max(FactNotificationStatus.local_date)
).filter(
FactNotificationStatus.template_id == template_id,
FactNotificationStatus.key_type != KEY_TYPE_TEST
@@ -806,7 +806,7 @@ def get_service_ids_with_notifications_on_date(notification_type, date):
FactNotificationStatus.service_id.label('service_id')
).filter(
FactNotificationStatus.notification_type == notification_type,
FactNotificationStatus.bst_date == date,
FactNotificationStatus.local_date == date,
)
return {

2
app/dao/provider_details_dao.py
View File

@@ -165,7 +165,7 @@ def dao_get_provider_stats():
func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label('current_month_billable_sms')
).filter(
FactBilling.notification_type == SMS_TYPE,
FactBilling.bst_date >= first_day_of_the_month
FactBilling.local_date >= first_day_of_the_month
).group_by(
FactBilling.provider
).subquery()

4
app/dao/services_dao.py
View File

@@ -97,8 +97,8 @@ def dao_fetch_live_services_data():
).subquery()
this_year_ft_billing = FactBilling.query.filter(
FactBilling.bst_date >= year_start_date,
FactBilling.bst_date <= year_end_date,
FactBilling.local_date >= year_start_date,
FactBilling.local_date <= year_end_date,
).subquery()
data = db.session.query(