From 1a1de39949147fabbfd9f6613184299d4b309bea Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 21 Nov 2024 09:38:43 -0800 Subject: [PATCH] change querie to stmt --- app/dao/fact_billing_dao.py | 68 ++++++++++++------------- app/dao/fact_notification_status_dao.py | 30 +++++------ app/dao/inbound_sms_dao.py | 6 +-- app/dao/notifications_dao.py | 28 +++++----- app/dao/provider_details_dao.py | 6 +-- app/dao/services_dao.py | 28 +++++----- app/dao/uploads_dao.py | 18 +++---- 7 files changed, 91 insertions(+), 93 deletions(-) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index 0371ae8e5..07e00621a 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -65,7 +65,7 @@ def fetch_sms_free_allowance_remainder_until_date(end_date): def fetch_sms_billing_for_all_services(start_date, end_date): # ASSUMPTION: AnnualBilling has been populated for year. - allowance_left_at_start_date_querie = fetch_sms_free_allowance_remainder_until_date( + allowance_left_at_start_date_stmt = fetch_sms_free_allowance_remainder_until_date( start_date ).subquery() @@ -76,14 +76,14 @@ def fetch_sms_billing_for_all_services(start_date, end_date): # subtract sms_billable_units units accrued since report's start date to get up-to-date # allowance remainder sms_allowance_left = func.greatest( - allowance_left_at_start_date_querie.c.sms_remainder - sms_billable_units, 0 + allowance_left_at_start_date_stmt.c.sms_remainder - sms_billable_units, 0 ) # billable units here are for period between start date and end date only, so to see # how many are chargeable, we need to see how much free allowance was used up in the # period up until report's start date and then do a subtraction chargeable_sms = func.greatest( - sms_billable_units - allowance_left_at_start_date_querie.c.sms_remainder, 0 + sms_billable_units - allowance_left_at_start_date_stmt.c.sms_remainder, 0 ) sms_cost = chargeable_sms * FactBilling.rate @@ -93,7 +93,7 @@ def fetch_sms_billing_for_all_services(start_date, end_date): Organization.id.label("organization_id"), Service.name.label("service_name"), Service.id.label("service_id"), - allowance_left_at_start_date_querie.c.free_sms_fragment_limit, + allowance_left_at_start_date_stmt.c.free_sms_fragment_limit, FactBilling.rate.label("sms_rate"), sms_allowance_left.label("sms_remainder"), sms_billable_units.label("sms_billable_units"), @@ -102,8 +102,8 @@ def fetch_sms_billing_for_all_services(start_date, end_date): ) .select_from(Service) .outerjoin( - allowance_left_at_start_date_querie, - Service.id == allowance_left_at_start_date_querie.c.service_id, + allowance_left_at_start_date_stmt, + Service.id == allowance_left_at_start_date_stmt.c.service_id, ) .outerjoin(Service.organization) .join( @@ -120,8 +120,8 @@ def fetch_sms_billing_for_all_services(start_date, end_date): Organization.id, Service.id, Service.name, - allowance_left_at_start_date_querie.c.free_sms_fragment_limit, - allowance_left_at_start_date_querie.c.sms_remainder, + allowance_left_at_start_date_stmt.c.free_sms_fragment_limit, + allowance_left_at_start_date_stmt.c.sms_remainder, FactBilling.rate, ) .order_by(Organization.name, Service.name) @@ -151,15 +151,15 @@ def fetch_billing_totals_for_year(service_id, year): union( *[ select( - querie.c.notification_type.label("notification_type"), - querie.c.rate.label("rate"), - func.sum(querie.c.notifications_sent).label("notifications_sent"), - func.sum(querie.c.chargeable_units).label("chargeable_units"), - func.sum(querie.c.cost).label("cost"), - func.sum(querie.c.free_allowance_used).label("free_allowance_used"), - func.sum(querie.c.charged_units).label("charged_units"), - ).group_by(querie.c.rate, querie.c.notification_type) - for querie in [ + stmt.c.notification_type.label("notification_type"), + stmt.c.rate.label("rate"), + func.sum(stmt.c.notifications_sent).label("notifications_sent"), + func.sum(stmt.c.chargeable_units).label("chargeable_units"), + func.sum(stmt.c.cost).label("cost"), + func.sum(stmt.c.free_allowance_used).label("free_allowance_used"), + func.sum(stmt.c.charged_units).label("charged_units"), + ).group_by(stmt.c.rate, stmt.c.notification_type) + for stmt in [ query_service_sms_usage_for_year(service_id, year).subquery(), query_service_email_usage_for_year(service_id, year).subquery(), ] @@ -206,22 +206,22 @@ def fetch_monthly_billing_for_year(service_id, year): union( *[ select( - querie.c.rate.label("rate"), - querie.c.notification_type.label("notification_type"), - func.date_trunc("month", querie.c.local_date) + stmt.c.rate.label("rate"), + stmt.c.notification_type.label("notification_type"), + func.date_trunc("month", stmt.c.local_date) .cast(Date) .label("month"), - func.sum(querie.c.notifications_sent).label("notifications_sent"), - func.sum(querie.c.chargeable_units).label("chargeable_units"), - func.sum(querie.c.cost).label("cost"), - func.sum(querie.c.free_allowance_used).label("free_allowance_used"), - func.sum(querie.c.charged_units).label("charged_units"), + func.sum(stmt.c.notifications_sent).label("notifications_sent"), + func.sum(stmt.c.chargeable_units).label("chargeable_units"), + func.sum(stmt.c.cost).label("cost"), + func.sum(stmt.c.free_allowance_used).label("free_allowance_used"), + func.sum(stmt.c.charged_units).label("charged_units"), ).group_by( - querie.c.rate, - querie.c.notification_type, + stmt.c.rate, + stmt.c.notification_type, "month", ) - for querie in [ + for stmt in [ query_service_sms_usage_for_year(service_id, year).subquery(), query_service_email_usage_for_year(service_id, year).subquery(), ] @@ -586,12 +586,12 @@ def fetch_email_usage_for_organization(organization_id, start_date, end_date): def fetch_sms_billing_for_organization(organization_id, financial_year): # ASSUMPTION: AnnualBilling has been populated for year. - ft_billing_subquerie = query_organization_sms_usage_for_year( + ft_billing_substmt = query_organization_sms_usage_for_year( organization_id, financial_year ).subquery() sms_billable_units = func.sum( - func.coalesce(ft_billing_subquerie.c.chargeable_units, 0) + func.coalesce(ft_billing_substmt.c.chargeable_units, 0) ) # subtract sms_billable_units units accrued since report's start date to get up-to-date @@ -600,8 +600,8 @@ def fetch_sms_billing_for_organization(organization_id, financial_year): AnnualBilling.free_sms_fragment_limit - sms_billable_units, 0 ) - chargeable_sms = func.sum(ft_billing_subquerie.c.charged_units) - sms_cost = func.sum(ft_billing_subquerie.c.cost) + chargeable_sms = func.sum(ft_billing_substmt.c.charged_units) + sms_cost = func.sum(ft_billing_substmt.c.cost) query = ( select( @@ -622,9 +622,7 @@ def fetch_sms_billing_for_organization(organization_id, financial_year): AnnualBilling.financial_year_start == financial_year, ), ) - .outerjoin( - ft_billing_subquerie, Service.id == ft_billing_subquerie.c.service_id - ) + .outerjoin(ft_billing_substmt, Service.id == ft_billing_substmt.c.service_id) .filter( Service.organization_id == organization_id, Service.restricted.is_(False) ) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index a0119fd91..eaa902bc0 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -191,7 +191,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days( all_stats_alias = aliased(all_stats_union, name="all_stats") # Final query with optional template joins - querie = select( + stmt = select( *( [ TemplateFolder.name.label("folder"), @@ -214,8 +214,8 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days( ) if by_template: - querie = ( - querie.join(Template, all_stats_alias.c.template_id == Template.id) + stmt = ( + stmt.join(Template, all_stats_alias.c.template_id == Template.id) .join(User, Template.created_by_id == User.id) .outerjoin( template_folder_map, Template.id == template_folder_map.c.template_id @@ -227,7 +227,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days( ) # Group by all necessary fields except date_used - querie = querie.group_by( + stmt = stmt.group_by( *( [ TemplateFolder.name, @@ -245,7 +245,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days( ) # Execute the query using Flask-SQLAlchemy's session - result = db.session.execute(querie) + result = db.session.execute(stmt) return result.mappings().all() @@ -361,7 +361,7 @@ def fetch_stats_for_all_services_by_date_range( if start_date <= utc_now().date() <= end_date: today = get_midnight_in_utc(utc_now()) - subquerie = ( + substmt = ( select( Notification.notification_type.label("notification_type"), Notification.status.label("status"), @@ -377,8 +377,8 @@ def fetch_stats_for_all_services_by_date_range( ) ) if not include_from_test_key: - subquerie = subquerie.filter(Notification.key_type != KeyType.TEST) - subquerie = subquerie.subquery() + substmt = substmt.filter(Notification.key_type != KeyType.TEST) + substmt = substmt.subquery() stats_for_today = select( Service.id.label("service_id"), @@ -386,10 +386,10 @@ def fetch_stats_for_all_services_by_date_range( Service.restricted.label("restricted"), Service.active.label("active"), Service.created_at.label("created_at"), - subquerie.c.notification_type.cast(db.Text).label("notification_type"), - subquerie.c.status.cast(db.Text).label("status"), - subquerie.c.count.label("count"), - ).outerjoin(subquerie, subquerie.c.service_id == Service.id) + substmt.c.notification_type.cast(db.Text).label("notification_type"), + substmt.c.status.cast(db.Text).label("status"), + substmt.c.count.label("count"), + ).outerjoin(substmt, substmt.c.service_id == Service.id) all_stats_table = stats.union_all(stats_for_today).subquery() query = ( @@ -515,7 +515,7 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id): def get_total_notifications_for_date_range(start_date, end_date): - querie = ( + stmt = ( select( FactNotificationStatus.local_date.label("local_date"), func.sum( @@ -546,11 +546,11 @@ def get_total_notifications_for_date_range(start_date, end_date): .order_by(FactNotificationStatus.local_date) ) if start_date and end_date: - querie = querie.filter( + stmt = stmt.filter( FactNotificationStatus.local_date >= start_date, FactNotificationStatus.local_date <= end_date, ) - return db.session.execute(querie).all() + return db.session.execute(stmt).all() def fetch_monthly_notification_statuses_per_service(start_date, end_date): diff --git a/app/dao/inbound_sms_dao.py b/app/dao/inbound_sms_dao.py index 1687bd56f..feb967b54 100644 --- a/app/dao/inbound_sms_dao.py +++ b/app/dao/inbound_sms_dao.py @@ -84,7 +84,7 @@ def dao_count_inbound_sms_for_service(service_id, limit_days): def _insert_inbound_sms_history(subquery, query_limit=10000): offset = 0 subquery_select = select(subquery) - inbound_sms_querie = select( + inbound_sms_stmt = select( InboundSms.id, InboundSms.created_at, InboundSms.service_id, @@ -94,13 +94,13 @@ def _insert_inbound_sms_history(subquery, query_limit=10000): InboundSms.provider, ).where(InboundSms.id.in_(subquery_select)) - count_query = select(func.count()).select_from(inbound_sms_querie.subquery()) + count_query = select(func.count()).select_from(inbound_sms_stmt.subquery()) inbound_sms_count = db.session.execute(count_query).scalar() or 0 while offset < inbound_sms_count: statement = insert(InboundSmsHistory).from_select( InboundSmsHistory.__table__.c, - inbound_sms_querie.limit(query_limit).offset(offset), + inbound_sms_stmt.limit(query_limit).offset(offset), ) statement = statement.on_conflict_do_nothing( diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 9b8f3614b..d93a002d8 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -194,11 +194,11 @@ def get_notifications_for_job( if page_size is None: page_size = current_app.config["PAGE_SIZE"] - querie = select(Notification).filter_by(service_id=service_id, job_id=job_id) - querie = _filter_query(querie, filter_dict) - querie = querie.order_by(asc(Notification.job_row_number)) + stmt = select(Notification).filter_by(service_id=service_id, job_id=job_id) + stmt = _filter_query(stmt, filter_dict) + stmt = stmt.order_by(asc(Notification.job_row_number)) - results = db.session.execute(querie).scalars().all() + results = db.session.execute(stmt).scalars().all() page_size = current_app.config["PAGE_SIZE"] offset = (page - 1) * page_size @@ -298,22 +298,22 @@ def get_notifications_for_service( if client_reference is not None: filters.append(Notification.client_reference == client_reference) - querie = select(Notification).where(*filters) - querie = _filter_query(querie, filter_dict) + stmt = select(Notification).where(*filters) + stmt = _filter_query(stmt, filter_dict) if personalisation: - querie = querie.options(joinedload(Notification.template)) + stmt = stmt.options(joinedload(Notification.template)) - querie = querie.order_by(desc(Notification.created_at)) - results = db.session.execute(querie).scalars().all() + stmt = stmt.order_by(desc(Notification.created_at)) + results = db.session.execute(stmt).scalars().all() offset = (page - 1) * page_size paginated_results = results[offset : offset + page_size] pagination = Pagination(paginated_results, page, page_size, len(results)) return pagination -def _filter_query(querie, filter_dict=None): +def _filter_query(stmt, filter_dict=None): if filter_dict is None: - return querie + return stmt multidict = MultiDict(filter_dict) @@ -321,14 +321,14 @@ def _filter_query(querie, filter_dict=None): statuses = multidict.getlist("status") if statuses: - querie = querie.where(Notification.status.in_(statuses)) + stmt = stmt.where(Notification.status.in_(statuses)) # filter by template template_types = multidict.getlist("template_type") if template_types: - querie = querie.where(Notification.notification_type.in_(template_types)) + stmt = stmt.where(Notification.notification_type.in_(template_types)) - return querie + return stmt def sanitize_successful_notification_by_id(notification_id, carrier, provider_response): diff --git a/app/dao/provider_details_dao.py b/app/dao/provider_details_dao.py index 90415820f..75adf5999 100644 --- a/app/dao/provider_details_dao.py +++ b/app/dao/provider_details_dao.py @@ -102,7 +102,7 @@ def dao_get_provider_stats(): current_datetime = utc_now() first_day_of_the_month = current_datetime.date().replace(day=1) - subquerie = ( + substmt = ( db.session.query( FactBilling.provider, func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label( @@ -127,11 +127,11 @@ def dao_get_provider_stats(): ProviderDetails.updated_at, ProviderDetails.supports_international, User.name.label("created_by_name"), - func.coalesce(subquerie.c.current_month_billable_sms, 0).label( + func.coalesce(substmt.c.current_month_billable_sms, 0).label( "current_month_billable_sms" ), ) - .outerjoin(subquerie, ProviderDetails.identifier == subquerie.c.provider) + .outerjoin(substmt, ProviderDetails.identifier == substmt.c.provider) .outerjoin(User, ProviderDetails.created_by_id == User.id) .order_by( ProviderDetails.notification_type, diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 6dd8cef91..31eaf2ef5 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -514,7 +514,7 @@ def dao_fetch_todays_stats_for_all_services( start_date = get_midnight_in_utc(today) end_date = get_midnight_in_utc(today + timedelta(days=1)) - subquerie = ( + substmt = ( select( Notification.notification_type, Notification.status, @@ -530,9 +530,9 @@ def dao_fetch_todays_stats_for_all_services( ) if not include_from_test_key: - subquerie = subquerie.filter(Notification.key_type != KeyType.TEST) + substmt = substmt.filter(Notification.key_type != KeyType.TEST) - subquerie = subquerie.subquery() + substmt = substmt.subquery() stmt = ( select( @@ -541,11 +541,11 @@ def dao_fetch_todays_stats_for_all_services( Service.restricted, Service.active, Service.created_at, - subquerie.c.notification_type, - subquerie.c.status, - subquerie.c.count, + substmt.c.notification_type, + substmt.c.status, + substmt.c.count, ) - .outerjoin(subquerie, subquerie.c.service_id == Service.id) + .outerjoin(substmt, substmt.c.service_id == Service.id) .order_by(Service.id) ) @@ -617,7 +617,7 @@ def dao_find_services_sending_to_tv_numbers(start_date, end_date, threshold=500) def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10000): - subquerie = ( + substmt = ( select( func.count(Notification.id).label("total_count"), Notification.service_id.label("service_id"), @@ -637,19 +637,19 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10 .having(func.count(Notification.id) >= threshold) ) - subquerie = subquerie.subquery() + substmt = substmt.subquery() stmt = ( select( Notification.service_id.label("service_id"), func.count(Notification.id).label("permanent_failure_count"), - subquerie.c.total_count.label("total_count"), + substmt.c.total_count.label("total_count"), ( cast(func.count(Notification.id), Float) - / cast(subquerie.c.total_count, Float) + / cast(substmt.c.total_count, Float) ).label("permanent_failure_rate"), ) - .join(subquerie, subquerie.c.service_id == Notification.service_id) + .join(substmt, substmt.c.service_id == Notification.service_id) .filter( Notification.service_id == Service.id, Notification.created_at >= start_date, @@ -660,10 +660,10 @@ def dao_find_services_with_high_failure_rates(start_date, end_date, threshold=10 Service.restricted == False, # noqa Service.active == True, # noqa ) - .group_by(Notification.service_id, subquerie.c.total_count) + .group_by(Notification.service_id, substmt.c.total_count) .having( cast(func.count(Notification.id), Float) - / cast(subquerie.c.total_count, Float) + / cast(substmt.c.total_count, Float) >= 0.25 ) ) diff --git a/app/dao/uploads_dao.py b/app/dao/uploads_dao.py index 96a0e6f43..48ee3bd73 100644 --- a/app/dao/uploads_dao.py +++ b/app/dao/uploads_dao.py @@ -52,7 +52,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size if limit_days is not None: jobs_query_filter.append(Job.created_at >= midnight_n_days_ago(limit_days)) - jobs_querie = ( + jobs_stmt = ( select( Job.id, Job.original_file_name, @@ -95,7 +95,7 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size Notification.created_at >= midnight_n_days_ago(limit_days) ) - letters_subquerie = ( + letters_substmt = ( select( func.count().label("notification_count"), _naive_gmt_to_utc(_get_printing_datetime(Notification.created_at)).label( @@ -117,28 +117,28 @@ def dao_get_uploads_by_service_id(service_id, limit_days=None, page=1, page_size .subquery() ) - letters_querie = ( + letters_stmt = ( select( literal(None).label("id"), literal("Uploaded letters").label("original_file_name"), - letters_subquerie.c.notification_count.label("notification_count"), + letters_substmt.c.notification_count.label("notification_count"), literal("letter").label("template_type"), literal(None).label("days_of_retention"), - letters_subquerie.c.printing_at.label("created_at"), + letters_substmt.c.printing_at.label("created_at"), literal(None).label("scheduled_for"), - letters_subquerie.c.printing_at.label("processing_started"), + letters_substmt.c.printing_at.label("processing_started"), literal(None).label("status"), literal("letter_day").label("upload_type"), literal(None).label("recipient"), ) .select_from(Notification) .group_by( - letters_subquerie.c.notification_count, - letters_subquerie.c.printing_at, + letters_substmt.c.notification_count, + letters_substmt.c.printing_at, ) ) - stmt = union(jobs_querie, letters_querie).order_by( + stmt = union(jobs_stmt, letters_stmt).order_by( desc("processing_started"), desc("created_at") )