From 5d838415d3d4c7fbb12fc93d55414c8973abe783 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 9 Jan 2019 11:26:08 +0000 Subject: [PATCH] fix filter to look at right table a query for notifications was filtering on FtNotificationStatus - we aren't joining to that table in the query, so sqlalchemy added a cross join between ft_notification_status (3.7k rows) and Notifications (3.9m rows), resulting in a 1.3 trillion row materialised table. This query took 17 hours and pending. Also, remove orders from querys other than the outer one, since we're grouping anyway. --- app/dao/fact_notification_status_dao.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index c273ca066..bf6ec3c8e 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -153,8 +153,6 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date): FactNotificationStatus.notification_type, FactNotificationStatus.notification_status, FactNotificationStatus.key_type, - ).order_by( - FactNotificationStatus.notification_type ) today = get_london_midnight_in_utc(datetime.utcnow()) if start_date <= today.date() <= end_date: @@ -184,7 +182,9 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date): all_stats_table.c.notification_type ) else: - query = stats + query = stats.order_by( + FactNotificationStatus.notification_type + ) return query.all() @@ -245,7 +245,7 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro Notification.service_id ) if not include_from_test_key: - subquery = subquery.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST) + subquery = subquery.filter(Notification.key_type != KEY_TYPE_TEST) subquery = subquery.subquery() stats_for_today = db.session.query( @@ -261,7 +261,7 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro ).outerjoin( subquery, subquery.c.service_id == Service.id - ).order_by(Service.id) + ) all_stats_table = stats.union_all(stats_for_today).subquery() query = db.session.query(