From 4a520bce788deef05cffef282c1e38e74ba2fb95 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Tue, 17 May 2022 17:24:27 +0100 Subject: [PATCH] Optimise billing query for notification history This follows the same pattern as for status aggregations [^1]. We haven't seen this problem for a long time because of [^2], but now we're trying to re-run the aggregation for some incorrect rows it's becoming apparent we need to fix it. The following query currently fails in Production after the 30 min SQLAlchemy timeout: select template_id, rate_multiplier, international, sum(billable_units), count(*) from notification_history where notification_status in ('delivered', 'sending') and key_type != 'test' and notification_type = 'sms' and service_id = '539d63a1-701d-400d-ab11-f3ee2319d4d4' and created_at >= '2021-07-07 23:00' and created_at < '2021-07-08 23:00' group by 1,2,3,4; Running a quick "explain analyze" with this change applied returns near immediately, but hangs without it. This is enough evidence for me that this change will fix the issue. [^1]: https://github.com/alphagov/notifications-api/pull/3417 [^2]: https://github.com/alphagov/notifications-api/commit/e5c76ffda732dcb572c6f3d4cd0c89464d41b82f --- app/dao/fact_billing_dao.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index 899cc780f..8b03a773e 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -15,7 +15,8 @@ from app.dao.organisation_dao import dao_get_organisation_live_services from app.models import ( EMAIL_TYPE, INTERNATIONAL_POSTAGE_TYPES, - KEY_TYPE_TEST, + KEY_TYPE_NORMAL, + KEY_TYPE_TEAM, LETTER_TYPE, NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS, NOTIFICATION_STATUS_TYPES_BILLABLE_SMS, @@ -493,7 +494,7 @@ def _query_for_billing_data(table, notification_type, start_date, end_date, serv func.count().label('notifications_sent'), ).filter( table.status.in_(NOTIFICATION_STATUS_TYPES_SENT_EMAILS), - table.key_type != KEY_TYPE_TEST, + table.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)), table.created_at >= start_date, table.created_at < end_date, table.notification_type == notification_type, @@ -520,7 +521,7 @@ def _query_for_billing_data(table, notification_type, start_date, end_date, serv func.count().label('notifications_sent'), ).filter( table.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE_SMS), - table.key_type != KEY_TYPE_TEST, + table.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)), table.created_at >= start_date, table.created_at < end_date, table.notification_type == notification_type, @@ -549,7 +550,7 @@ def _query_for_billing_data(table, notification_type, start_date, end_date, serv func.count().label('notifications_sent'), ).filter( table.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE_FOR_LETTERS), - table.key_type != KEY_TYPE_TEST, + table.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)), table.created_at >= start_date, table.created_at < end_date, table.notification_type == notification_type,