darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-02 09:26:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

Update insert_update_notification_history to take a query limit

Browse Source 
The nightly job to delete email notifications was failing because it was
timing out (`psycopg2.errors.QueryCanceled: canceling statement due to statement timeout`).

This adds a query limit to the query which inserts or updates
notification history so that it only updates a maximum of 10000 rows at
a time.
This commit is contained in:
Katie Smith
2019-10-14 16:43:37 +01:00
parent 0152dcf254
commit fb80a9c92e
2 changed files with 56 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
app/dao/notifications_dao.py
View File

@@ -367,8 +367,10 @@ def _delete_for_query(subquery):
return deleted
def insert_update_notification_history(notification_type, date_to_delete_from, service_id):
notifications = db.session.query(
def insert_update_notification_history(notification_type, date_to_delete_from, service_id, query_limit=10000):
offset = 0
notification_query = db.session.query(
*[x.name for x in NotificationHistory.__table__.c]
).filter(
Notification.notification_type == notification_type,
@@ -376,23 +378,29 @@ def insert_update_notification_history(notification_type, date_to_delete_from, s
Notification.created_at < date_to_delete_from,
Notification.key_type != KEY_TYPE_TEST
)
stmt = insert(NotificationHistory).from_select(
NotificationHistory.__table__.c,
notifications
)
notifications_count = notification_query.count()
stmt = stmt.on_conflict_do_update(
constraint="notification_history_pkey",
set_={"notification_status": stmt.excluded.status,
"reference": stmt.excluded.reference,
"billable_units": stmt.excluded.billable_units,
"updated_at": stmt.excluded.updated_at,
"sent_at": stmt.excluded.sent_at,
"sent_by": stmt.excluded.sent_by
}
)
db.session.connection().execute(stmt)
db.session.commit()
while offset < notifications_count:
stmt = insert(NotificationHistory).from_select(
NotificationHistory.__table__.c,
notification_query.limit(query_limit).offset(offset)
)
stmt = stmt.on_conflict_do_update(
constraint="notification_history_pkey",
set_={
"notification_status": stmt.excluded.status,
"reference": stmt.excluded.reference,
"billable_units": stmt.excluded.billable_units,
"updated_at": stmt.excluded.updated_at,
"sent_at": stmt.excluded.sent_at,
"sent_by": stmt.excluded.sent_by
}
)
db.session.connection().execute(stmt)
db.session.commit()
offset += query_limit
def _delete_letters_from_s3(