darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-20 23:41:17 -05:00
Code Issues Packages Projects Releases Wiki Activity

Updated the query to improve the performance.

Browse Source 
The scheduled job for deleting emails has failed silently for many nights now.
This commit is contained in:
Rebecca Law
2018-12-20 16:09:38 +00:00
parent 7d026ad385
commit 89923eab71
1 changed files with 30 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
app/dao/notifications_dao.py
View File

@@ -44,7 +44,8 @@ from app.models import (
NOTIFICATION_SENT, NOTIFICATION_SENT,
SMS_TYPE, SMS_TYPE,
EMAIL_TYPE, EMAIL_TYPE,
ServiceDataRetention ServiceDataRetention,
Service
) )
from app.utils import get_london_midnight_in_utc from app.utils import get_london_midnight_in_utc
from app.utils import midnight_n_days_ago, escape_special_characters from app.utils import midnight_n_days_ago, escape_special_characters
@@ -315,8 +316,9 @@ def _filter_query(query, filter_dict=None):
@statsd(namespace="dao") @statsd(namespace="dao")
def delete_notifications_created_more_than_a_week_ago_by_type(notification_type): def delete_notifications_created_more_than_a_week_ago_by_type(notification_type, qry_limit=10000):
current_app.logger.info('Deleting {} notifications for services with flexible data retention'.format(notification_type)) current_app.logger.info(
'Deleting {} notifications for services with flexible data retention'.format(notification_type))
flexible_data_retention = ServiceDataRetention.query.filter( flexible_data_retention = ServiceDataRetention.query.filter(
ServiceDataRetention.notification_type == notification_type ServiceDataRetention.notification_type == notification_type
@@ -332,18 +334,34 @@ def delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
deleted += query.delete(synchronize_session='fetch') deleted += query.delete(synchronize_session='fetch')
db.session.commit() db.session.commit()
current_app.logger.info('Deleting {} notifications for services without flexible data retention'.format(notification_type)) current_app.logger.info(
'Deleting {} notifications for services without flexible data retention'.format(notification_type))
seven_days_ago = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=7) seven_days_ago = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=7)
services_with_data_retention = [x.service_id for x in flexible_data_retention] services_with_data_retention = [x.service_id for x in flexible_data_retention]
query = db.session.query(Notification).filter(func.date(Notification.created_at) < seven_days_ago, service_ids_to_purge = db.session.query(Service.id).filter(Service.id.notin_(services_with_data_retention)).all()
Notification.notification_type == notification_type,
Notification.service_id.notin_( for service_id in service_ids_to_purge:
services_with_data_retention)) subquery = db.session.query(
if notification_type == LETTER_TYPE: Notification
_delete_letters_from_s3(query=query) ).filter(
deleted += query.delete(synchronize_session='fetch') Notification.notification_type == notification_type,
db.session.commit() func.date(Notification.created_at) < seven_days_ago,
Notification.service_id == service_id
).limit(qry_limit)
if notification_type == LETTER_TYPE:
_delete_letters_from_s3(query=subquery)
number_deleted = db.session.query(Notification).filter(
Notification.id.in_([x.id for x in subquery.all()])).delete(synchronize_session='fetch')
deleted += number_deleted
db.session.commit()
while number_deleted > 0:
number_deleted = db.session.query(Notification).filter(
Notification.id.in_([x.id for x in subquery.all()])).delete(synchronize_session='fetch')
deleted += number_deleted
db.session.commit()
current_app.logger.info('Finished deleting {} notifications'.format(notification_type)) current_app.logger.info('Finished deleting {} notifications'.format(notification_type))