darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-05 02:41:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

Commit the deletes every 10,000 rows.

Browse Source
This commit is contained in:
Rebecca Law
2018-12-21 13:57:35 +00:00
parent 3ec50c4fac
commit 62a8076161
2 changed files with 28 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
app/dao/notifications_dao.py
View File

@@ -325,19 +325,22 @@ def delete_notifications_created_more_than_a_week_ago_by_type(notification_type,
).all()
deleted = 0
for f in flexible_data_retention:
days_of_retention = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
days_of_retention = get_london_midnight_in_utc(
convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=f.days_of_retention)
query = db.session.query(Notification).filter(
func.date(Notification.created_at) < days_of_retention,
Notification.notification_type == f.notification_type, Notification.service_id == f.service_id)
Notification.created_at < days_of_retention,
Notification.notification_type == f.notification_type,
Notification.service_id == f.service_id
)
if notification_type == LETTER_TYPE:
_delete_letters_from_s3(query)
deleted += query.delete(synchronize_session='fetch')
db.session.commit()
deleted += _delete_notifications(deleted, query)
current_app.logger.info(
'Deleting {} notifications for services without flexible data retention'.format(notification_type))
seven_days_ago = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=7)
seven_days_ago = get_london_midnight_in_utc(convert_utc_to_bst(datetime.utcnow()).date()) - timedelta(days=7)
services_with_data_retention = [x.service_id for x in flexible_data_retention]
service_ids_to_purge = db.session.query(Service.id).filter(Service.id.notin_(services_with_data_retention)).all()
@@ -346,28 +349,33 @@ def delete_notifications_created_more_than_a_week_ago_by_type(notification_type,
Notification
).filter(
Notification.notification_type == notification_type,
func.date(Notification.created_at) < seven_days_ago,
Notification.created_at < seven_days_ago,
Notification.service_id == service_id
).limit(qry_limit)
if notification_type == LETTER_TYPE:
_delete_letters_from_s3(query=subquery)
number_deleted = db.session.query(Notification).filter(
Notification.id.in_([x.id for x in subquery.all()])).delete(synchronize_session='fetch')
deleted += number_deleted
db.session.commit()
while number_deleted > 0:
number_deleted = db.session.query(Notification).filter(
Notification.id.in_([x.id for x in subquery.all()])).delete(synchronize_session='fetch')
deleted += number_deleted
db.session.commit()
deleted += _delete_notifications(deleted, subquery)
current_app.logger.info('Finished deleting {} notifications'.format(notification_type))
return deleted
def _delete_notifications(deleted, query):
number_deleted = db.session.query(Notification).filter(
Notification.id.in_([x.id for x in query.all()])).delete(synchronize_session='fetch')
deleted += number_deleted
db.session.commit()
while number_deleted > 0:
number_deleted = db.session.query(Notification).filter(
Notification.id.in_([x.id for x in query.all()])).delete(synchronize_session='fetch')
deleted += number_deleted
db.session.commit()
return deleted
def _delete_letters_from_s3(query):
letters_to_delete_from_s3 = query.all()
for letter in letters_to_delete_from_s3: