Merge pull request #2381 from alphagov/inbound-sms-retention

Inbound sms now deletes according to data retention

app/dao/inbound_sms_dao.py

@@ -1,8 +1,3 @@
-from datetime import (
-    timedelta,
-    datetime,
-    date
-)
 from flask import current_app
 from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import desc, and_
@@ -10,8 +5,8 @@ from sqlalchemy.orm import aliased
 
 from app import db
 from app.dao.dao_utils import transactional
-from app.models import InboundSms
-from app.utils import get_london_midnight_in_utc
+from app.models import InboundSms, Service, ServiceDataRetention, SMS_TYPE
+from app.utils import midnight_n_days_ago
 
 
 @transactional
@@ -19,14 +14,15 @@ def dao_create_inbound_sms(inbound_sms):
     db.session.add(inbound_sms)
 
 
-def dao_get_inbound_sms_for_service(service_id, limit=None, user_number=None):
-    start_date = get_london_midnight_in_utc(date.today() - timedelta(days=6))
+def dao_get_inbound_sms_for_service(service_id, limit=None, user_number=None, limit_days=7):
     q = InboundSms.query.filter(
-        InboundSms.service_id == service_id,
-        InboundSms.created_at >= start_date
+        InboundSms.service_id == service_id
     ).order_by(
         InboundSms.created_at.desc()
     )
+    if limit_days is not None:
+        start_date = midnight_n_days_ago(limit_days)
+        q = q.filter(InboundSms.created_at >= start_date)
 
     if user_number:
         q = q.filter(InboundSms.user_number == user_number)
@@ -60,21 +56,63 @@ def dao_get_paginated_inbound_sms_for_service_for_public_api(
 
 
 def dao_count_inbound_sms_for_service(service_id):
-    start_date = get_london_midnight_in_utc(date.today() - timedelta(days=6))
+    start_date = midnight_n_days_ago(6)
     return InboundSms.query.filter(
         InboundSms.service_id == service_id,
         InboundSms.created_at >= start_date
     ).count()
 
 
+def _delete_inbound_sms(datetime_to_delete_from, query_filter):
+    query_limit = 10000
+
+    subquery = db.session.query(
+        InboundSms.id
+    ).filter(
+        InboundSms.created_at < datetime_to_delete_from,
+        *query_filter
+    ).limit(
+        query_limit
+    ).subquery()
+
+    deleted = 0
+    # set to nonzero just to enter the loop
+    number_deleted = 1
+    while number_deleted > 0:
+        number_deleted = InboundSms.query.filter(InboundSms.id.in_(subquery)).delete(synchronize_session='fetch')
+        deleted += number_deleted
+
+    return deleted
+
+
 @statsd(namespace="dao")
 @transactional
-def delete_inbound_sms_created_more_than_a_week_ago():
-    seven_days_ago = datetime.utcnow() - timedelta(days=7)
+def delete_inbound_sms_older_than_retention():
+    current_app.logger.info('Deleting inbound sms for services with flexible data retention')
 
-    deleted = db.session.query(InboundSms).filter(
-        InboundSms.created_at < seven_days_ago
-    ).delete(synchronize_session='fetch')
+    flexible_data_retention = ServiceDataRetention.query.join(
+        ServiceDataRetention.service,
+        Service.inbound_number
+    ).filter(
+        ServiceDataRetention.notification_type == SMS_TYPE
+    ).all()
+
+    deleted = 0
+    for f in flexible_data_retention:
+        n_days_ago = midnight_n_days_ago(f.days_of_retention)
+
+        current_app.logger.info("Deleting inbound sms for service id: {}".format(f.service_id))
+        deleted += _delete_inbound_sms(n_days_ago, query_filter=[InboundSms.service_id == f.service_id])
+
+    current_app.logger.info('Deleting inbound sms for services without flexible data retention')
+
+    seven_days_ago = midnight_n_days_ago(7)
+
+    deleted += _delete_inbound_sms(seven_days_ago, query_filter=[
+        InboundSms.service_id.notin_(x.service_id for x in flexible_data_retention),
+    ])
+
+    current_app.logger.info('Deleted {} inbound sms'.format(deleted))
 
     return deleted
 
@@ -108,7 +146,7 @@ def dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service(
     ORDER BY t1.created_at DESC;
     LIMIT 50 OFFSET :page
     """
-    start_date = get_london_midnight_in_utc(date.today() - timedelta(days=6))
+    start_date = midnight_n_days_ago(6)
     t2 = aliased(InboundSms)
     q = db.session.query(
         InboundSms

app/dao/notifications_dao.py

@@ -293,7 +293,7 @@ def _filter_query(query, filter_dict=None):
 
 
 @statsd(namespace="dao")
-def delete_notifications_created_more_than_a_week_ago_by_type(notification_type, qry_limit=10000):
+def delete_notifications_older_than_retention_by_type(notification_type, qry_limit=10000):
     current_app.logger.info(
         'Deleting {} notifications for services with flexible data retention'.format(notification_type))