Merge pull request #2381 from alphagov/inbound-sms-retention

Inbound sms now deletes according to data retention

tests/app/dao/notification_dao/test_notification_dao.py

@@ -19,7 +19,7 @@ from app.dao.notifications_dao import (
     dao_timeout_notifications,
     dao_update_notification,
     dao_update_notifications_by_reference,
-    delete_notifications_created_more_than_a_week_ago_by_type,
+    delete_notifications_older_than_retention_by_type,
     get_notification_by_id,
     get_notification_for_job,
     get_notification_with_personalisation,
@@ -79,7 +79,7 @@ def test_should_have_decorated_notifications_dao_functions():
     assert get_notification_with_personalisation.__wrapped__.__name__ == 'get_notification_with_personalisation'  # noqa
     assert get_notifications_for_service.__wrapped__.__name__ == 'get_notifications_for_service'  # noqa
     assert get_notification_by_id.__wrapped__.__name__ == 'get_notification_by_id'  # noqa
-    assert delete_notifications_created_more_than_a_week_ago_by_type.__wrapped__.__name__ == 'delete_notifications_created_more_than_a_week_ago_by_type'  # noqa
+    assert delete_notifications_older_than_retention_by_type.__wrapped__.__name__ == 'delete_notifications_older_than_retention_by_type'  # noqa
     assert dao_delete_notifications_and_history_by_id.__wrapped__.__name__ == 'dao_delete_notifications_and_history_by_id'  # noqa
 
 

tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py

@@ -6,7 +6,7 @@ from datetime import (
 import pytest
 from flask import current_app
 from freezegun import freeze_time
-from app.dao.notifications_dao import delete_notifications_created_more_than_a_week_ago_by_type
+from app.dao.notifications_dao import delete_notifications_older_than_retention_by_type
 from app.models import Notification, NotificationHistory
 from tests.app.db import (
     create_template,
@@ -48,7 +48,7 @@ def test_should_delete_notifications_by_type_after_seven_days(
     assert len(all_notifications) == 30
     # Records from before 3rd should be deleted
     with freeze_time(delete_run_time):
-        delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
+        delete_notifications_older_than_retention_by_type(notification_type)
     remaining_sms_notifications = Notification.query.filter_by(notification_type='sms').all()
     remaining_letter_notifications = Notification.query.filter_by(notification_type='letter').all()
     remaining_email_notifications = Notification.query.filter_by(notification_type='email').all()
@@ -77,7 +77,7 @@ def test_should_not_delete_notification_history(sample_service, notification_typ
         create_notification(template=letter_template, status='permanent-failure')
     assert Notification.query.count() == 3
     assert NotificationHistory.query.count() == 3
-    delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
+    delete_notifications_older_than_retention_by_type(notification_type)
     assert Notification.query.count() == 2
     assert NotificationHistory.query.count() == 3
 
@@ -109,7 +109,7 @@ def test_delete_notifications_for_days_of_retention(sample_service, notification
                         created_at=datetime.utcnow() - timedelta(days=8))
     create_service_data_retention(service_id=sample_service.id, notification_type=notification_type)
     assert len(Notification.query.all()) == 9
-    delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
+    delete_notifications_older_than_retention_by_type(notification_type)
     assert len(Notification.query.all()) == 7
     assert len(Notification.query.filter_by(notification_type=notification_type).all()) == 1
     if notification_type == 'letter':
@@ -146,7 +146,7 @@ def test_delete_notifications_keep_data_for_days_of_retention_is_longer(sample_s
     create_notification(template=default_letter_template, status='temporary-failure',
                         created_at=datetime.utcnow() - timedelta(days=8))
     assert len(Notification.query.all()) == 9
-    delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
+    delete_notifications_older_than_retention_by_type(notification_type)
     assert len(Notification.query.filter_by().all()) == 8
     assert len(Notification.query.filter_by(notification_type=notification_type).all()) == 2
     if notification_type == 'letter':
@@ -171,7 +171,7 @@ def test_delete_notifications_delete_notification_type_for_default_time_if_no_da
     create_notification(template=letter_template, status='temporary-failure',
                         created_at=datetime.utcnow() - timedelta(days=14))
     assert len(Notification.query.all()) == 6
-    delete_notifications_created_more_than_a_week_ago_by_type('email')
+    delete_notifications_older_than_retention_by_type('email')
     assert len(Notification.query.filter_by().all()) == 5
     assert len(Notification.query.filter_by(notification_type='email').all()) == 1
 
@@ -182,7 +182,7 @@ def test_delete_notifications_does_try_to_delete_from_s3_when_letter_has_not_bee
 
     create_notification(template=letter_template, status='sending',
                         reference='LETTER_REF')
-    delete_notifications_created_more_than_a_week_ago_by_type('email', qry_limit=1)
+    delete_notifications_older_than_retention_by_type('email', qry_limit=1)
     mock_get_s3.assert_not_called()
 
 
@@ -196,7 +196,7 @@ def test_delete_notifications_calls_subquery(
     create_notification(template=sms_template, created_at=datetime.now() - timedelta(days=8))
 
     assert Notification.query.count() == 3
-    delete_notifications_created_more_than_a_week_ago_by_type('sms', qry_limit=1)
+    delete_notifications_older_than_retention_by_type('sms', qry_limit=1)
     assert Notification.query.count() == 0
 
 

tests/app/dao/test_inbound_sms_dao.py

@@ -1,19 +1,18 @@
-from datetime import datetime, timedelta
+from datetime import datetime
+from itertools import product
 
 from freezegun import freeze_time
 
 from app.dao.inbound_sms_dao import (
     dao_get_inbound_sms_for_service,
     dao_count_inbound_sms_for_service,
-    delete_inbound_sms_created_more_than_a_week_ago,
+    delete_inbound_sms_older_than_retention,
     dao_get_inbound_sms_by_id,
     dao_get_paginated_inbound_sms_for_service_for_public_api,
     dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service
 )
 from tests.conftest import set_config
-from tests.app.db import create_inbound_sms, create_service
-
-from app.models import InboundSms
+from tests.app.db import create_inbound_sms, create_service, create_service_data_retention
 
 
 def test_get_all_inbound_sms(sample_service):
@@ -59,15 +58,10 @@ def test_get_all_inbound_sms_filters_on_service(notify_db_session):
 
 
 def test_get_all_inbound_sms_filters_on_time(sample_service, notify_db_session):
-    create_inbound_sms(sample_service, user_number='447700900111', content='111 1', created_at=datetime(2017, 1, 2))
-    sms_two = create_inbound_sms(
-        sample_service,
-        user_number='447700900111',
-        content='111 2',
-        created_at=datetime(2017, 1, 3)
-    )
+    create_inbound_sms(sample_service, created_at=datetime(2017, 8, 6, 22, 59))  # sunday evening
+    sms_two = create_inbound_sms(sample_service, created_at=datetime(2017, 8, 6, 23, 0))  # monday (7th) morning
 
-    with freeze_time('2017-01-09'):
+    with freeze_time('2017-08-14 12:00'):
         res = dao_get_inbound_sms_for_service(sample_service.id)
 
     assert len(res) == 1
@@ -93,28 +87,43 @@ def test_count_inbound_sms_for_service_filters_messages_older_than_seven_days(sa
         assert dao_count_inbound_sms_for_service(sample_service.id) == 1
 
 
-@freeze_time("2017-01-01 12:00:00")
-def test_should_delete_inbound_sms_older_than_seven_days(sample_service):
-    older_than_seven_days = datetime.utcnow() - timedelta(days=7, seconds=1)
-    create_inbound_sms(sample_service, created_at=older_than_seven_days)
-    delete_inbound_sms_created_more_than_a_week_ago()
+@freeze_time("2017-06-08 12:00:00")
+def test_should_delete_inbound_sms_according_to_data_retention(notify_db_session):
+    no_retention_service = create_service(service_name='no retention')
+    short_retention_service = create_service(service_name='three days')
+    long_retention_service = create_service(service_name='thirty days')
 
-    assert len(InboundSms.query.all()) == 0
+    services = [short_retention_service, no_retention_service, long_retention_service]
 
+    create_service_data_retention(long_retention_service.id, notification_type='sms', days_of_retention=30)
+    create_service_data_retention(short_retention_service.id, notification_type='sms', days_of_retention=3)
+    # email retention doesn't affect anything
+    create_service_data_retention(short_retention_service.id, notification_type='email', days_of_retention=4)
 
-@freeze_time("2017-01-01 12:00:00")
-def test_should_not_delete_inbound_sms_before_seven_days(sample_service):
-    yesterday = datetime.utcnow() - timedelta(days=1)
-    just_before_seven_days = datetime.utcnow() - timedelta(days=6, hours=23, minutes=59, seconds=59)
-    older_than_seven_days = datetime.utcnow() - timedelta(days=7, seconds=1)
+    dates = [
+        datetime(2017, 6, 4, 23, 00),  # just before three days
+        datetime(2017, 6, 4, 22, 59),  # older than three days
+        datetime(2017, 5, 31, 23, 00),  # just before seven days
+        datetime(2017, 5, 31, 22, 59),  # older than seven days
+        datetime(2017, 5, 1, 0, 0),  # older than thirty days
+    ]
 
-    create_inbound_sms(sample_service, created_at=yesterday)
-    create_inbound_sms(sample_service, created_at=just_before_seven_days)
-    create_inbound_sms(sample_service, created_at=older_than_seven_days)
+    for date, service in product(dates, services):
+        create_inbound_sms(service, created_at=date)
 
-    delete_inbound_sms_created_more_than_a_week_ago()
+    deleted_count = delete_inbound_sms_older_than_retention()
 
-    assert len(InboundSms.query.all()) == 2
+    # four deleted for the 3-day service, two for the default seven days one, one for the 30 day
+    assert deleted_count == 7
+    assert {
+        x.created_at for x in dao_get_inbound_sms_for_service(short_retention_service.id, limit_days=None)
+    } == set(dates[:1])
+    assert {
+        x.created_at for x in dao_get_inbound_sms_for_service(no_retention_service.id, limit_days=None)
+    } == set(dates[:3])
+    assert {
+        x.created_at for x in dao_get_inbound_sms_for_service(long_retention_service.id, limit_days=None)
+    } == set(dates[:4])
 
 
 def test_get_inbound_sms_by_id_returns(sample_service):