Scheduled tasks to clean up the database

- tasks run hourly
- uses celery beat to schedule the tasks

4 new tasks
- delete verify codes (after 1 day)
- delete invitations (after 1 day)
- delete successful notifications  (after 1 day)
- delete failed notifications (after 7 days)

Delete methods in the DAO classes

tests/app/dao/test_invited_user_dao.py

@@ -1,4 +1,6 @@
+from datetime import datetime, timedelta
 import uuid
+from app import db
 
 from app.models import InvitedUser
 
@@ -6,7 +8,8 @@ from app.dao.invited_user_dao import (
     save_invited_user,
     get_invited_user,
     get_invited_users_for_service,
-    get_invited_user_by_id
+    get_invited_user_by_id,
+    delete_invitations_older_created_more_than_a_day_ago
 )
 
 
@@ -52,7 +55,6 @@ def test_get_unknown_invited_user_returns_none(notify_db, notify_db_session, sam
 
 
 def test_get_invited_users_for_service(notify_db, notify_db_session, sample_service):
-
     from tests.app.conftest import sample_invited_user
     invites = []
     for i in range(0, 5):
@@ -71,7 +73,6 @@ def test_get_invited_users_for_service(notify_db, notify_db_session, sample_serv
 
 
 def test_get_invited_users_for_service_that_has_no_invites(notify_db, notify_db_session, sample_service):
-
     invites = get_invited_users_for_service(sample_service.id)
     assert len(invites) == 0
 
@@ -85,3 +86,40 @@ def test_save_invited_user_sets_status_to_cancelled(notify_db, notify_db_session
     assert InvitedUser.query.count() == 1
     cancelled_invited_user = InvitedUser.query.get(sample_invited_user.id)
     assert cancelled_invited_user.status == 'cancelled'
+
+
+def test_should_delete_all_invitations_more_than_one_day_old(
+        sample_user,
+        sample_service):
+    make_invitation(sample_user, sample_service, age=timedelta(hours=24))
+    make_invitation(sample_user, sample_service, age=timedelta(hours=24))
+    assert len(InvitedUser.query.all()) == 2
+    delete_invitations_older_created_more_than_a_day_ago()
+    assert len(InvitedUser.query.all()) == 0
+
+
+def test_should_not_delete_invitations_less_than_one_day_old(
+        sample_user,
+        sample_service):
+    make_invitation(sample_user, sample_service, age=timedelta(hours=23, minutes=59, seconds=59),
+                    email_address="valid@2.com")
+    make_invitation(sample_user, sample_service, age=timedelta(hours=24),
+                    email_address="expired@1.com")
+
+    assert len(InvitedUser.query.all()) == 2
+    delete_invitations_older_created_more_than_a_day_ago()
+    assert len(InvitedUser.query.all()) == 1
+    assert InvitedUser.query.first().email_address == "valid@2.com"
+
+
+def make_invitation(user, service, age=timedelta(hours=0), email_address="test@test.com"):
+    verify_code = InvitedUser(
+        email_address=email_address,
+        from_user=user,
+        service=service,
+        status='pending',
+        created_at=datetime.utcnow() - age,
+        permissions='manage_settings'
+    )
+    db.session.add(verify_code)
+    db.session.commit()

tests/app/dao/test_notification_dao.py

@@ -10,9 +10,12 @@ from app.dao.notifications_dao import (
     get_notification,
     get_notification_for_job,
     get_notifications_for_job,
-    dao_get_notification_statistics_for_service
+    dao_get_notification_statistics_for_service,
+    delete_successful_notifications_created_more_than_a_day_ago,
+    delete_failed_notifications_created_more_than_a_week_ago
 )
 from tests.app.conftest import sample_job
+from tests.app.conftest import sample_notification
 
 
 def test_should_be_able_to_get_statistics_for_a_service(sample_template):
@@ -489,3 +492,44 @@ def test_update_notification(sample_notification, sample_template):
     dao_update_notification(sample_notification)
     notification_from_db = Notification.query.get(sample_notification.id)
     assert notification_from_db.status == 'failed'
+
+
+def test_should_delete_sent_notifications_after_one_day(notify_db, notify_db_session):
+    created_at = datetime.utcnow() - timedelta(hours=24)
+    sample_notification(notify_db, notify_db_session, created_at=created_at)
+    sample_notification(notify_db, notify_db_session, created_at=created_at)
+    assert len(Notification.query.all()) == 2
+    delete_successful_notifications_created_more_than_a_day_ago()
+    assert len(Notification.query.all()) == 0
+
+
+def test_should_delete_failed_notifications_after_seven_days(notify_db, notify_db_session):
+    created_at = datetime.utcnow() - timedelta(hours=24 * 7)
+    sample_notification(notify_db, notify_db_session, created_at=created_at, status="failed")
+    sample_notification(notify_db, notify_db_session, created_at=created_at, status="failed")
+    assert len(Notification.query.all()) == 2
+    delete_failed_notifications_created_more_than_a_week_ago()
+    assert len(Notification.query.all()) == 0
+
+
+def test_should_not_delete_sent_notifications_before_one_day(notify_db, notify_db_session):
+    expired = datetime.utcnow() - timedelta(hours=24)
+    valid = datetime.utcnow() - timedelta(hours=23, minutes=59, seconds=59)
+    sample_notification(notify_db, notify_db_session, created_at=expired, to_field="expired")
+    sample_notification(notify_db, notify_db_session, created_at=valid, to_field="valid")
+
+    assert len(Notification.query.all()) == 2
+    delete_successful_notifications_created_more_than_a_day_ago()
+    assert len(Notification.query.all()) == 1
+    assert Notification.query.first().to == 'valid'
+
+
+def test_should_not_delete_failed_notifications_before_seven_days(notify_db, notify_db_session):
+    expired = datetime.utcnow() - timedelta(hours=24 * 7)
+    valid = datetime.utcnow() - timedelta(hours=(24 * 6) + 23,  minutes=59, seconds=59)
+    sample_notification(notify_db, notify_db_session, created_at=expired, status="failed", to_field="expired")
+    sample_notification(notify_db, notify_db_session, created_at=valid, status="failed", to_field="valid")
+    assert len(Notification.query.all()) == 2
+    delete_failed_notifications_created_more_than_a_week_ago()
+    assert len(Notification.query.all()) == 1
+    assert Notification.query.first().to == 'valid'

tests/app/dao/test_users_dao.py

@@ -1,5 +1,6 @@
+from datetime import datetime, timedelta
 from sqlalchemy.exc import DataError
-
+from app import db
 import pytest
 
 from app.dao.users_dao import (
@@ -8,11 +9,12 @@ from app.dao.users_dao import (
     delete_model_user,
     increment_failed_login_count,
     reset_failed_login_count,
-    get_user_by_email
+    get_user_by_email,
+    delete_codes_older_created_more_than_a_day_ago
 )
 
 from tests.app.conftest import sample_user as create_sample_user
-from app.models import User
+from app.models import User, VerifyCode
 
 
 def test_create_user(notify_api, notify_db, notify_db_session):
@@ -86,7 +88,37 @@ def test_reset_failed_login_should_set_failed_logins_to_0(notify_api, notify_db,
     assert sample_user.failed_login_count == 0
 
 
-def test_get_user_by_email(notify_api, notify_db, notify_db_session, sample_user):
+def test_get_user_by_email(sample_user):
     email = sample_user.email_address
     user_from_db = get_user_by_email(email)
     assert sample_user == user_from_db
+
+
+def test_should_delete_all_verification_codes_more_than_one_day_old(sample_user):
+    make_verify_code(sample_user, age=timedelta(hours=24), code="54321")
+    make_verify_code(sample_user, age=timedelta(hours=24), code="54321")
+    assert len(VerifyCode.query.all()) == 2
+    delete_codes_older_created_more_than_a_day_ago()
+    assert len(VerifyCode.query.all()) == 0
+
+
+def test_should_not_delete_verification_codes_less_than_one_day_old(sample_user):
+    make_verify_code(sample_user, age=timedelta(hours=23, minutes=59, seconds=59), code="12345")
+    make_verify_code(sample_user, age=timedelta(hours=24), code="54321")
+
+    assert len(VerifyCode.query.all()) == 2
+    delete_codes_older_created_more_than_a_day_ago()
+    assert len(VerifyCode.query.all()) == 1
+    assert VerifyCode.query.first()._code == "12345"
+
+
+def make_verify_code(user, age=timedelta(hours=0), code="12335"):
+    verify_code = VerifyCode(
+        code_type='sms',
+        _code=code,
+        created_at=datetime.utcnow() - age,
+        expiry_datetime=datetime.utcnow(),
+        user=user
+    )
+    db.session.add(verify_code)
+    db.session.commit()