Scheduled tasks to clean up the database

- tasks run hourly
- uses celery beat to schedule the tasks

4 new tasks
- delete verify codes (after 1 day)
- delete invitations (after 1 day)
- delete successful notifications  (after 1 day)
- delete failed notifications (after 7 days)

Delete methods in the DAO classes

tests/app/celery/test_tasks.py

@@ -1,13 +1,19 @@
 import uuid
 import pytest
 from flask import current_app
-from app.celery.tasks import (send_sms,
-                              send_sms_code,
-                              send_email_code,
-                              send_email,
-                              process_job,
-                              email_invited_user,
-                              email_reset_password)
+from app.celery.tasks import (
+    send_sms,
+    send_sms_code,
+    send_email_code,
+    send_email,
+    process_job,
+    email_invited_user,
+    email_reset_password,
+    delete_verify_codes,
+    delete_invitations,
+    delete_failed_notifications,
+    delete_successful_notifications
+)
 from app import (firetext_client, aws_ses_client, encryption, DATETIME_FORMAT)
 from app.clients.email.aws_ses import AwsSesClientException
 from app.clients.sms.firetext import FiretextClientException
@@ -22,13 +28,36 @@ from freezegun import freeze_time
 from tests.app.conftest import (
     sample_service,
     sample_user,
-    sample_template,
-    sample_template_with_placeholders
+    sample_template
 )
 
 
+def test_should_call_delete_successful_notifications_in_task(notify_api, mocker):
+    mocker.patch('app.celery.tasks.delete_successful_notifications_created_more_than_a_day_ago')
+    delete_successful_notifications()
+    assert tasks.delete_successful_notifications_created_more_than_a_day_ago.call_count == 1
+
+
+def test_should_call_delete_failed_notifications_in_task(notify_api, mocker):
+    mocker.patch('app.celery.tasks.delete_failed_notifications_created_more_than_a_week_ago')
+    delete_failed_notifications()
+    assert tasks.delete_failed_notifications_created_more_than_a_week_ago.call_count == 1
+
+
+def test_should_call_delete_codes_on_delete_verify_codes_task(notify_api, mocker):
+    mocker.patch('app.celery.tasks.delete_codes_older_created_more_than_a_day_ago')
+    delete_verify_codes()
+    assert tasks.delete_codes_older_created_more_than_a_day_ago.call_count == 1
+
+
+def test_should_call_delete_invotations_on_delete_invitations_task(notify_api, mocker):
+    mocker.patch('app.celery.tasks.delete_invitations_older_created_more_than_a_day_ago')
+    delete_invitations()
+    assert tasks.delete_invitations_older_created_more_than_a_day_ago.call_count == 1
+
+
 @freeze_time("2016-01-01 11:09:00.061258")
-def test_should_process_sms_job(sample_job, sample_template, mocker):
+def test_should_process_sms_job(sample_job, mocker):
     mocker.patch('app.celery.tasks.s3.get_job_from_s3', return_value=load_example_csv('sms'))
     mocker.patch('app.celery.tasks.send_sms.apply_async')
     mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
@@ -161,7 +190,6 @@ def test_should_send_sms_without_personalisation(sample_template, mocker):
 
 
 def test_should_send_sms_if_restricted_service_and_valid_number(notify_db, notify_db_session, mocker):
-
     user = sample_user(notify_db, notify_db_session, mobile_numnber="+441234123123")
     service = sample_service(notify_db, notify_db_session, user=user, restricted=True)
     template = sample_template(notify_db, notify_db_session, service=service)
@@ -187,7 +215,6 @@ def test_should_send_sms_if_restricted_service_and_valid_number(notify_db, notif
 
 
 def test_should_not_send_sms_if_restricted_service_and_invalid_number(notify_db, notify_db_session, mocker):
-
     user = sample_user(notify_db, notify_db_session, mobile_numnber="+441234123123")
     service = sample_service(notify_db, notify_db_session, user=user, restricted=True)
     template = sample_template(notify_db, notify_db_session, service=service)
@@ -213,7 +240,6 @@ def test_should_not_send_sms_if_restricted_service_and_invalid_number(notify_db,
 
 
 def test_should_send_email_if_restricted_service_and_valid_email(notify_db, notify_db_session, mocker):
-
     user = sample_user(notify_db, notify_db_session, email="test@restricted.com")
     service = sample_service(notify_db, notify_db_session, user=user, restricted=True)
     template = sample_template(notify_db, notify_db_session, service=service)
@@ -313,7 +339,7 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh
 
 
 def test_should_use_email_template_and_persist_without_personalisation(
-    sample_email_template, mocker
+        sample_email_template, mocker
 ):
     mocker.patch('app.encryption.decrypt', return_value={
         "template": sample_email_template.id,

tests/app/conftest.py

@@ -278,7 +278,9 @@ def sample_notification(notify_db,
                         service=None,
                         template=None,
                         job=None,
-                        to_field=None):
+                        to_field=None,
+                        status='sent',
+                        created_at=datetime.utcnow()):
     if service is None:
         service = sample_service(notify_db, notify_db_session)
     if template is None:
@@ -300,7 +302,8 @@ def sample_notification(notify_db,
         'service_id': service.id,
         'service': service,
         'template': template,
-        'created_at': datetime.utcnow()
+        'status': status,
+        'created_at': created_at
     }
     notification = Notification(**data)
     dao_create_notification(notification, template.template_type)

tests/app/dao/test_invited_user_dao.py

@@ -1,4 +1,6 @@
+from datetime import datetime, timedelta
 import uuid
+from app import db
 
 from app.models import InvitedUser
 
@@ -6,7 +8,8 @@ from app.dao.invited_user_dao import (
     save_invited_user,
     get_invited_user,
     get_invited_users_for_service,
-    get_invited_user_by_id
+    get_invited_user_by_id,
+    delete_invitations_older_created_more_than_a_day_ago
 )
 
 
@@ -52,7 +55,6 @@ def test_get_unknown_invited_user_returns_none(notify_db, notify_db_session, sam
 
 
 def test_get_invited_users_for_service(notify_db, notify_db_session, sample_service):
-
     from tests.app.conftest import sample_invited_user
     invites = []
     for i in range(0, 5):
@@ -71,7 +73,6 @@ def test_get_invited_users_for_service(notify_db, notify_db_session, sample_serv
 
 
 def test_get_invited_users_for_service_that_has_no_invites(notify_db, notify_db_session, sample_service):
-
     invites = get_invited_users_for_service(sample_service.id)
     assert len(invites) == 0
 
@@ -85,3 +86,40 @@ def test_save_invited_user_sets_status_to_cancelled(notify_db, notify_db_session
     assert InvitedUser.query.count() == 1
     cancelled_invited_user = InvitedUser.query.get(sample_invited_user.id)
     assert cancelled_invited_user.status == 'cancelled'
+
+
+def test_should_delete_all_invitations_more_than_one_day_old(
+        sample_user,
+        sample_service):
+    make_invitation(sample_user, sample_service, age=timedelta(hours=24))
+    make_invitation(sample_user, sample_service, age=timedelta(hours=24))
+    assert len(InvitedUser.query.all()) == 2
+    delete_invitations_older_created_more_than_a_day_ago()
+    assert len(InvitedUser.query.all()) == 0
+
+
+def test_should_not_delete_invitations_less_than_one_day_old(
+        sample_user,
+        sample_service):
+    make_invitation(sample_user, sample_service, age=timedelta(hours=23, minutes=59, seconds=59),
+                    email_address="valid@2.com")
+    make_invitation(sample_user, sample_service, age=timedelta(hours=24),
+                    email_address="expired@1.com")
+
+    assert len(InvitedUser.query.all()) == 2
+    delete_invitations_older_created_more_than_a_day_ago()
+    assert len(InvitedUser.query.all()) == 1
+    assert InvitedUser.query.first().email_address == "valid@2.com"
+
+
+def make_invitation(user, service, age=timedelta(hours=0), email_address="test@test.com"):
+    verify_code = InvitedUser(
+        email_address=email_address,
+        from_user=user,
+        service=service,
+        status='pending',
+        created_at=datetime.utcnow() - age,
+        permissions='manage_settings'
+    )
+    db.session.add(verify_code)
+    db.session.commit()

tests/app/dao/test_notification_dao.py

@@ -10,9 +10,12 @@ from app.dao.notifications_dao import (
     get_notification,
     get_notification_for_job,
     get_notifications_for_job,
-    dao_get_notification_statistics_for_service
+    dao_get_notification_statistics_for_service,
+    delete_successful_notifications_created_more_than_a_day_ago,
+    delete_failed_notifications_created_more_than_a_week_ago
 )
 from tests.app.conftest import sample_job
+from tests.app.conftest import sample_notification
 
 
 def test_should_be_able_to_get_statistics_for_a_service(sample_template):
@@ -489,3 +492,44 @@ def test_update_notification(sample_notification, sample_template):
     dao_update_notification(sample_notification)
     notification_from_db = Notification.query.get(sample_notification.id)
     assert notification_from_db.status == 'failed'
+
+
+def test_should_delete_sent_notifications_after_one_day(notify_db, notify_db_session):
+    created_at = datetime.utcnow() - timedelta(hours=24)
+    sample_notification(notify_db, notify_db_session, created_at=created_at)
+    sample_notification(notify_db, notify_db_session, created_at=created_at)
+    assert len(Notification.query.all()) == 2
+    delete_successful_notifications_created_more_than_a_day_ago()
+    assert len(Notification.query.all()) == 0
+
+
+def test_should_delete_failed_notifications_after_seven_days(notify_db, notify_db_session):
+    created_at = datetime.utcnow() - timedelta(hours=24 * 7)
+    sample_notification(notify_db, notify_db_session, created_at=created_at, status="failed")
+    sample_notification(notify_db, notify_db_session, created_at=created_at, status="failed")
+    assert len(Notification.query.all()) == 2
+    delete_failed_notifications_created_more_than_a_week_ago()
+    assert len(Notification.query.all()) == 0
+
+
+def test_should_not_delete_sent_notifications_before_one_day(notify_db, notify_db_session):
+    expired = datetime.utcnow() - timedelta(hours=24)
+    valid = datetime.utcnow() - timedelta(hours=23, minutes=59, seconds=59)
+    sample_notification(notify_db, notify_db_session, created_at=expired, to_field="expired")
+    sample_notification(notify_db, notify_db_session, created_at=valid, to_field="valid")
+
+    assert len(Notification.query.all()) == 2
+    delete_successful_notifications_created_more_than_a_day_ago()
+    assert len(Notification.query.all()) == 1
+    assert Notification.query.first().to == 'valid'
+
+
+def test_should_not_delete_failed_notifications_before_seven_days(notify_db, notify_db_session):
+    expired = datetime.utcnow() - timedelta(hours=24 * 7)
+    valid = datetime.utcnow() - timedelta(hours=(24 * 6) + 23,  minutes=59, seconds=59)
+    sample_notification(notify_db, notify_db_session, created_at=expired, status="failed", to_field="expired")
+    sample_notification(notify_db, notify_db_session, created_at=valid, status="failed", to_field="valid")
+    assert len(Notification.query.all()) == 2
+    delete_failed_notifications_created_more_than_a_week_ago()
+    assert len(Notification.query.all()) == 1
+    assert Notification.query.first().to == 'valid'

tests/app/dao/test_users_dao.py

@@ -1,5 +1,6 @@
+from datetime import datetime, timedelta
 from sqlalchemy.exc import DataError
-
+from app import db
 import pytest
 
 from app.dao.users_dao import (
@@ -8,11 +9,12 @@ from app.dao.users_dao import (
     delete_model_user,
     increment_failed_login_count,
     reset_failed_login_count,
-    get_user_by_email
+    get_user_by_email,
+    delete_codes_older_created_more_than_a_day_ago
 )
 
 from tests.app.conftest import sample_user as create_sample_user
-from app.models import User
+from app.models import User, VerifyCode
 
 
 def test_create_user(notify_api, notify_db, notify_db_session):
@@ -86,7 +88,37 @@ def test_reset_failed_login_should_set_failed_logins_to_0(notify_api, notify_db,
     assert sample_user.failed_login_count == 0
 
 
-def test_get_user_by_email(notify_api, notify_db, notify_db_session, sample_user):
+def test_get_user_by_email(sample_user):
     email = sample_user.email_address
     user_from_db = get_user_by_email(email)
     assert sample_user == user_from_db
+
+
+def test_should_delete_all_verification_codes_more_than_one_day_old(sample_user):
+    make_verify_code(sample_user, age=timedelta(hours=24), code="54321")
+    make_verify_code(sample_user, age=timedelta(hours=24), code="54321")
+    assert len(VerifyCode.query.all()) == 2
+    delete_codes_older_created_more_than_a_day_ago()
+    assert len(VerifyCode.query.all()) == 0
+
+
+def test_should_not_delete_verification_codes_less_than_one_day_old(sample_user):
+    make_verify_code(sample_user, age=timedelta(hours=23, minutes=59, seconds=59), code="12345")
+    make_verify_code(sample_user, age=timedelta(hours=24), code="54321")
+
+    assert len(VerifyCode.query.all()) == 2
+    delete_codes_older_created_more_than_a_day_ago()
+    assert len(VerifyCode.query.all()) == 1
+    assert VerifyCode.query.first()._code == "12345"
+
+
+def make_verify_code(user, age=timedelta(hours=0), code="12335"):
+    verify_code = VerifyCode(
+        code_type='sms',
+        _code=code,
+        created_at=datetime.utcnow() - age,
+        expiry_datetime=datetime.utcnow(),
+        user=user
+    )
+    db.session.add(verify_code)
+    db.session.commit()