Scheduled tasks to clean up the database

- tasks run hourly
- uses celery beat to schedule the tasks

4 new tasks
- delete verify codes (after 1 day)
- delete invitations (after 1 day)
- delete successful notifications  (after 1 day)
- delete failed notifications (after 7 days)

Delete methods in the DAO classes

app/celery/tasks.py

@@ -4,9 +4,20 @@ from app.clients.email.aws_ses import AwsSesClientException
 from app.clients.sms.firetext import FiretextClientException
 from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
-from app.dao.notifications_dao import dao_create_notification, dao_update_notification
+from app.dao.notifications_dao import (
+    dao_create_notification,
+    dao_update_notification,
+    delete_failed_notifications_created_more_than_a_week_ago,
+    delete_successful_notifications_created_more_than_a_day_ago
+)
 from app.dao.jobs_dao import dao_update_job, dao_get_job_by_id
-from app.models import Notification, TEMPLATE_TYPE_EMAIL, TEMPLATE_TYPE_SMS
+from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
+from app.dao.invited_user_dao import delete_invitations_older_created_more_than_a_day_ago
+from app.models import (
+    Notification,
+    TEMPLATE_TYPE_EMAIL,
+    TEMPLATE_TYPE_SMS
+)
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
@@ -15,11 +26,64 @@ from utils.template import Template
 from utils.recipients import RecipientCSV
 
 
-@notify_celery.task(name="log_this")
-def do_test():
-    current_app.logger.info(
-        "here"
-    )
+@notify_celery.task(name="delete-verify-codes")
+def delete_verify_codes():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_codes_older_created_more_than_a_day_ago()
+        current_app.logger.info(
+            "Delete job started {} finished {} deleted {} verify codes".format(start, datetime.utcnow(), deleted)
+        )
+    except SQLAlchemyError:
+        current_app.logger.info("Failed to delete verify codes")
+        raise
+
+
+@notify_celery.task(name="delete-successful-notifications")
+def delete_successful_notifications():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_successful_notifications_created_more_than_a_day_ago()
+        current_app.logger.info(
+            "Delete job started {} finished {} deleted {} successful notifications".format(
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.info("Failed to delete successful notifications")
+        raise
+
+
+@notify_celery.task(name="delete-failed-notifications")
+def delete_failed_notifications():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_failed_notifications_created_more_than_a_week_ago()
+        current_app.logger.info(
+            "Delete job started {} finished {} deleted {} failed notifications".format(
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.info("Failed to delete failed notifications")
+        raise
+
+
+@notify_celery.task(name="delete-invitations")
+def delete_invitations():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_invitations_older_created_more_than_a_day_ago()
+        current_app.logger.info(
+            "Delete job started {} finished {} deleted {} invitations".format(start, datetime.utcnow(), deleted)
+        )
+    except SQLAlchemyError:
+        current_app.logger.info("Failed to delete invitations")
+        raise
 
 
 @notify_celery.task(name="process-job")

app/dao/invited_user_dao.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 from app import db
 
 from app.models import InvitedUser
@@ -18,3 +19,11 @@ def get_invited_user_by_id(invited_user_id):
 
 def get_invited_users_for_service(service_id):
     return InvitedUser.query.filter_by(service_id=service_id).all()
+
+
+def delete_invitations_older_created_more_than_a_day_ago():
+    deleted = db.session.query(InvitedUser).filter(
+        InvitedUser.created_at <= datetime.utcnow() - timedelta(days=1)
+    ).delete()
+    db.session.commit()
+    return deleted

app/dao/notifications_dao.py

@@ -2,7 +2,7 @@ from flask import current_app
 from app import db
 from app.models import Notification, Job, NotificationStatistics, TEMPLATE_TYPE_SMS, TEMPLATE_TYPE_EMAIL
 from sqlalchemy import desc
-from datetime import datetime
+from datetime import datetime, timedelta
 
 
 def dao_get_notification_statistics_for_service(service_id):
@@ -85,3 +85,21 @@ def get_notifications_for_service(service_id, page=1):
         per_page=current_app.config['PAGE_SIZE']
     )
     return query
+
+
+def delete_successful_notifications_created_more_than_a_day_ago():
+    deleted = db.session.query(Notification).filter(
+        Notification.created_at < datetime.utcnow() - timedelta(hours=24),
+        Notification.status == 'sent'
+    ).delete()
+    db.session.commit()
+    return deleted
+
+
+def delete_failed_notifications_created_more_than_a_week_ago():
+    deleted = db.session.query(Notification).filter(
+        Notification.created_at < datetime.utcnow() - timedelta(hours=24 * 7),
+        Notification.status == 'failed'
+    ).delete()
+    db.session.commit()
+    return deleted

app/dao/users_dao.py

@@ -47,6 +47,14 @@ def get_user_code(user, code, code_type):
     return retval
 
 
+def delete_codes_older_created_more_than_a_day_ago():
+    deleted = db.session.query(VerifyCode).filter(
+        VerifyCode.created_at < datetime.utcnow() - timedelta(hours=24)
+    ).delete()
+    db.session.commit()
+    return deleted
+
+
 def use_user_code(id):
     verify_code = VerifyCode.query.get(id)
     verify_code.code_used = True