Merge pull request #144 from alphagov/delete-expired-things

Delete expired things

app/celery/tasks.py

@@ -7,10 +7,18 @@ from app.dao.templates_dao import dao_get_template_by_id
 from app.dao.notifications_dao import (
     dao_create_notification,
     dao_update_notification,
+    delete_failed_notifications_created_more_than_a_week_ago,
+    delete_successful_notifications_created_more_than_a_day_ago,
     dao_get_notification_statistics_for_service_and_day
 )
 from app.dao.jobs_dao import dao_update_job, dao_get_job_by_id
-from app.models import Notification, TEMPLATE_TYPE_EMAIL, TEMPLATE_TYPE_SMS
+from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
+from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
+from app.models import (
+    Notification,
+    TEMPLATE_TYPE_EMAIL,
+    TEMPLATE_TYPE_SMS
+)
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
@@ -19,6 +27,66 @@ from utils.template import Template
 from utils.recipients import RecipientCSV
 
 
+@notify_celery.task(name="delete-verify-codes")
+def delete_verify_codes():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_codes_older_created_more_than_a_day_ago()
+        current_app.logger.info(
+            "Delete job started {} finished {} deleted {} verify codes".format(start, datetime.utcnow(), deleted)
+        )
+    except SQLAlchemyError:
+        current_app.logger.info("Failed to delete verify codes")
+        raise
+
+
+@notify_celery.task(name="delete-successful-notifications")
+def delete_successful_notifications():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_successful_notifications_created_more_than_a_day_ago()
+        current_app.logger.info(
+            "Delete job started {} finished {} deleted {} successful notifications".format(
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.info("Failed to delete successful notifications")
+        raise
+
+
+@notify_celery.task(name="delete-failed-notifications")
+def delete_failed_notifications():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_failed_notifications_created_more_than_a_week_ago()
+        current_app.logger.info(
+            "Delete job started {} finished {} deleted {} failed notifications".format(
+                start,
+                datetime.utcnow(),
+                deleted
+            )
+        )
+    except SQLAlchemyError:
+        current_app.logger.info("Failed to delete failed notifications")
+        raise
+
+
+@notify_celery.task(name="delete-invitations")
+def delete_invitations():
+    try:
+        start = datetime.utcnow()
+        deleted = delete_invitations_created_more_than_two_days_ago()
+        current_app.logger.info(
+            "Delete job started {} finished {} deleted {} invitations".format(start, datetime.utcnow(), deleted)
+        )
+    except SQLAlchemyError:
+        current_app.logger.info("Failed to delete invitations")
+        raise
+
+
 @notify_celery.task(name="process-job")
 def process_job(job_id):
     start = datetime.utcnow()
@@ -52,9 +120,9 @@ def process_job(job_id):
     )
 
     for recipient, personalisation in RecipientCSV(
-        s3.get_job_from_s3(job.bucket_name, job_id),
-        template_type=template.template_type,
-        placeholders=template.placeholders
+            s3.get_job_from_s3(job.bucket_name, job_id),
+            template_type=template.template_type,
+            placeholders=template.placeholders
     ).recipients_and_personalisation:
 
         encrypted = encryption.encrypt({

app/dao/invited_user_dao.py

@@ -1,3 +1,4 @@
+from datetime import datetime, timedelta
 from app import db
 
 from app.models import InvitedUser
@@ -18,3 +19,11 @@ def get_invited_user_by_id(invited_user_id):
 
 def get_invited_users_for_service(service_id):
     return InvitedUser.query.filter_by(service_id=service_id).all()
+
+
+def delete_invitations_created_more_than_two_days_ago():
+    deleted = db.session.query(InvitedUser).filter(
+        InvitedUser.created_at <= datetime.utcnow() - timedelta(days=2)
+    ).delete()
+    db.session.commit()
+    return deleted

app/dao/notifications_dao.py

@@ -2,7 +2,7 @@ from flask import current_app
 from app import db
 from app.models import Notification, Job, NotificationStatistics, TEMPLATE_TYPE_SMS, TEMPLATE_TYPE_EMAIL
 from sqlalchemy import desc
-from datetime import datetime
+from datetime import datetime, timedelta
 
 
 def dao_get_notification_statistics_for_service(service_id):
@@ -92,3 +92,21 @@ def get_notifications_for_service(service_id, page=1):
         per_page=current_app.config['PAGE_SIZE']
     )
     return query
+
+
+def delete_successful_notifications_created_more_than_a_day_ago():
+    deleted = db.session.query(Notification).filter(
+        Notification.created_at < datetime.utcnow() - timedelta(days=1),
+        Notification.status == 'sent'
+    ).delete()
+    db.session.commit()
+    return deleted
+
+
+def delete_failed_notifications_created_more_than_a_week_ago():
+    deleted = db.session.query(Notification).filter(
+        Notification.created_at < datetime.utcnow() - timedelta(days=7),
+        Notification.status == 'failed'
+    ).delete()
+    db.session.commit()
+    return deleted

app/dao/users_dao.py

@@ -47,6 +47,14 @@ def get_user_code(user, code, code_type):
     return retval
 
 
+def delete_codes_older_created_more_than_a_day_ago():
+    deleted = db.session.query(VerifyCode).filter(
+        VerifyCode.created_at < datetime.utcnow() - timedelta(hours=24)
+    ).delete()
+    db.session.commit()
+    return deleted
+
+
 def use_user_code(id):
     verify_code = VerifyCode.query.get(id)
     verify_code.code_used = True

app/notifications/rest.py

@@ -133,6 +133,7 @@ def send_notification(notification_type):
         service_id,
         datetime.utcnow().strftime(DATE_FORMAT)
     )
+
     if service_stats:
         total_sms_count = service_stats.sms_requested
         total_email_count = service_stats.emails_requested