Merge pull request #2767 from alphagov/optimise-delete-high-volume

Transaction to insert history and delete notifications.

app/dao/notifications_dao.py

@@ -332,6 +332,63 @@ def delete_notifications_older_than_retention_by_type(notification_type, qry_lim
     return deleted
 
 
+@statsd(namespace="dao")
+@transactional
+def insert_notification_history_delete_notifications(
+    notification_type, service_id, start_time, end_time, qry_limit=10000
+):
+    drop_table_if_exists = """
+        DROP TABLE if exists NOTIFICATION_ARCHIVE
+        """
+    select_into_temp_table = """
+         CREATE TEMP TABLE NOTIFICATION_ARCHIVE AS
+         SELECT id
+          FROM notifications
+        WHERE service_id = :service_id
+          AND notification_type = :notification_type
+          AND created_at >= :start_time
+          AND created_at <= :end_time
+          AND key_type = 'normal'
+          AND notification_status in ('delivered', 'permanent-failure', 'temporary-failure')
+        limit :qry_limit
+        """
+    insert_query = """
+        insert into notification_history
+         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
+             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
+             client_reference, international, phone_prefix, rate_multiplier, notification_status,
+              created_by_id, postage, document_download_count
+           FROM notifications
+          WHERE id in (select id from NOTIFICATION_ARCHIVE)
+    """
+    delete_query = """
+        DELETE FROM notifications
+        where id in (select id from NOTIFICATION_ARCHIVE)
+    """
+    input_params = {
+        "service_id": service_id,
+        "notification_type": notification_type,
+        "start_time": start_time,
+        "end_time": end_time,
+        "qry_limit": qry_limit
+    }
+    current_app.logger.info(f"Start insert_notification_history_delete_notifications for input params {input_params}")
+    db.session.execute(drop_table_if_exists)
+    current_app.logger.info('Start executing select into temp table')
+    db.session.execute(select_into_temp_table, input_params)
+
+    result = db.session.execute("select * from NOTIFICATION_ARCHIVE")
+
+    current_app.logger.info('Start executing insert into history')
+    db.session.execute(insert_query)
+
+    current_app.logger.info('Start deleting notifications')
+    db.session.execute(delete_query)
+
+    db.session.execute("DROP TABLE NOTIFICATION_ARCHIVE")
+    return result.rowcount
+
+
 def _move_notifications_to_notification_history(notification_type, service_id, day_to_delete_backwards_from, qry_limit):
     deleted = 0
     if notification_type == LETTER_TYPE:

tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py

@@ -10,8 +10,8 @@ from freezegun import freeze_time
 from app.dao.notifications_dao import (
     delete_notifications_older_than_retention_by_type,
     db,
-    insert_update_notification_history
-)
+    insert_update_notification_history,
+    insert_notification_history_delete_notifications)
 from app.models import Notification, NotificationHistory
 from tests.app.db import (
     create_template,
@@ -346,3 +346,40 @@ def test_insert_update_notification_history_updates_history_with_new_status(samp
     history = NotificationHistory.query.get(notification_2.id)
     assert history.status == 'delivered'
     assert not NotificationHistory.query.get(notification_1.id)
+
+
+@freeze_time('2020-03-20 14:00')
+def test_insert_notification_history_delete_notifications(sample_email_template):
+    # should be deleted
+    create_notification(template=sample_email_template,
+                        created_at=datetime.utcnow() + timedelta(minutes=4), status='delivered')
+    create_notification(template=sample_email_template,
+                        created_at=datetime.utcnow() + timedelta(minutes=20), status='permanent-failure')
+    create_notification(template=sample_email_template,
+                        created_at=datetime.utcnow() + timedelta(minutes=30), status='temporary-failure')
+
+    # should NOT be deleted
+    create_notification(template=sample_email_template,
+                        created_at=datetime.utcnow() - timedelta(minutes=59), status='temporary-failure')
+    create_notification(template=sample_email_template,
+                        created_at=datetime.utcnow() + timedelta(hours=1, seconds=1), status='temporary-failure')
+    create_notification(template=sample_email_template,
+                        created_at=datetime.utcnow() - timedelta(hours=1), status='delivered')
+    create_notification(template=sample_email_template,
+                        created_at=datetime.utcnow() + timedelta(minutes=20), status='created')
+    create_notification(template=sample_email_template,
+                        created_at=datetime.utcnow() - timedelta(days=1), status='sending')
+    create_notification(template=sample_email_template,
+                        created_at=datetime.utcnow() - timedelta(days=1), status='technical-failure')
+
+    del_count = insert_notification_history_delete_notifications(
+        notification_type=sample_email_template.template_type,
+        service_id=sample_email_template.service_id,
+        start_time=datetime.utcnow(),
+        end_time=datetime.utcnow() + timedelta(hours=1))
+
+    assert del_count == 3
+    notifications = Notification.query.all()
+    history_rows = NotificationHistory.query.all()
+    assert len(history_rows) == 3
+    assert len(notifications) == 6