We want to staop inserting and updating NotificationHistory each time we insert/update Notification.

This PR adds a function to upsert (insert or update if exists) NotificationHistory all the rows from Notification that we are about to delete in the nightly task. This will happen just before the delete function. Since it is a upsert query the function can be called more than once.
This should allow us remove all the insert/updates to NotificationHistory.

However, there is a consern that this will double the length of time the tasks take. So do we do these upserts in a separate task or in the same one?

app/dao/notifications_dao.py

@@ -19,6 +19,7 @@ from sqlalchemy import (desc, func, asc)
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql import functions
 from sqlalchemy.sql.expression import case
+from sqlalchemy.dialects.postgresql import insert
 from werkzeug.datastructures import MultiDict
 
 from app import db, create_uuid
@@ -310,6 +311,8 @@ def delete_notifications_older_than_retention_by_type(notification_type, qry_lim
                 notification_type, f.service_id, days_of_retention, qry_limit
             )
 
+        insert_update_notification_history(notification_type, days_of_retention, f.service_id)
+
         current_app.logger.info(
             "Deleting {} notifications for service id: {}".format(notification_type, f.service_id))
         deleted += _delete_notifications(
@@ -328,7 +331,7 @@ def delete_notifications_older_than_retention_by_type(notification_type, qry_lim
             _delete_letters_from_s3(
                 notification_type, service_id, seven_days_ago, qry_limit
             )
-
+        insert_update_notification_history(notification_type, seven_days_ago, service_id)
         deleted += _delete_notifications(
             deleted, notification_type, seven_days_ago, service_id, qry_limit
         )
@@ -361,6 +364,54 @@ def _delete_notifications(
     return deleted
 
 
+def insert_update_notification_history(notification_type, date_to_delete_from, service_id):
+    notifications = db.session.query(
+        Notification.id,
+        Notification.job_id,
+        Notification.job_row_number,
+        Notification.service_id,
+        Notification.template_id,
+        Notification.template_version,
+        Notification.api_key_id,
+        Notification.key_type,
+        Notification.billable_units,
+        Notification.notification_type,
+        Notification.created_at,
+        Notification.sent_at,
+        Notification.sent_by,
+        Notification.updated_at,
+        Notification.status,
+        Notification.reference,
+        Notification.client_reference,
+        Notification.international,
+        Notification.phone_prefix,
+        Notification.rate_multiplier,
+        Notification.created_by_id,
+        Notification.postage
+    ).filter(
+        Notification.notification_type == notification_type,
+        Notification.service_id == service_id,
+        Notification.created_at < date_to_delete_from,
+        Notification.key_type != KEY_TYPE_TEST
+    ).all()
+
+    stmt = insert(NotificationHistory).values(
+        notifications
+    )
+
+    stmt = stmt.on_conflict_do_update(
+        constraint="notification_history_pkey",
+        set_={"notification_status": stmt.excluded.status,
+              "billable_units": stmt.excluded.billable_units,
+              "updated_at": stmt.excluded.updated_at,
+              "sent_at": stmt.excluded.sent_at,
+              "sent_by": stmt.excluded.sent_by
+              }
+    )
+    db.session.connection().execute(stmt)
+    db.session.commit()
+
+
 def _delete_letters_from_s3(
         notification_type, service_id, date_to_delete_from, query_limit
 ):