Trying to find an efficient way to cycle the data

app/commands.py

@@ -904,24 +904,30 @@ def process_row_from_job(job_id, job_row_number):
 
 @notify_command()
 @click.option('-l', '--limit_row_count', required=True, help='Limit row count for the insert stmt')
+@click.option('-s', '--start_date', required=True, help='Start')
+@click.option('-e', '--end_date', required=True, help='Limit row count for the insert stmt')
 def cycle_notification_history_table(limit_row_count):
     # This relies on the notification_history_pivot table being created
     # and a trigger on notification_history has been created
     # what limit should we use here
 
+    # If this command needs to be run more than once you will need to drop nh_temp.
+    # Especially if the nightly task to delete notifications has run (more data has been added to notification_history)
     populate_temp_table = """
-        SELECT id
-        INTO nh_temp
-        FROM notification_history    
+       CREATE TABLE IF NOT EXISTS nh_temp AS SELECT id FROM notification_history
     """
-    rows_to_insert = "SELECT COUNT(*) FROM nh_temp"
+    index_temp_table = """
+        CREATE INDEX IF NOT EXISTS nh_temp_idx ON nh_temp (id)   
+    """
+
+    rows_in_temp = "SELECT COUNT(*) FROM nh_temp"
 
     delete_temp_rows = """
         DELETE FROM nh_temp t
         USING notification_history_pivot p
         WHERE t.id = p.id    
     """
-    rows_remaining = "SELECT COUNT(*) FROM nh_temp"
+
     # In each function call, using same database connection as used for the above SQL
     # (needs to be in a transaction; this can be inside a stored function or in a transaction from the code)
 
@@ -931,5 +937,23 @@ def cycle_notification_history_table(limit_row_count):
         FROM notification_history n,
              nh_temp t
         WHERE n.id = t.id    
+        limit :limit_row_count
     """
+    print("Starting cycle notification history: ", datetime.utcnow())
+    db.session.execute(populate_temp_table)
+    db.session.execute(delete_temp_rows)
+    db.session.execute(index_temp_table)
+    rows_remaining = db.session.execute(rows_in_temp).fetchall()[0][0]
+    while rows_remaining > 0:
+        print("rows_remaining: ", rows_remaining)
+        db.session.execute(insert_sql, {"limit_row_count": limit_row_count})
+        db.session.execute(delete_temp_rows)
+        db.session.commit()
+
+        rows_remaining = db.session.execute(rows_in_temp).fetchall()[0][0]
+
+    db.session.execute("DROP TABLE nh_temp")
+    db.session.commit()
+    print("End cycle notification history: ", datetime.utcnow())
+
 

database_maintenance/README.md

@@ -15,4 +15,6 @@ Add to MakeFile
 CF_APP=notify-cycle-history CF_SPACE=staging make generate-manifest > cycle-history-manifest.yml
 cf v3-create-app notify-cycle-history 
 cf v3-apply-manifest -f cycle-history-manifest.yml
-cf v3-push notify-cycle-history
+cf v3-push notify-cycle-history
+
+cf run-task notify-cycle-history "flask command cycle-notification-history-table -l 100000"

database_maintenance/constraints.sql

@@ -6,15 +6,6 @@ ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_key_t
 ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_service_id_fkey FOREIGN KEY (service_id) REFERENCES services(id)
 ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_templates_history_fkey FOREIGN KEY (template_id, template_version) REFERENCES templates_history(id, version)
 
--- we could possibly not create this check constraint in the new table since we want to drop it in an outstanding PR.
-ALTER TABLE notification_history_pivot ADD CONSTRAINT chk_notification_history_postage_null CHECK (
-CASE
-    WHEN notification_type = 'letter'::notification_type
-    THEN postage IS NOT NULL AND (postage::text = ANY (ARRAY['first'::character varying, 'second'::character varying]::text[]))
-    ELSE postage IS NULL
-END)
-
-
 --- Create indexes after drop table since the names need to be unique
 --- Or rename these indexes (index names have to be unique in db)
 CREATE INDEX CONCURRENTLY ix_notification_history_job_id ON notification_history_pivot (job_id);
@@ -22,6 +13,14 @@ CREATE INDEX CONCURRENTLY ix_notification_history_reference ON notification_hist
 CREATE INDEX CONCURRENTLY ix_notification_history_template_id ON notification_history_pivot (template_id);
 CREATE INDEX CONCURRENTLY ix_notifications_service_id_composite ON notification_history_pivot (service_id, key_type, notification_type, created_at);
 
+-- we could possibly not create this check constraint in the new table since we want to drop it in an outstanding PR.
+--ALTER TABLE notification_history_pivot ADD CONSTRAINT chk_notification_history_postage_null CHECK (
+--CASE
+--    WHEN notification_type = 'letter'::notification_type
+--    THEN postage IS NOT NULL AND (postage::text = ANY (ARRAY['first'::character varying, 'second'::character varying]::text[]))
+--    ELSE postage IS NULL
+--END)
+
 -- Not creating these ones since we want to drop them any way.
 --DROP INDEX CONCURRENTLY ix_notification_history_api_key_id;
 --DROP INDEX CONCURRENTLY ix_notification_history_created_at;