From b9441c528d5fa1724145232db52a4bc03ea03c8a Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Mon, 4 May 2020 17:39:13 +0100 Subject: [PATCH] Trying to find an efficient way to cycle the data --- app/commands.py | 34 ++++++++++++++++++++++++---- database_maintenance/README.md | 4 +++- database_maintenance/constraints.sql | 17 +++++++------- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/app/commands.py b/app/commands.py index a57466771..65a35b85d 100644 --- a/app/commands.py +++ b/app/commands.py @@ -904,24 +904,30 @@ def process_row_from_job(job_id, job_row_number): @notify_command() @click.option('-l', '--limit_row_count', required=True, help='Limit row count for the insert stmt') +@click.option('-s', '--start_date', required=True, help='Start') +@click.option('-e', '--end_date', required=True, help='Limit row count for the insert stmt') def cycle_notification_history_table(limit_row_count): # This relies on the notification_history_pivot table being created # and a trigger on notification_history has been created # what limit should we use here + # If this command needs to be run more than once you will need to drop nh_temp. + # Especially if the nightly task to delete notifications has run (more data has been added to notification_history) populate_temp_table = """ - SELECT id - INTO nh_temp - FROM notification_history + CREATE TABLE IF NOT EXISTS nh_temp AS SELECT id FROM notification_history """ - rows_to_insert = "SELECT COUNT(*) FROM nh_temp" + index_temp_table = """ + CREATE INDEX IF NOT EXISTS nh_temp_idx ON nh_temp (id) + """ + + rows_in_temp = "SELECT COUNT(*) FROM nh_temp" delete_temp_rows = """ DELETE FROM nh_temp t USING notification_history_pivot p WHERE t.id = p.id """ - rows_remaining = "SELECT COUNT(*) FROM nh_temp" + # In each function call, using same database connection as used for the above SQL # (needs to be in a transaction; this can be inside a stored function or in a transaction from the code) @@ -931,5 +937,23 @@ def cycle_notification_history_table(limit_row_count): FROM notification_history n, nh_temp t WHERE n.id = t.id + limit :limit_row_count """ + print("Starting cycle notification history: ", datetime.utcnow()) + db.session.execute(populate_temp_table) + db.session.execute(delete_temp_rows) + db.session.execute(index_temp_table) + rows_remaining = db.session.execute(rows_in_temp).fetchall()[0][0] + while rows_remaining > 0: + print("rows_remaining: ", rows_remaining) + db.session.execute(insert_sql, {"limit_row_count": limit_row_count}) + db.session.execute(delete_temp_rows) + db.session.commit() + + rows_remaining = db.session.execute(rows_in_temp).fetchall()[0][0] + + db.session.execute("DROP TABLE nh_temp") + db.session.commit() + print("End cycle notification history: ", datetime.utcnow()) + diff --git a/database_maintenance/README.md b/database_maintenance/README.md index 1443f8469..5421bbeb5 100644 --- a/database_maintenance/README.md +++ b/database_maintenance/README.md @@ -15,4 +15,6 @@ Add to MakeFile CF_APP=notify-cycle-history CF_SPACE=staging make generate-manifest > cycle-history-manifest.yml cf v3-create-app notify-cycle-history cf v3-apply-manifest -f cycle-history-manifest.yml -cf v3-push notify-cycle-history \ No newline at end of file +cf v3-push notify-cycle-history + +cf run-task notify-cycle-history "flask command cycle-notification-history-table -l 100000" \ No newline at end of file diff --git a/database_maintenance/constraints.sql b/database_maintenance/constraints.sql index 20bc03c9d..21c95c072 100644 --- a/database_maintenance/constraints.sql +++ b/database_maintenance/constraints.sql @@ -6,15 +6,6 @@ ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_key_t ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_service_id_fkey FOREIGN KEY (service_id) REFERENCES services(id) ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_templates_history_fkey FOREIGN KEY (template_id, template_version) REFERENCES templates_history(id, version) --- we could possibly not create this check constraint in the new table since we want to drop it in an outstanding PR. -ALTER TABLE notification_history_pivot ADD CONSTRAINT chk_notification_history_postage_null CHECK ( -CASE - WHEN notification_type = 'letter'::notification_type - THEN postage IS NOT NULL AND (postage::text = ANY (ARRAY['first'::character varying, 'second'::character varying]::text[])) - ELSE postage IS NULL -END) - - --- Create indexes after drop table since the names need to be unique --- Or rename these indexes (index names have to be unique in db) CREATE INDEX CONCURRENTLY ix_notification_history_job_id ON notification_history_pivot (job_id); @@ -22,6 +13,14 @@ CREATE INDEX CONCURRENTLY ix_notification_history_reference ON notification_hist CREATE INDEX CONCURRENTLY ix_notification_history_template_id ON notification_history_pivot (template_id); CREATE INDEX CONCURRENTLY ix_notifications_service_id_composite ON notification_history_pivot (service_id, key_type, notification_type, created_at); +-- we could possibly not create this check constraint in the new table since we want to drop it in an outstanding PR. +--ALTER TABLE notification_history_pivot ADD CONSTRAINT chk_notification_history_postage_null CHECK ( +--CASE +-- WHEN notification_type = 'letter'::notification_type +-- THEN postage IS NOT NULL AND (postage::text = ANY (ARRAY['first'::character varying, 'second'::character varying]::text[])) +-- ELSE postage IS NULL +--END) + -- Not creating these ones since we want to drop them any way. --DROP INDEX CONCURRENTLY ix_notification_history_api_key_id; --DROP INDEX CONCURRENTLY ix_notification_history_created_at;