Update with timings.

This branch is likely to be closed as we can more easily perform an upgrade than cycle the notification_history table.

app/commands.py

@@ -900,3 +900,75 @@ def process_row_from_job(job_id, job_row_number):
             notification_id = process_row(row, template, job, job.service)
             current_app.logger.info("Process row {} for job {} created notification_id: {}".format(
                 job_row_number, job_id, notification_id))
+
+
+@notify_command()
+@click.option('-l', '--limit_row_count', required=True, help='Limit row count for the insert stmt')
+@click.option('-s', '--start_date', required=True, help='Start date', type=click_dt(format='%Y-%m-%d %H:%M'))
+@click.option('-e', '--end_date', required=True, help='End date', type=click_dt(format='%Y-%m-%d %H:%M'))
+def cycle_notification_history_table(limit_row_count, start_date, end_date):
+    print(f"Starting cycle_notification_history_table for {start_date} to {end_date}")
+    day_start = start_date
+    day_end = start_date + timedelta(days=1)
+
+    while day_end < end_date:
+        populate_notification_history_pivot(limit_row_count, day_start, day_end)
+        day_start = day_start + timedelta(days=1)
+        day_end = day_end + timedelta(days=1)
+
+
+def populate_notification_history_pivot(limit_row_count, start_date, end_date):
+    # This relies on the notification_history_pivot table being created
+    # and a trigger on notification_history has been created
+    # what limit should we use here
+
+    # If this command needs to be run more than once you will need to drop nh_temp.
+    # Especially if the nightly task to delete notifications has run (more data has been added to notification_history)
+    populate_temp_table = """
+       CREATE TABLE IF NOT EXISTS nh_temp AS SELECT id FROM notification_history 
+       where created_at >= :start_date
+       and created_at < :end_date
+    """
+    index_temp_table = """
+        CREATE INDEX IF NOT EXISTS nh_temp_idx ON nh_temp (id)   
+    """
+
+    rows_in_temp = "SELECT COUNT(*) FROM nh_temp"
+
+    delete_temp_rows = """
+        DELETE FROM nh_temp t
+        USING notification_history_pivot p
+        WHERE t.id = p.id    
+    """
+
+    # In each function call, using same database connection as used for the above SQL
+    # (needs to be in a transaction; this can be inside a stored function or in a transaction from the code)
+
+    insert_sql = """
+        INSERT INTO notification_history_pivot
+        SELECT n.*
+        FROM notification_history n,
+             nh_temp t
+        WHERE n.id = t.id    
+        limit :limit_row_count
+    """
+    print(f"Starting cycle notification history for start_date: {start_date} and "
+          f"end_date {end_date} and limit: {limit_row_count}")
+
+    db.session.execute(populate_temp_table, {"start_date": start_date, "end_date": end_date})
+    db.session.execute(delete_temp_rows)
+    db.session.execute(index_temp_table)
+    rows_remaining = db.session.execute(rows_in_temp).fetchall()[0][0]
+    while rows_remaining > 0:
+        print("rows_remaining: ", rows_remaining)
+        db.session.execute(insert_sql, {"limit_row_count": limit_row_count})
+        db.session.execute(delete_temp_rows)
+        db.session.commit()
+
+        rows_remaining = db.session.execute(rows_in_temp).fetchall()[0][0]
+
+    db.session.execute("DROP TABLE nh_temp")
+    db.session.commit()
+    print("End cycle notification history: ", datetime.utcnow())
+
+

app/dao/notifications_dao.py

@@ -372,6 +372,14 @@ def insert_notification_history_delete_notifications(
           ON CONFLICT ON CONSTRAINT notification_history_pkey
           DO NOTHING
     """
+    # Insert data into the pivot table as well. Only needed while we are trying to cycle the history table.
+    insert_pivot_query = """
+           insert into notification_history_pivot
+            SELECT * from NOTIFICATION_ARCHIVE
+             ON CONFLICT ON CONSTRAINT notification_history_pivot_pkey
+             DO NOTHING
+       """
+
     delete_query = """
         DELETE FROM notifications
         where id in (select id from NOTIFICATION_ARCHIVE)
@@ -390,6 +398,8 @@ def insert_notification_history_delete_notifications(
 
     db.session.execute(insert_query)
 
+    db.session.execute(insert_pivot_query)
+
     db.session.execute(delete_query)
 
     db.session.execute("DROP TABLE NOTIFICATION_ARCHIVE")

database_maintenance/README.md

@@ -0,0 +1,40 @@
+Create a new app to test command
+
+DO NOT commit new manifest file
+
+Add to MakeFile
+'notify-cycle-history': {
+    'NOTIFY_APP_NAME': 'api',
+    'instances': {
+      'preview': 0,
+      'staging': 0,
+      'production': 0
+    },
+  },
+
+CF_APP=notify-cycle-history CF_SPACE=staging make generate-manifest > cycle-history-manifest.yml
+cf v3-create-app notify-cycle-history 
+cf v3-apply-manifest -f cycle-history-manifest.yml
+cf v3-push notify-cycle-history
+
+cf run-task notify-cycle-history "flask command cycle-notification-history-table -l 100000 -s '2020-03-18 00:00' -e '2020-03-19 00:00"
+  
+
+
+
+- Deploy new command - cycle_notification_history_table to prod
+- script_1.sql
+    - create nh_pivot
+    - create nh trigger
+    - create new index on nh
+- foreign_keys.sql
+- run command for each day in nh --> start October 1, 2019
+-- all data in NH
+- 
+
+2020-05-05T14:35:47.18+0100 [APP/TASK/6a5ea53b/0] OUT Starting cycle notification history for start_date: 2020-03-19 00:00:00 and end_date 2020-03-29 00:00:00 and limit: 100000
+2020-05-05T14:41:21.16+0100 [APP/TASK/6a5ea53b/0] OUT rows_remaining:  67800240
+2020-05-06T02:48:36.84+0100 [APP/TASK/6a5ea53b/0] OUT End cycle notification history:  2020-05-06 01:48:36.849611
+
+
+5,650,020

database_maintenance/constraints.sql

@@ -0,0 +1,30 @@
+
+ALTER TABLE notification_history_pivot ADD CONSTRAINT fk_notification_history_notification_status FOREIGN KEY (notification_status) REFERENCES notification_status_types(name)
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_api_key_id_fkey FOREIGN KEY (api_key_id) REFERENCES api_keys(id)
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_job_id_fkey FOREIGN KEY (job_id) REFERENCES jobs(id)
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_key_type_fkey FOREIGN KEY (key_type) REFERENCES key_types(name)
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_service_id_fkey FOREIGN KEY (service_id) REFERENCES services(id)
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_templates_history_fkey FOREIGN KEY (template_id, template_version) REFERENCES templates_history(id, version)
+
+--- Create indexes after drop table since the names need to be unique
+--- Or rename these indexes (index names have to be unique in db)
+CREATE INDEX CONCURRENTLY ix_notification_history_job_id ON notification_history_pivot (job_id);
+CREATE INDEX CONCURRENTLY ix_notification_history_reference ON notification_history_pivot (reference);
+CREATE INDEX CONCURRENTLY ix_notification_history_template_id ON notification_history_pivot (template_id);
+CREATE INDEX CONCURRENTLY ix_notifications_service_id_composite ON notification_history_pivot (service_id, key_type, notification_type, created_at);
+
+-- we could possibly not create this check constraint in the new table since we want to drop it in an outstanding PR.
+--ALTER TABLE notification_history_pivot ADD CONSTRAINT chk_notification_history_postage_null CHECK (
+--CASE
+--    WHEN notification_type = 'letter'::notification_type
+--    THEN postage IS NOT NULL AND (postage::text = ANY (ARRAY['first'::character varying, 'second'::character varying]::text[]))
+--    ELSE postage IS NULL
+--END)
+
+-- Not creating these ones since we want to drop them any way.
+--DROP INDEX CONCURRENTLY ix_notification_history_api_key_id;
+--DROP INDEX CONCURRENTLY ix_notification_history_created_at;
+--DROP INDEX CONCURRENTLY ix_notification_history_notification_status;
+--DROP INDEX CONCURRENTLY ix_notification_history_notification_type;
+--DROP INDEX CONCURRENTLY ix_notification_history_service_id;
+--DROP INDEX CONCURRENTLY ix_notification_history_service_id_created_at;

database_maintenance/cycle_notification_history.sql

@@ -0,0 +1,88 @@
+-- Stage 1 - manual process
+
+-- Create new notification_history table and primary key.
+CREATE TABLE notification_history_pivot AS SELECT * FROM notification_history WHERE 1=2;
+ALTER TABLE notification_history_pivot ADD PRIMARY KEY (id);
+
+-- Update values of notification_status, billable_units, updated_at, sent_by, sent_at based on new updates coming into the notification_history table.
+CREATE OR REPLACE FUNCTION update_pivot_table()
+RETURNS TRIGGER
+LANGUAGE plpgsql AS 
+$$
+BEGIN
+    UPDATE notification_history_pivot SET   notification_status = NEW.notification_status,
+                                            billable_units = NEW.billable_units,
+                                            updated_at = NEW.updated_at,
+                                            sent_by = NEW.sent_by,
+                                            sent_at = NEW.sent_at
+    WHERE notification_history_pivot.id = NEW.id;
+
+    RETURN NEW;
+END
+$$;
+
+DROP TRIGGER IF EXISTS update_pivot on notification_history;
+-- Following may be blocked if running vacuum.
+-- SELECT pg_cancel_backend(pid);
+CREATE TRIGGER update_pivot AFTER UPDATE OF notification_status, billable_units, updated_at, sent_by, sent_at ON notification_history
+FOR EACH ROW
+  EXECUTE PROCEDURE update_pivot_table();
+
+
+-- Create foreign key constraints in notification_history_pivot using same names as used in notification_history
+ALTER TABLE notification_history_pivot ADD CONSTRAINT fk_notification_history_notification_status FOREIGN KEY (notification_status) REFERENCES notification_status_types(name);
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_api_key_id_fkey FOREIGN KEY (api_key_id) REFERENCES api_keys(id);
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_job_id_fkey FOREIGN KEY (job_id) REFERENCES jobs(id);
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_key_type_fkey FOREIGN KEY (key_type) REFERENCES key_types(name);
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_service_id_fkey FOREIGN KEY (service_id) REFERENCES services(id);
+ALTER TABLE notification_history_pivot ADD CONSTRAINT notification_history_templates_history_fkey FOREIGN KEY (template_id, template_version) REFERENCES templates_history(id, version);
+
+-- Index used for data population
+create index CONCURRENTLY created_id_nh on notification_history (created_at, id);
+
+-----
+
+-- Stage 2 - automated process
+
+cf run-task notify-cycle-history "flask command cycle-notification-history-table -l 100000 -s '2019-10-01 00:00' -e '2019-12-01 00:00'"
+cf run-task notify-cycle-history "flask command cycle-notification-history-table -l 100000 -s '2020-03-19 00:00' -e '2020-03-29 00:00'"
+ ... etc emd = May 11
+ ... On Monday - run again to get last nights inserts
+
+-- Once this Python process has completed, then,
+
+-- Stage 3 - manual process
+
+-- Run basic sanity checks on data in notification_history_pivot, ensuring same number of entries present in notification_history and notification_history_pivot
+SELECT COUNT(*) FROM notification_history_pivot;
+SELECT COUNT(*) FROM notification_history where created_at >= '2019-10-01 00:00';
+-- Ensure these counts match.
+
+ALTER INDEX ix_notification_history_job_id RENAME TO ix_notification_history_job_id_old;
+ALTER INDEX ix_notification_history_reference RENAME TO ix_notification_history_reference_old;
+ALTER INDEX ix_notification_history_template_id RENAME TO ix_notification_history_template_id_old;
+
+CREATE INDEX CONCURRENTLY ix_notification_history_job_id ON notification_history_pivot (job_id);
+CREATE INDEX CONCURRENTLY ix_notification_history_reference ON notification_history_pivot (reference);
+CREATE INDEX CONCURRENTLY ix_notification_history_template_id ON notification_history_pivot (template_id);
+CREATE INDEX CONCURRENTLY ix_notifications_service_id_composite ON notification_history_pivot (service_id, key_type, notification_type, created_at);
+
+ALTER TABLE notification_history RENAME TO notification_history_old;
+ALTER TABLE notification_history_pivot RENAME TO notification_history;
+
+
+
+-- When sure data in new notification_history table are ok:
+-- This can be done later: DROP TABLE notification_history_old;
+
+-- Could rename primary key on new notification_history table.
+ALTER TABLE notification_history DROP CONSTRAINT notification_history_pivot_pkey;
+ALTER TABLE notification_history ADD PRIMARY KEY (id); -- Will create a new primary key named "notification_history_pkey".
+
+-- Create remaining indices on notification_history_pivot
+--- Create indexes after drop table since the names need to be unique
+--- Or rename these indexes (index names have to be unique in db)
+CREATE INDEX CONCURRENTLY ix_notification_history_job_id ON notification_history (job_id);
+CREATE INDEX CONCURRENTLY ix_notification_history_reference ON notification_history (reference);
+CREATE INDEX CONCURRENTLY ix_notification_history_template_id ON notification_history (template_id);
+CREATE INDEX CONCURRENTLY ix_notifications_service_id_composite ON notification_history (service_id, key_type, notification_type, created_at);

database_maintenance/cycle_notification_history_in_batches.sql

@@ -0,0 +1,33 @@
+-- Call once at the start of the process
+
+create index created_id_nh on notification_history (created_at, id);
+
+CREATE TABLE notification_history_pivot AS SELECT * from notification_history WHERE 1=2;
+
+CREATE TEMPORARY TABLE nh_temp AS SELECT id FROM notification_history;
+
+SELECT COUNT(*) AS "Total number of rows in nh" FROM nh_temp;
+
+DELETE FROM nh_temp t
+USING notification_history_pivot p
+WHERE t.id = p.id;
+
+SELECT COUNT(*) AS "Number of rows remaining that need moving across from nh to nh_pivot" FROM nh_temp;
+
+CREATE INDEX nh_temp_idx ON nh_temp (id);
+
+-- In each function call, using same database connection as used for the above SQL (needs to be in a transaction; this can be inside a stored function or in a transaction from the code)
+
+INSERT INTO notification_history_pivot
+SELECT n.*
+FROM notification_history n,
+     nh_temp t
+WHERE n.id = t.id;
+
+DELETE FROM nh_temp t
+USING notification_history_pivot p
+WHERE t.id = p.id;
+
+SELECT COUNT(*) from nh_temp;
+
+-- Loop until this result is zero.

migrations/versions/0321_notification_history_pivot.py

@@ -0,0 +1,23 @@
+"""
+
+Revision ID: 0321_notification_history_pivot
+Revises: 0320_optimise_notifications
+Create Date: 2020-03-26 11:16:12.389524
+
+"""
+import os
+
+from alembic import op
+
+revision = '0321_notification_history_pivot'
+down_revision = '0320_optimise_notifications'
+environment = os.environ['NOTIFY_ENVIRONMENT']
+
+
+def upgrade():
+    op.execute('CREATE TABLE notification_history_pivot AS SELECT * FROM notification_history WHERE 1=2')
+    op.execute('ALTER TABLE notification_history_pivot ADD PRIMARY KEY (id)')
+
+
+def downgrade():
+    op.execute('DROP TABLE notifications_history_pivot')