From 35136704086b5ffa43e5b9d2b2ac88b85f43c167 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Mon, 8 May 2017 16:40:56 +0100 Subject: [PATCH 1/4] Script to update the international flag in the notifications and notification_history tables. This script may move where it is not run on db migration but as a one-off script from a server command. --- .../set_notifications_international.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 migrations/versions/set_notifications_international.py diff --git a/migrations/versions/set_notifications_international.py b/migrations/versions/set_notifications_international.py new file mode 100644 index 000000000..e476fae4e --- /dev/null +++ b/migrations/versions/set_notifications_international.py @@ -0,0 +1,37 @@ +"""empty message + +Revision ID: set_notifications_international +Revises: 0080_fix_rate_start_date +Create Date: 2017-05-05 15:26:34.621670 + +""" + +# revision identifiers, used by Alembic. +from datetime import datetime + +revision = 'set_notifications_international' +down_revision = '0080_fix_rate_start_date' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + conn = op.get_bind() + start = datetime.utcnow() + all_notifications = "select id from notifications where international is null limit 1000" + + results = conn.execute(all_notifications) + res = results.fetchall() + while len(res) > 0: + conn.execute("update notifications set international = False where id in ({})".format(all_notifications)) + conn.execute("update notification_history set international = False where id in ({})".format(all_notifications)) + conn.commit() + results = conn.execute(all_notifications) + res = results.fetchall() + end = datetime.utcnow() + print("Started at: {} ended at: {}".format(start, end)) + +def downgrade(): + # There is no way to downgrade this update. + pass \ No newline at end of file From f0a5851d73b76f3a59e2c15d7498ab074239eeaa Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 10 May 2017 14:02:11 +0100 Subject: [PATCH 2/4] The script updates 10,000 rows at a time, committing after each one. This should limit the distrubtion to the environment as only the rows being updated will be locked. It is likely those rows will not have a conflicting update at the same time since the records are older than 3 days. --- ...ons_international.py => 0082_set_international.py} | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) rename migrations/versions/{set_notifications_international.py => 0082_set_international.py} (69%) diff --git a/migrations/versions/set_notifications_international.py b/migrations/versions/0082_set_international.py similarity index 69% rename from migrations/versions/set_notifications_international.py rename to migrations/versions/0082_set_international.py index e476fae4e..97d3f5881 100644 --- a/migrations/versions/set_notifications_international.py +++ b/migrations/versions/0082_set_international.py @@ -1,6 +1,6 @@ """empty message -Revision ID: set_notifications_international +Revision ID: 0082_set_international Revises: 0080_fix_rate_start_date Create Date: 2017-05-05 15:26:34.621670 @@ -9,7 +9,7 @@ Create Date: 2017-05-05 15:26:34.621670 # revision identifiers, used by Alembic. from datetime import datetime -revision = 'set_notifications_international' +revision = '0082_set_international' down_revision = '0080_fix_rate_start_date' from alembic import op @@ -19,14 +19,17 @@ import sqlalchemy as sa def upgrade(): conn = op.get_bind() start = datetime.utcnow() - all_notifications = "select id from notifications where international is null limit 1000" + all_notifications = "select id from notification_history where international is null limit 10000" results = conn.execute(all_notifications) res = results.fetchall() + + conn.execute("update notifications set international = False where id in ({})".format(all_notifications)) + conn.execute("update notification_history set international = False where id in ({})".format(all_notifications)) + while len(res) > 0: conn.execute("update notifications set international = False where id in ({})".format(all_notifications)) conn.execute("update notification_history set international = False where id in ({})".format(all_notifications)) - conn.commit() results = conn.execute(all_notifications) res = results.fetchall() end = datetime.utcnow() From d1e9586fbbadd8278d1d4023490df3348915b217 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 10 May 2017 14:38:21 +0100 Subject: [PATCH 3/4] Update the script to set the international flag to do the notifications and notification_history in separate loops. It takes about 1.5 minutes to update 27,000 notifications and 27,000 notification_history. The update is a row level lock so will only affect updates to the same row. This is unlikely as the data being updated should be older than 3 days. The second scripts updates the table to set international as not null, to make the model. --- migrations/versions/0082_set_international.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/migrations/versions/0082_set_international.py b/migrations/versions/0082_set_international.py index 97d3f5881..fa89bd2f7 100644 --- a/migrations/versions/0082_set_international.py +++ b/migrations/versions/0082_set_international.py @@ -1,40 +1,45 @@ """empty message Revision ID: 0082_set_international -Revises: 0080_fix_rate_start_date +Revises: 0081_noti_status_as_enum Create Date: 2017-05-05 15:26:34.621670 """ +from datetime import datetime +from alembic import op # revision identifiers, used by Alembic. -from datetime import datetime - revision = '0082_set_international' -down_revision = '0080_fix_rate_start_date' - -from alembic import op -import sqlalchemy as sa +down_revision = '0081_noti_status_as_enum' def upgrade(): conn = op.get_bind() start = datetime.utcnow() - all_notifications = "select id from notification_history where international is null limit 10000" + notification_history = "select id from notification_history where international is null limit 10000" - results = conn.execute(all_notifications) + results = conn.execute(notification_history) res = results.fetchall() - conn.execute("update notifications set international = False where id in ({})".format(all_notifications)) - conn.execute("update notification_history set international = False where id in ({})".format(all_notifications)) - while len(res) > 0: - conn.execute("update notifications set international = False where id in ({})".format(all_notifications)) - conn.execute("update notification_history set international = False where id in ({})".format(all_notifications)) - results = conn.execute(all_notifications) + conn.execute("update notification_history set international = False where id in ({})".format( + notification_history)) + results = conn.execute(notification_history) res = results.fetchall() + + notifications = "select id from notifications where international is null limit 10000" + results2 = conn.execute(notifications) + res2 = results2.fetchall() + while len(res2) > 0: + conn.execute("update notifications set international = False where id in ({})".format(notifications)) + + results2 = conn.execute(notifications) + res2 = results2.fetchall() + end = datetime.utcnow() print("Started at: {} ended at: {}".format(start, end)) + def downgrade(): # There is no way to downgrade this update. - pass \ No newline at end of file + pass From ed4a8fba6c80be7d4611cfbafef158adb56e2122 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 10 May 2017 16:14:30 +0100 Subject: [PATCH 4/4] Script to update the international column to be not null --- .../0083_set_international_not_null.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 migrations/versions/0083_set_international_not_null.py diff --git a/migrations/versions/0083_set_international_not_null.py b/migrations/versions/0083_set_international_not_null.py new file mode 100644 index 000000000..da3a7491b --- /dev/null +++ b/migrations/versions/0083_set_international_not_null.py @@ -0,0 +1,31 @@ +"""empty message + +Revision ID: 0083_set_international_not_null +Revises: 0082_set_international +Create Date: 2017-05-10 14:08:51.067762 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '0083_set_international_not_null' +down_revision = '0082_set_international' + + +def upgrade(): + op.alter_column('notification_history', 'international', + existing_type=sa.BOOLEAN(), + nullable=False) + op.alter_column('notifications', 'international', + existing_type=sa.BOOLEAN(), + nullable=False) + + +def downgrade(): + op.alter_column('notifications', 'international', + existing_type=sa.BOOLEAN(), + nullable=True) + op.alter_column('notification_history', 'international', + existing_type=sa.BOOLEAN(), + nullable=True)