From 2e864411af191b54bf710a9ec4cd56a8f6c63255 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Fri, 19 May 2017 17:04:39 +0100 Subject: [PATCH 1/3] Create a server command to run custom db scripts. --- app/commands.py | 21 ++++++++++++++++++++- application.py | 1 + server_commands.py | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/app/commands.py b/app/commands.py index 197403eba..5b405ca54 100644 --- a/app/commands.py +++ b/app/commands.py @@ -2,7 +2,10 @@ import uuid from datetime import datetime from decimal import Decimal from flask.ext.script import Command, Manager, Option -from app.models import (PROVIDERS, Service, User) + + +from app import db +from app.models import (PROVIDERS, Service, User, NotificationHistory) from app.dao.services_dao import ( delete_service_and_all_associated_db_objects, dao_fetch_all_services_by_user @@ -60,3 +63,19 @@ class PurgeFunctionalTestDataCommand(Command): else: delete_user_verify_codes(usr) delete_model_user(usr) + + +class CustomDbScript(Command): + def run(self): + self.update_notification_international_flag() + + def update_notification_international_flag(self): + # 250,000 rows takes 30 seconds to update. + subq = "select id from notification_history where international is null limit 250000" + update = "update notification_history set international = False where id in ({})".format(subq) + result = db.session.execute(subq).fetchall() + while len(result) > 0: + db.session.execute(update) + print('commit 10000 updates at {}'.format(datetime.utcnow())) + db.session.commit() + result = db.session.execute(subq).fetchall() diff --git a/application.py b/application.py index 0b940dc87..b2a59a6fd 100644 --- a/application.py +++ b/application.py @@ -15,6 +15,7 @@ migrate = Migrate(application, db) manager.add_command('db', MigrateCommand) manager.add_command('create_provider_rate', commands.CreateProviderRateCommand) manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand) +manager.add_command('custom_db_script', commands.CustomDbScript) @manager.command diff --git a/server_commands.py b/server_commands.py index 6289da765..85bf13137 100644 --- a/server_commands.py +++ b/server_commands.py @@ -25,6 +25,7 @@ manager = Manager(application) migrate = Migrate(application, db) manager.add_command('db', MigrateCommand) manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand) +manager.add_command('custom_db_script', commands.CustomDbScript) if __name__ == '__main__': manager.run() From 6536177ac232b5e443cebd398548feacd55318b1 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Tue, 23 May 2017 13:35:15 +0100 Subject: [PATCH 2/3] Added the delete query for notifications --- app/commands.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/app/commands.py b/app/commands.py index 5b405ca54..d4ec5e364 100644 --- a/app/commands.py +++ b/app/commands.py @@ -71,11 +71,21 @@ class CustomDbScript(Command): def update_notification_international_flag(self): # 250,000 rows takes 30 seconds to update. - subq = "select id from notification_history where international is null limit 250000" - update = "update notification_history set international = False where id in ({})".format(subq) + subq = "select id from notifications where international is null limit 250000" + update = "update notifications set international = False where id in ({})".format(subq) result = db.session.execute(subq).fetchall() while len(result) > 0: db.session.execute(update) - print('commit 10000 updates at {}'.format(datetime.utcnow())) + print('commit 250000 updates at {}'.format(datetime.utcnow())) db.session.commit() result = db.session.execute(subq).fetchall() + + # Now update notification_history + subq_history = "select id from notification_history where international is null limit 250000" + update_history = "update notification_history set international = False where id in ({})".format(subq) + result_history = db.session.execute(subq_history).fetchall() + while len(result_history) > 0: + db.session.execute(update_history) + print('commit 250000 updates at {}'.format(datetime.utcnow())) + db.session.commit() + result_history = db.session.execute(subq_history).fetchall() From 54f7e732d9de4ad548261356a524f6f41fae3bfb Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Tue, 23 May 2017 15:36:33 +0100 Subject: [PATCH 3/3] Fix typo --- app/commands.py | 2 +- celerybeat.pid | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 celerybeat.pid diff --git a/app/commands.py b/app/commands.py index d4ec5e364..04b5186ab 100644 --- a/app/commands.py +++ b/app/commands.py @@ -82,7 +82,7 @@ class CustomDbScript(Command): # Now update notification_history subq_history = "select id from notification_history where international is null limit 250000" - update_history = "update notification_history set international = False where id in ({})".format(subq) + update_history = "update notification_history set international = False where id in ({})".format(subq_history) result_history = db.session.execute(subq_history).fetchall() while len(result_history) > 0: db.session.execute(update_history) diff --git a/celerybeat.pid b/celerybeat.pid deleted file mode 100644 index d2acb3a18..000000000 --- a/celerybeat.pid +++ /dev/null @@ -1 +0,0 @@ -9772