diff --git a/Makefile b/Makefile index 732d669a4..dff7a7b69 100644 --- a/Makefile +++ b/Makefile @@ -287,7 +287,7 @@ cf-deploy-api-db-migration: cf unbind-service notify-api-db-migration notify-config cf unbind-service notify-api-db-migration notify-aws cf push notify-api-db-migration -f manifest-api-${CF_SPACE}.yml - cf run-task notify-api-db-migration "python db.py db upgrade" --name api_db_migration + cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task @@ -310,4 +310,3 @@ cf-push: .PHONY: check-if-migrations-to-run check-if-migrations-to-run: @echo $(shell python3 scripts/check_if_new_migration.py) - diff --git a/README.md b/README.md index 8252c9bbf..dce3dbad6 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,9 @@ export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY' export STATSD_PREFIX='YOU_OWN_PREFIX' export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX' export REDIS_URL="redis://localhost:6379/0" +export FLASK_APP=application.py +export FLASK_DEBUG=1 +export WERKZEUG_DEBUG_PIN=off "> environment.sh ``` @@ -102,17 +105,20 @@ That will run pycodestyle for code analysis and our unit test suite. If you wish -## To remove functional test data +## To run one off tasks -NOTE: There is assumption that both the server name prefix and user name prefix are followed by a uuid. -The script will search for all services/users with that prefix and only remove it if the prefix is followed by a uuid otherwise it will be skipped. +Tasks are run through the `flask` command - run `flask --help` for more information. There are two sections we need to +care about: `flask db` contains alembic migration commands, and `flask command` contains all of our custom commands. For +example, to purge all dynamically generated functional test data, do the following: Locally ``` -python application.py purge_functional_test_data -u # Remove the user and associated services. +flask command purge_functional_test_data -u ``` On the server ``` -python server_commands.py purge_functional_test_data -u # Remove the user and associated services. +cf run-task notify-api "flask command purge_functional_test_data -u " ``` + +All commands and command options have a --help command if you need more information. diff --git a/app/__ b/app/__ new file mode 100644 index 000000000..e69de29bb diff --git a/app/__init__.py b/app/__init__.py index d4e9c8136..a1172736e 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -6,6 +6,7 @@ import uuid from flask import Flask, _request_ctx_stack, request, g, jsonify from flask_sqlalchemy import SQLAlchemy from flask_marshmallow import Marshmallow +from flask_migrate import Migrate from monotonic import monotonic from notifications_utils.clients.statsd.statsd_client import StatsdClient from notifications_utils.clients.redis.redis_client import RedisClient @@ -25,6 +26,7 @@ DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" DATE_FORMAT = "%Y-%m-%d" db = SQLAlchemy() +migrate = Migrate() ma = Marshmallow() notify_celery = NotifyCelery() firetext_client = FiretextClient() @@ -42,21 +44,19 @@ api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user) authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service) -def create_app(app_name=None): - application = Flask(__name__) - +def create_app(application): from app.config import configs notify_environment = os.environ['NOTIFY_ENVIRONMENT'] application.config.from_object(configs[notify_environment]) - if app_name: - application.config['NOTIFY_APP_NAME'] = app_name + application.config['NOTIFY_APP_NAME'] = application.name init_app(application) request_helper.init_app(application) db.init_app(application) + migrate.init_app(application, db=db) ma.init_app(application) statsd_client.init_app(application) logging.init_app(application, statsd_client) @@ -73,6 +73,10 @@ def create_app(app_name=None): register_blueprint(application) register_v2_blueprints(application) + # avoid circular imports by importing this file later 😬 + from app.commands import setup_commands + setup_commands(application) + return application diff --git a/app/commands.py b/app/commands.py index f1e125e01..522aea87e 100644 --- a/app/commands.py +++ b/app/commands.py @@ -1,7 +1,10 @@ import uuid from datetime import datetime, timedelta from decimal import Decimal -from flask_script import Command, Option + +import flask +from flask import current_app +import click from app import db from app.dao.monthly_billing_dao import ( @@ -14,181 +17,178 @@ from app.dao.services_dao import ( delete_service_and_all_associated_db_objects, dao_fetch_all_services_by_user ) -from app.dao.provider_rates_dao import create_provider_rates +from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates from app.dao.users_dao import (delete_model_user, delete_user_verify_codes) from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc from app.performance_platform.processing_time import send_processing_time_for_start_and_end -class CreateProviderRateCommand(Command): +@click.group(name='command', help='Additional commands') +def commands(): + pass - option_list = ( - Option('-p', '--provider_name', dest="provider_name", help='Provider name'), - Option('-c', '--cost', dest="cost", help='Cost (pence) per message including decimals'), - Option('-d', '--valid_from', dest="valid_from", help="Date (%Y-%m-%dT%H:%M:%S) valid from") - ) - def run(self, provider_name, cost, valid_from): - if provider_name not in PROVIDERS: - raise Exception("Invalid provider name, must be one of ({})".format(', '.join(PROVIDERS))) +@commands.command() +@click.option('-p', '--provider_name', required=True, help='Provider name') +@click.option('-c', '--cost', required=True, help='Cost (pence) per message including decimals') +@click.option('-d', '--valid_from', required=True, help="Date (%Y-%m-%dT%H:%M:%S) valid from") +def create_provider_rates(provider_name, cost, valid_from): + """ + Backfill rates for a given provider + """ + if provider_name not in PROVIDERS: + raise Exception("Invalid provider name, must be one of ({})".format(', '.join(PROVIDERS))) + try: + cost = Decimal(cost) + except: + raise Exception("Invalid cost value.") + + try: + valid_from = datetime.strptime('%Y-%m-%dT%H:%M:%S', valid_from) + except: + raise Exception("Invalid valid_from date. Use the format %Y-%m-%dT%H:%M:%S") + + dao_create_provider_rates(provider_name, valid_from, cost) + + +@commands.command() +@click.option('-u', '--user_email_prefix', required=True, help=""" + Functional test user email prefix. eg "notify-test-preview" +""") # noqa +def purge_functional_test_data(user_email_prefix): + """ + Remove non-seeded functional test data + + users, services, etc. Give an email prefix. Probably "notify-test-preview". + """ + users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all() + for usr in users: + # Make sure the full email includes a uuid in it + # Just in case someone decides to use a similar email address. try: - cost = Decimal(cost) - except: - raise Exception("Invalid cost value.") - - try: - valid_from = datetime.strptime('%Y-%m-%dT%H:%M:%S', valid_from) - except: - raise Exception("Invalid valid_from date. Use the format %Y-%m-%dT%H:%M:%S") - - create_provider_rates(provider_name, valid_from, cost) - - -class PurgeFunctionalTestDataCommand(Command): - - option_list = ( - Option('-u', '-user-email-prefix', dest='user_email_prefix', help="Functional test user email prefix."), - ) - - def run(self, user_email_prefix=None): - if user_email_prefix: - users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all() - for usr in users: - # Make sure the full email includes a uuid in it - # Just in case someone decides to use a similar email address. - try: - uuid.UUID(usr.email_address.split("@")[0].split('+')[1]) - except ValueError: - print("Skipping {} as the user email doesn't contain a UUID.".format(usr.email_address)) - else: - services = dao_fetch_all_services_by_user(usr.id) - if services: - for service in services: - delete_service_and_all_associated_db_objects(service) - else: - delete_user_verify_codes(usr) - delete_model_user(usr) - - -class CustomDbScript(Command): - - option_list = ( - Option('-n', '-name-of-db-function', dest='name_of_db_function', help="Function name of the DB script to run"), - ) - - def run(self, name_of_db_function): - db_function = getattr(self, name_of_db_function, None) - if callable(db_function): - db_function() + uuid.UUID(usr.email_address.split("@")[0].split('+')[1]) + except ValueError: + print("Skipping {} as the user email doesn't contain a UUID.".format(usr.email_address)) else: - print('The specified function does not exist.') + services = dao_fetch_all_services_by_user(usr.id) + if services: + for service in services: + delete_service_and_all_associated_db_objects(service) + else: + delete_user_verify_codes(usr) + delete_model_user(usr) - def backfill_notification_statuses(self): - """ - This will be used to populate the new `Notification._status_fkey` with the old - `Notification._status_enum` - """ - LIMIT = 250000 - subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) - update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) + +@commands.command() +def backfill_notification_statuses(): + """ + DEPRECATED. Populates notification_status. + + This will be used to populate the new `Notification._status_fkey` with the old + `Notification._status_enum` + """ + LIMIT = 250000 + subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) + update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) + result = db.session.execute(subq).fetchall() + + while len(result) > 0: + db.session.execute(update) + print('commit {} updates at {}'.format(LIMIT, datetime.utcnow())) + db.session.commit() result = db.session.execute(subq).fetchall() - while len(result) > 0: - db.session.execute(update) - print('commit {} updates at {}'.format(LIMIT, datetime.utcnow())) - db.session.commit() - result = db.session.execute(subq).fetchall() - def update_notification_international_flag(self): - # 250,000 rows takes 30 seconds to update. - subq = "select id from notifications where international is null limit 250000" - update = "update notifications set international = False where id in ({})".format(subq) +@commands.command() +def update_notification_international_flag(): + """ + DEPRECATED. Set notifications.international=false. + """ + # 250,000 rows takes 30 seconds to update. + subq = "select id from notifications where international is null limit 250000" + update = "update notifications set international = False where id in ({})".format(subq) + result = db.session.execute(subq).fetchall() + + while len(result) > 0: + db.session.execute(update) + print('commit 250000 updates at {}'.format(datetime.utcnow())) + db.session.commit() result = db.session.execute(subq).fetchall() - while len(result) > 0: - db.session.execute(update) - print('commit 250000 updates at {}'.format(datetime.utcnow())) - db.session.commit() - result = db.session.execute(subq).fetchall() - - # Now update notification_history - subq_history = "select id from notification_history where international is null limit 250000" - update_history = "update notification_history set international = False where id in ({})".format(subq_history) + # Now update notification_history + subq_history = "select id from notification_history where international is null limit 250000" + update_history = "update notification_history set international = False where id in ({})".format(subq_history) + result_history = db.session.execute(subq_history).fetchall() + while len(result_history) > 0: + db.session.execute(update_history) + print('commit 250000 updates at {}'.format(datetime.utcnow())) + db.session.commit() result_history = db.session.execute(subq_history).fetchall() - while len(result_history) > 0: - db.session.execute(update_history) - print('commit 250000 updates at {}'.format(datetime.utcnow())) - db.session.commit() - result_history = db.session.execute(subq_history).fetchall() - def fix_notification_statuses_not_in_sync(self): - """ - This will be used to correct an issue where Notification._status_enum and NotificationHistory._status_fkey - became out of sync. See 979e90a. - Notification._status_enum is the source of truth so NotificationHistory._status_fkey will be updated with - these values. - """ - MAX = 10000 +@commands.command() +def fix_notification_statuses_not_in_sync(): + """ + DEPRECATED. + This will be used to correct an issue where Notification._status_enum and NotificationHistory._status_fkey + became out of sync. See 979e90a. - subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) - update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) + Notification._status_enum is the source of truth so NotificationHistory._status_fkey will be updated with + these values. + """ + MAX = 10000 + + subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) + update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) + result = db.session.execute(subq).fetchall() + + while len(result) > 0: + db.session.execute(update) + print('Committed {} updates at {}'.format(len(result), datetime.utcnow())) + db.session.commit() result = db.session.execute(subq).fetchall() - while len(result) > 0: - db.session.execute(update) - print('Committed {} updates at {}'.format(len(result), datetime.utcnow())) - db.session.commit() - result = db.session.execute(subq).fetchall() + subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \ + .format(MAX) + update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) + result = db.session.execute(subq_hist).fetchall() - subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \ - .format(MAX) - update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) + while len(result) > 0: + db.session.execute(update) + print('Committed {} updates at {}'.format(len(result), datetime.utcnow())) + db.session.commit() result = db.session.execute(subq_hist).fetchall() - while len(result) > 0: - db.session.execute(update) - print('Committed {} updates at {}'.format(len(result), datetime.utcnow())) - db.session.commit() - result = db.session.execute(subq_hist).fetchall() - def link_inbound_numbers_to_service(self): - update = """ - UPDATE inbound_numbers SET - service_id = services.id, - updated_at = now() - FROM services - WHERE services.sms_sender = inbound_numbers.number AND - inbound_numbers.service_id is null - """ - result = db.session.execute(update) - db.session.commit() +@commands.command() +def link_inbound_numbers_to_service(): + """ + DEPRECATED. - print("Linked {} inbound numbers to service".format(result.rowcount)) + Matches inbound numbers and service ids based on services.sms_sender + """ + update = """ + UPDATE inbound_numbers SET + service_id = services.id, + updated_at = now() + FROM services + WHERE services.sms_sender = inbound_numbers.number AND + inbound_numbers.service_id is null + """ + result = db.session.execute(update) + db.session.commit() + + print("Linked {} inbound numbers to service".format(result.rowcount)) -class PopulateMonthlyBilling(Command): - option_list = ( - Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017"), - ) - - def run(self, year): - service_ids = get_service_ids_that_need_billing_populated( - start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16) - ) - start, end = 1, 13 - if year == '2016': - start = 4 - - for service_id in service_ids: - print('Starting to populate data for service {}'.format(str(service_id))) - print('Starting populating monthly billing for {}'.format(year)) - for i in range(start, end): - print('Population for {}-{}'.format(i, year)) - self.populate(service_id, year, i) - - def populate(self, service_id, year, month): +@commands.command() +@click.option('-y', '--year', required=True, help="Use for integer value for year, e.g. 2017") +def populate_monthly_billing(year): + """ + Populate monthly billing table for all services for a given year. + """ + def populate(service_id, year, month): create_or_update_monthly_billing(service_id, datetime(int(year), int(month), 1)) sms_res = get_monthly_billing_by_notification_type( service_id, datetime(int(year), int(month), 1), SMS_TYPE @@ -200,165 +200,203 @@ class PopulateMonthlyBilling(Command): print('SMS: {}'.format(sms_res.monthly_totals)) print('Email: {}'.format(email_res.monthly_totals)) - -class BackfillProcessingTime(Command): - option_list = ( - Option('-s', '--start_date', dest='start_date', help="Date (%Y-%m-%d) start date inclusive"), - Option('-e', '--end_date', dest='end_date', help="Date (%Y-%m-%d) end date inclusive"), + service_ids = get_service_ids_that_need_billing_populated( + start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16) ) + start, end = 1, 13 - def run(self, start_date, end_date): - start_date = datetime.strptime(start_date, '%Y-%m-%d') - end_date = datetime.strptime(end_date, '%Y-%m-%d') + if year == '2016': + start = 4 - delta = end_date - start_date - - print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date)) - - for i in range(delta.days + 1): - # because the tz conversion funcs talk about midnight, and the midnight before last, - # we want to pretend we're running this from the next morning, so add one. - process_date = start_date + timedelta(days=i + 1) - - process_start_date = get_midnight_for_day_before(process_date) - process_end_date = get_london_midnight_in_utc(process_date) - - print('Sending notification processing-time for {} - {}'.format( - process_start_date.isoformat(), - process_end_date.isoformat() - )) - send_processing_time_for_start_and_end(process_start_date, process_end_date) + for service_id in service_ids: + print('Starting to populate data for service {}'.format(str(service_id))) + print('Starting populating monthly billing for {}'.format(year)) + for i in range(start, end): + print('Population for {}-{}'.format(i, year)) + populate(service_id, year, i) -class PopulateServiceEmailReplyTo(Command): +@commands.command() +@click.option('-s', '--start_date', required=True, help="Date (%Y-%m-%d) start date inclusive") +@click.option('-e', '--end_date', required=True, help="Date (%Y-%m-%d) end date inclusive") +def backfill_processing_time(start_date, end_date): + """ + Send historical performance platform stats. + """ + start_date = datetime.strptime(start_date, '%Y-%m-%d') + end_date = datetime.strptime(end_date, '%Y-%m-%d') - def run(self): - services_to_update = """ - INSERT INTO service_email_reply_to(id, service_id, email_address, is_default, created_at) - SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, reply_to_email_address, true, '{}' - FROM services - WHERE reply_to_email_address IS NOT NULL - AND id NOT IN( - SELECT service_id - FROM service_email_reply_to - ) - """.format(datetime.utcnow()) + delta = end_date - start_date - result = db.session.execute(services_to_update) - db.session.commit() + print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date)) - print("Populated email reply to addresses for {}".format(result.rowcount)) + for i in range(delta.days + 1): + # because the tz conversion funcs talk about midnight, and the midnight before last, + # we want to pretend we're running this from the next morning, so add one. + process_date = start_date + timedelta(days=i + 1) + + process_start_date = get_midnight_for_day_before(process_date) + process_end_date = get_london_midnight_in_utc(process_date) + + print('Sending notification processing-time for {} - {}'.format( + process_start_date.isoformat(), + process_end_date.isoformat() + )) + send_processing_time_for_start_and_end(process_start_date, process_end_date) -class PopulateServiceSmsSender(Command): +@commands.command() +def populate_service_email_reply_to(): + """ + Migrate reply to emails. + """ + services_to_update = """ + INSERT INTO service_email_reply_to(id, service_id, email_address, is_default, created_at) + SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, reply_to_email_address, true, '{}' + FROM services + WHERE reply_to_email_address IS NOT NULL + AND id NOT IN( + SELECT service_id + FROM service_email_reply_to + ) + """.format(datetime.utcnow()) - def run(self): - services_to_update = """ - INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at) - SELECT uuid_in(md5(random()::text || now()::text)::cstring), service_id, number, id, true, '{}' - FROM inbound_numbers - WHERE service_id NOT IN( - SELECT service_id - FROM service_sms_senders - ) - """.format(datetime.utcnow()) + result = db.session.execute(services_to_update) + db.session.commit() - services_to_update_from_services = """ - INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at) - SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, sms_sender, null, true, '{}' + print("Populated email reply to addresses for {}".format(result.rowcount)) + + +@commands.command() +def populate_service_sms_sender(): + """ + Migrate sms senders. Must be called when working on a fresh db! + """ + services_to_update = """ + INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at) + SELECT uuid_in(md5(random()::text || now()::text)::cstring), service_id, number, id, true, '{}' + FROM inbound_numbers + WHERE service_id NOT IN( + SELECT service_id + FROM service_sms_senders + ) + """.format(datetime.utcnow()) + + services_to_update_from_services = """ + INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at) + SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, sms_sender, null, true, '{}' + FROM services + WHERE id NOT IN( + SELECT service_id + FROM service_sms_senders + ) + """.format(datetime.utcnow()) + + result = db.session.execute(services_to_update) + second_result = db.session.execute(services_to_update_from_services) + db.session.commit() + + services_count_query = db.session.execute("Select count(*) from services").fetchall()[0][0] + + service_sms_sender_count_query = db.session.execute("Select count(*) from service_sms_senders").fetchall()[0][0] + + print("Populated sms sender {} services from inbound_numbers".format(result.rowcount)) + print("Populated sms sender {} services from services".format(second_result.rowcount)) + print("{} services in table".format(services_count_query)) + print("{} service_sms_senders".format(service_sms_sender_count_query)) + + +@commands.command() +def populate_service_letter_contact(): + """ + Migrates letter contact blocks. + """ + services_to_update = """ + INSERT INTO service_letter_contacts(id, service_id, contact_block, is_default, created_at) + SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, letter_contact_block, true, '{}' + FROM services + WHERE letter_contact_block IS NOT NULL + AND id NOT IN( + SELECT service_id + FROM service_letter_contacts + ) + """.format(datetime.utcnow()) + + result = db.session.execute(services_to_update) + db.session.commit() + + print("Populated letter contacts for {} services".format(result.rowcount)) + + +@commands.command() +def populate_service_and_service_history_free_sms_fragment_limit(): + """ + DEPRECATED. Set services to have 250k sms limit. + """ + services_to_update = """ + UPDATE services + SET free_sms_fragment_limit = 250000 + WHERE free_sms_fragment_limit IS NULL + """ + + services_history_to_update = """ + UPDATE services_history + SET free_sms_fragment_limit = 250000 + WHERE free_sms_fragment_limit IS NULL + """ + + services_result = db.session.execute(services_to_update) + services_history_result = db.session.execute(services_history_to_update) + + db.session.commit() + + print("Populated free sms fragment limits for {} services".format(services_result.rowcount)) + print("Populated free sms fragment limits for {} services history".format(services_history_result.rowcount)) + + +@commands.command() +def populate_annual_billing(): + """ + add annual_billing for 2016, 2017 and 2018. + """ + financial_year = [2016, 2017, 2018] + + for fy in financial_year: + populate_data = """ + INSERT INTO annual_billing(id, service_id, free_sms_fragment_limit, financial_year_start, + created_at, updated_at) + SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, 250000, {}, '{}', '{}' FROM services WHERE id NOT IN( SELECT service_id - FROM service_sms_senders - ) - """.format(datetime.utcnow()) + FROM annual_billing + WHERE financial_year_start={}) + """.format(fy, datetime.utcnow(), datetime.utcnow(), fy) - result = db.session.execute(services_to_update) - second_result = db.session.execute(services_to_update_from_services) + services_result1 = db.session.execute(populate_data) db.session.commit() - services_count_query = db.session.execute("Select count(*) from services").fetchall()[0][0] - - service_sms_sender_count_query = db.session.execute("Select count(*) from service_sms_senders").fetchall()[0][0] - - print("Populated sms sender {} services from inbound_numbers".format(result.rowcount)) - print("Populated sms sender {} services from services".format(second_result.rowcount)) - print("{} services in table".format(services_count_query)) - print("{} service_sms_senders".format(service_sms_sender_count_query)) + print("Populated annual billing {} for {} services".format(fy, services_result1.rowcount)) -class PopulateServiceLetterContact(Command): - - def run(self): - services_to_update = """ - INSERT INTO service_letter_contacts(id, service_id, contact_block, is_default, created_at) - SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, letter_contact_block, true, '{}' - FROM services - WHERE letter_contact_block IS NOT NULL - AND id NOT IN( - SELECT service_id - FROM service_letter_contacts - ) - """.format(datetime.utcnow()) - - result = db.session.execute(services_to_update) - db.session.commit() - - print("Populated letter contacts for {} services".format(result.rowcount)) +@commands.command() +@click.option('-j', '--job_id', required=True, help="Enter the job id to rebuild the dvla file for") +def re_run_build_dvla_file_for_job(job_id): + """ + Rebuild dvla file for a job. + """ + from app.celery.tasks import build_dvla_file + from app.config import QueueNames + build_dvla_file.apply_async([job_id], queue=QueueNames.JOBS) -class PopulateServiceAndServiceHistoryFreeSmsFragmentLimit(Command): - - def run(self): - services_to_update = """ - UPDATE services - SET free_sms_fragment_limit = 250000 - WHERE free_sms_fragment_limit IS NULL - """ - - services_history_to_update = """ - UPDATE services_history - SET free_sms_fragment_limit = 250000 - WHERE free_sms_fragment_limit IS NULL - """ - - services_result = db.session.execute(services_to_update) - services_history_result = db.session.execute(services_history_to_update) - - db.session.commit() - - print("Populated free sms fragment limits for {} services".format(services_result.rowcount)) - print("Populated free sms fragment limits for {} services history".format(services_history_result.rowcount)) +@commands.command(name='list-routes') +@flask.cli.with_appcontext +def list_routes(): + """List URLs of all application routes.""" + for rule in sorted(current_app.url_map.iter_rules(), key=lambda r: r.rule): + print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule)) -class PopulateAnnualBilling(Command): - def run(self): - financial_year = [2016, 2017, 2018] - - for fy in financial_year: - populate_data = """ - INSERT INTO annual_billing(id, service_id, free_sms_fragment_limit, financial_year_start, - created_at, updated_at) - SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, 250000, {}, '{}', '{}' - FROM services - WHERE id NOT IN( - SELECT service_id - FROM annual_billing - WHERE financial_year_start={}) - """.format(fy, datetime.utcnow(), datetime.utcnow(), fy) - - services_result1 = db.session.execute(populate_data) - db.session.commit() - - print("Populated annual billing {} for {} services".format(fy, services_result1.rowcount)) - - -class ReRunBuildDvlaFileForJob(Command): - option_list = ( - Option('-j', '--job_id', dest='job_id', help="Enter the job id to rebuild the dvla file for"), - ) - - def run(self, job_id): - from app.celery.tasks import build_dvla_file - from app.config import QueueNames - build_dvla_file.apply_async([job_id], queue=QueueNames.JOBS) +def setup_commands(application): + application.cli.add_command(commands) diff --git a/application.py b/application.py index b606aee81..8d94e6d0c 100644 --- a/application.py +++ b/application.py @@ -1,38 +1,10 @@ -#!/usr/bin/env python - +##!/usr/bin/env python from __future__ import print_function -import os -from flask_script import Manager, Server -from flask_migrate import Migrate, MigrateCommand -from app import (create_app, db, commands) -application = create_app() -manager = Manager(application) -port = int(os.environ.get('PORT', 6011)) -manager.add_command("runserver", Server(host='0.0.0.0', port=port)) +from flask import Flask -migrate = Migrate(application, db) -manager.add_command('db', MigrateCommand) -manager.add_command('create_provider_rate', commands.CreateProviderRateCommand) -manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand) -manager.add_command('custom_db_script', commands.CustomDbScript) -manager.add_command('populate_monthly_billing', commands.PopulateMonthlyBilling) -manager.add_command('backfill_processing_time', commands.BackfillProcessingTime) -manager.add_command('populate_service_email_reply_to', commands.PopulateServiceEmailReplyTo) -manager.add_command('populate_service_sms_sender', commands.PopulateServiceSmsSender) -manager.add_command('populate_service_letter_contact', commands.PopulateServiceLetterContact) -manager.add_command('populate_service_and_service_history_free_sms_fragment_limit', - commands.PopulateServiceAndServiceHistoryFreeSmsFragmentLimit) -manager.add_command('populate_annual_billing', commands.PopulateAnnualBilling) -manager.add_command('rerun_build_dvla_file', commands.ReRunBuildDvlaFileForJob) +from app import create_app +application = Flask('app') -@manager.command -def list_routes(): - """List URLs of all application routes.""" - for rule in sorted(application.url_map.iter_rules(), key=lambda r: r.rule): - print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule)) - - -if __name__ == '__main__': - manager.run() +create_app(application) diff --git a/manifest-api-base.yml b/manifest-api-base.yml index 28cf8a8cd..9afb9f860 100644 --- a/manifest-api-base.yml +++ b/manifest-api-base.yml @@ -1,7 +1,7 @@ --- buildpack: python_buildpack -command: scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py --error-logfile /home/vcap/logs/gunicorn_error.log -w 5 -b 0.0.0.0:$PORT wsgi +command: scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py --error-logfile /home/vcap/logs/gunicorn_error.log -w 5 -b 0.0.0.0:$PORT application services: - notify-aws - notify-config @@ -14,6 +14,8 @@ services: env: NOTIFY_APP_NAME: public-api CW_APP_NAME: api + # required by cf run-task + FLASK_APP: application.py instances: 1 memory: 1G diff --git a/migrations/README b/migrations/README deleted file mode 100644 index 7c44eae44..000000000 --- a/migrations/README +++ /dev/null @@ -1,9 +0,0 @@ -Generic single-database configuration. - -python application.py db migrate to generate migration script. - -python application.py db upgrade to upgrade db with script. - -python application.py db downgrade to rollback db changes. - -python application.py db current to show current script. diff --git a/migrations/README.md b/migrations/README.md new file mode 100644 index 000000000..b2977a1cf --- /dev/null +++ b/migrations/README.md @@ -0,0 +1,9 @@ +Generic single-database configuration. + +flask db migrate to generate migration script. + +flask db upgrade to upgrade db with script. + +flask db downgrade to rollback db changes. + +flask db current to show current script. diff --git a/requirements.txt b/requirements.txt index 8d69902c5..37351cacb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ docopt==0.6.2 Flask-Bcrypt==0.7.1 Flask-Marshmallow==0.8.0 Flask-Migrate==2.1.1 -Flask-Script==2.0.5 Flask-SQLAlchemy==2.3.2 Flask==0.12.2 gunicorn==19.7.1 diff --git a/run_celery.py b/run_celery.py index 013499615..4fb28ae08 100644 --- a/run_celery.py +++ b/run_celery.py @@ -1,6 +1,10 @@ #!/usr/bin/env python # notify_celery is referenced from manifest_delivery_base.yml, and cannot be removed +from flask import Flask + from app import notify_celery, create_app -application = create_app('delivery') + +application = Flask('delivery') +create_app(application) application.app_context().push() diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 9ee35d24b..c46d03aa6 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -36,4 +36,4 @@ createdb notification_api # Upgrade databases source environment.sh -python application.py db upgrade +flask db upgrade diff --git a/scripts/check_if_new_migration.py b/scripts/check_if_new_migration.py index 2cd1614d0..2878086f6 100644 --- a/scripts/check_if_new_migration.py +++ b/scripts/check_if_new_migration.py @@ -8,7 +8,7 @@ def get_latest_db_migration_to_apply(): project_dir = dirname(dirname(abspath(__file__))) # Get the main project directory migrations_dir = '{}/migrations/versions/'.format(project_dir) migration_files = [migration_file for migration_file in os.listdir(migrations_dir) if migration_file.endswith('py')] - # sometimes there's a trailing underscore, if script was created with `python app.py db migrate --rev-id=...` + # sometimes there's a trailing underscore, if script was created with `flask db migrate --rev-id=...` latest_file = sorted(migration_files, reverse=True)[0].replace('_.py', '').replace('.py', '') return latest_file diff --git a/scripts/run_app.sh b/scripts/run_app.sh index e8ab00bb6..9997d1072 100755 --- a/scripts/run_app.sh +++ b/scripts/run_app.sh @@ -3,4 +3,4 @@ set -e source environment.sh -python3 application.py runserver +flask run -p 6011 diff --git a/server_commands.py b/server_commands.py deleted file mode 100644 index 9f0ce8a60..000000000 --- a/server_commands.py +++ /dev/null @@ -1,26 +0,0 @@ -from flask_script import Manager, Server -from flask_migrate import Migrate, MigrateCommand -from app import (create_app, db, commands) -import os - -default_env_file = '/home/ubuntu/environment' -environment = 'live' - -if os.path.isfile(default_env_file): - with open(default_env_file, 'r') as environment_file: - environment = environment_file.readline().strip() - -from app.config import configs - -os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment] - -application = create_app() - -manager = Manager(application) -migrate = Migrate(application, db) -manager.add_command('db', MigrateCommand) -manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand) -manager.add_command('custom_db_script', commands.CustomDbScript) - -if __name__ == '__main__': - manager.run() diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index 1c5ceedd9..2ee188d4a 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -1,12 +1,12 @@ from datetime import datetime -from app.commands import BackfillProcessingTime +from app.commands import backfill_processing_time def test_backfill_processing_time_works_for_correct_dates(mocker): send_mock = mocker.patch('app.commands.send_processing_time_for_start_and_end') - BackfillProcessingTime().run('2017-08-01', '2017-08-03') + backfill_processing_time.callback('2017-08-01', '2017-08-03') assert send_mock.call_count == 3 send_mock.assert_any_call(datetime(2017, 7, 31, 23, 0), datetime(2017, 8, 1, 23, 0)) diff --git a/tests/conftest.py b/tests/conftest.py index 321d063a0..266346ba9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,9 @@ from contextlib import contextmanager import os +from flask import Flask from alembic.command import upgrade from alembic.config import Config -from flask_migrate import Migrate, MigrateCommand -from flask_script import Manager import boto3 import pytest import sqlalchemy @@ -14,7 +13,8 @@ from app import create_app, db @pytest.fixture(scope='session') def notify_api(): - app = create_app() + app = Flask('test') + create_app(app) # deattach server-error error handlers - error_handler_spec looks like: # {'blueprint_name': { @@ -76,8 +76,6 @@ def notify_db(notify_api, worker_id): current_app.config['SQLALCHEMY_DATABASE_URI'] += '_{}'.format(worker_id) create_test_db(current_app.config['SQLALCHEMY_DATABASE_URI']) - Migrate(notify_api, db) - Manager(db, MigrateCommand) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) ALEMBIC_CONFIG = os.path.join(BASE_DIR, 'migrations') config = Config(ALEMBIC_CONFIG + '/alembic.ini') diff --git a/wsgi.py b/wsgi.py deleted file mode 100644 index 9fbeb28ac..000000000 --- a/wsgi.py +++ /dev/null @@ -1,7 +0,0 @@ -from app import create_app - - -application = create_app() - -if __name__ == "__main__": - application.run()