darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-04-18 16:19:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1418 from alphagov/remove-flask-script

Browse Source 
Remove flask script
This commit is contained in:
Leo Hemsted
2017-11-24 10:28:56 +00:00
committed by GitHub
parent 87f99b6edb 61b28ca523
commit fcf3932bb4
18 changed files with 374 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@@ -287,7 +287,7 @@ cf-deploy-api-db-migration:
cf unbind-service notify-api-db-migration notify-config
cf unbind-service notify-api-db-migration notify-aws
cf push notify-api-db-migration -f manifest-api-${CF_SPACE}.yml
cf run-task notify-api-db-migration "python db.py db upgrade" --name api_db_migration
cf run-task notify-api-db-migration "flask db upgrade" --name api_db_migration
.PHONY: cf-check-api-db-migration-task
cf-check-api-db-migration-task: ## Get the status for the last notify-api-db-migration task
@@ -310,4 +310,3 @@ cf-push:
.PHONY: check-if-migrations-to-run
check-if-migrations-to-run:
@echo $(shell python3 scripts/check_if_new_migration.py)

16
README.md
View File

@@ -44,6 +44,9 @@ export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY'
export STATSD_PREFIX='YOU_OWN_PREFIX'
export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX'
export REDIS_URL="redis://localhost:6379/0"
export FLASK_APP=application.py
export FLASK_DEBUG=1
export WERKZEUG_DEBUG_PIN=off
"> environment.sh
```
@@ -102,17 +105,20 @@ That will run pycodestyle for code analysis and our unit test suite. If you wish
## To remove functional test data
## To run one off tasks
NOTE: There is assumption that both the server name prefix and user name prefix are followed by a uuid.
The script will search for all services/users with that prefix and only remove it if the prefix is followed by a uuid otherwise it will be skipped.
Tasks are run through the `flask` command - run `flask --help` for more information. There are two sections we need to
care about: `flask db` contains alembic migration commands, and `flask command` contains all of our custom commands. For
example, to purge all dynamically generated functional test data, do the following:
Locally
```
python application.py purge_functional_test_data -u <functional tests user name prefix> # Remove the user and associated services.
flask command purge_functional_test_data -u <functional tests user name prefix>
```
On the server
```
python server_commands.py purge_functional_test_data -u <functional tests user name prefix> # Remove the user and associated services.
cf run-task notify-api "flask command purge_functional_test_data -u <functional tests user name prefix>"
```
All commands and command options have a --help command if you need more information.

0
app/__ Normal file
View File

14
app/__init__.py
View File

@@ -6,6 +6,7 @@ import uuid
from flask import Flask, _request_ctx_stack, request, g, jsonify
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow
from flask_migrate import Migrate
from monotonic import monotonic
from notifications_utils.clients.statsd.statsd_client import StatsdClient
from notifications_utils.clients.redis.redis_client import RedisClient
@@ -25,6 +26,7 @@ DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
DATE_FORMAT = "%Y-%m-%d"
db = SQLAlchemy()
migrate = Migrate()
ma = Marshmallow()
notify_celery = NotifyCelery()
firetext_client = FiretextClient()
@@ -42,21 +44,19 @@ api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service)
def create_app(app_name=None):
application = Flask(__name__)
def create_app(application):
from app.config import configs
notify_environment = os.environ['NOTIFY_ENVIRONMENT']
application.config.from_object(configs[notify_environment])
if app_name:
application.config['NOTIFY_APP_NAME'] = app_name
application.config['NOTIFY_APP_NAME'] = application.name
init_app(application)
request_helper.init_app(application)
db.init_app(application)
migrate.init_app(application, db=db)
ma.init_app(application)
statsd_client.init_app(application)
logging.init_app(application, statsd_client)
@@ -73,6 +73,10 @@ def create_app(app_name=None):
register_blueprint(application)
register_v2_blueprints(application)
# avoid circular imports by importing this file later 😬
from app.commands import setup_commands
setup_commands(application)
return application

608
app/commands.py
View File

@@ -1,7 +1,10 @@
import uuid
from datetime import datetime, timedelta
from decimal import Decimal
from flask_script import Command, Option
import flask
from flask import current_app
import click
from app import db
from app.dao.monthly_billing_dao import (
@@ -14,181 +17,178 @@ from app.dao.services_dao import (
delete_service_and_all_associated_db_objects,
dao_fetch_all_services_by_user
)
from app.dao.provider_rates_dao import create_provider_rates
from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
from app.performance_platform.processing_time import send_processing_time_for_start_and_end
class CreateProviderRateCommand(Command):
@click.group(name='command', help='Additional commands')
def commands():
pass
option_list = (
Option('-p', '--provider_name', dest="provider_name", help='Provider name'),
Option('-c', '--cost', dest="cost", help='Cost (pence) per message including decimals'),
Option('-d', '--valid_from', dest="valid_from", help="Date (%Y-%m-%dT%H:%M:%S) valid from")
)
def run(self, provider_name, cost, valid_from):
if provider_name not in PROVIDERS:
raise Exception("Invalid provider name, must be one of ({})".format(', '.join(PROVIDERS)))
@commands.command()
@click.option('-p', '--provider_name', required=True, help='Provider name')
@click.option('-c', '--cost', required=True, help='Cost (pence) per message including decimals')
@click.option('-d', '--valid_from', required=True, help="Date (%Y-%m-%dT%H:%M:%S) valid from")
def create_provider_rates(provider_name, cost, valid_from):
"""
Backfill rates for a given provider
"""
if provider_name not in PROVIDERS:
raise Exception("Invalid provider name, must be one of ({})".format(', '.join(PROVIDERS)))
try:
cost = Decimal(cost)
except:
raise Exception("Invalid cost value.")
try:
valid_from = datetime.strptime('%Y-%m-%dT%H:%M:%S', valid_from)
except:
raise Exception("Invalid valid_from date. Use the format %Y-%m-%dT%H:%M:%S")
dao_create_provider_rates(provider_name, valid_from, cost)
@commands.command()
@click.option('-u', '--user_email_prefix', required=True, help="""
Functional test user email prefix. eg "notify-test-preview"
""") # noqa
def purge_functional_test_data(user_email_prefix):
"""
Remove non-seeded functional test data
users, services, etc. Give an email prefix. Probably "notify-test-preview".
"""
users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all()
for usr in users:
# Make sure the full email includes a uuid in it
# Just in case someone decides to use a similar email address.
try:
cost = Decimal(cost)
except:
raise Exception("Invalid cost value.")
try:
valid_from = datetime.strptime('%Y-%m-%dT%H:%M:%S', valid_from)
except:
raise Exception("Invalid valid_from date. Use the format %Y-%m-%dT%H:%M:%S")
create_provider_rates(provider_name, valid_from, cost)
class PurgeFunctionalTestDataCommand(Command):
option_list = (
Option('-u', '-user-email-prefix', dest='user_email_prefix', help="Functional test user email prefix."),
)
def run(self, user_email_prefix=None):
if user_email_prefix:
users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all()
for usr in users:
# Make sure the full email includes a uuid in it
# Just in case someone decides to use a similar email address.
try:
uuid.UUID(usr.email_address.split("@")[0].split('+')[1])
except ValueError:
print("Skipping {} as the user email doesn't contain a UUID.".format(usr.email_address))
else:
services = dao_fetch_all_services_by_user(usr.id)
if services:
for service in services:
delete_service_and_all_associated_db_objects(service)
else:
delete_user_verify_codes(usr)
delete_model_user(usr)
class CustomDbScript(Command):
option_list = (
Option('-n', '-name-of-db-function', dest='name_of_db_function', help="Function name of the DB script to run"),
)
def run(self, name_of_db_function):
db_function = getattr(self, name_of_db_function, None)
if callable(db_function):
db_function()
uuid.UUID(usr.email_address.split("@")[0].split('+')[1])
except ValueError:
print("Skipping {} as the user email doesn't contain a UUID.".format(usr.email_address))
else:
print('The specified function does not exist.')
services = dao_fetch_all_services_by_user(usr.id)
if services:
for service in services:
delete_service_and_all_associated_db_objects(service)
else:
delete_user_verify_codes(usr)
delete_model_user(usr)
def backfill_notification_statuses(self):
"""
This will be used to populate the new `Notification._status_fkey` with the old
`Notification._status_enum`
"""
LIMIT = 250000
subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT)
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq)
@commands.command()
def backfill_notification_statuses():
"""
DEPRECATED. Populates notification_status.
This will be used to populate the new `Notification._status_fkey` with the old
`Notification._status_enum`
"""
LIMIT = 250000
subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT)
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq)
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit {} updates at {}'.format(LIMIT, datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit {} updates at {}'.format(LIMIT, datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
def update_notification_international_flag(self):
# 250,000 rows takes 30 seconds to update.
subq = "select id from notifications where international is null limit 250000"
update = "update notifications set international = False where id in ({})".format(subq)
@commands.command()
def update_notification_international_flag():
"""
DEPRECATED. Set notifications.international=false.
"""
# 250,000 rows takes 30 seconds to update.
subq = "select id from notifications where international is null limit 250000"
update = "update notifications set international = False where id in ({})".format(subq)
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
# Now update notification_history
subq_history = "select id from notification_history where international is null limit 250000"
update_history = "update notification_history set international = False where id in ({})".format(subq_history)
# Now update notification_history
subq_history = "select id from notification_history where international is null limit 250000"
update_history = "update notification_history set international = False where id in ({})".format(subq_history)
result_history = db.session.execute(subq_history).fetchall()
while len(result_history) > 0:
db.session.execute(update_history)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result_history = db.session.execute(subq_history).fetchall()
while len(result_history) > 0:
db.session.execute(update_history)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result_history = db.session.execute(subq_history).fetchall()
def fix_notification_statuses_not_in_sync(self):
"""
This will be used to correct an issue where Notification._status_enum and NotificationHistory._status_fkey
became out of sync. See 979e90a.
Notification._status_enum is the source of truth so NotificationHistory._status_fkey will be updated with
these values.
"""
MAX = 10000
@commands.command()
def fix_notification_statuses_not_in_sync():
"""
DEPRECATED.
This will be used to correct an issue where Notification._status_enum and NotificationHistory._status_fkey
became out of sync. See 979e90a.
subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)
update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq)
Notification._status_enum is the source of truth so NotificationHistory._status_fkey will be updated with
these values.
"""
MAX = 10000
subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)
update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq)
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \
.format(MAX)
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist)
result = db.session.execute(subq_hist).fetchall()
subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \
.format(MAX)
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist)
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq_hist).fetchall()
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq_hist).fetchall()
def link_inbound_numbers_to_service(self):
update = """
UPDATE inbound_numbers SET
service_id = services.id,
updated_at = now()
FROM services
WHERE services.sms_sender = inbound_numbers.number AND
inbound_numbers.service_id is null
"""
result = db.session.execute(update)
db.session.commit()
@commands.command()
def link_inbound_numbers_to_service():
"""
DEPRECATED.
print("Linked {} inbound numbers to service".format(result.rowcount))
Matches inbound numbers and service ids based on services.sms_sender
"""
update = """
UPDATE inbound_numbers SET
service_id = services.id,
updated_at = now()
FROM services
WHERE services.sms_sender = inbound_numbers.number AND
inbound_numbers.service_id is null
"""
result = db.session.execute(update)
db.session.commit()
print("Linked {} inbound numbers to service".format(result.rowcount))
class PopulateMonthlyBilling(Command):
option_list = (
Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017"),
)
def run(self, year):
service_ids = get_service_ids_that_need_billing_populated(
start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16)
)
start, end = 1, 13
if year == '2016':
start = 4
for service_id in service_ids:
print('Starting to populate data for service {}'.format(str(service_id)))
print('Starting populating monthly billing for {}'.format(year))
for i in range(start, end):
print('Population for {}-{}'.format(i, year))
self.populate(service_id, year, i)
def populate(self, service_id, year, month):
@commands.command()
@click.option('-y', '--year', required=True, help="Use for integer value for year, e.g. 2017")
def populate_monthly_billing(year):
"""
Populate monthly billing table for all services for a given year.
"""
def populate(service_id, year, month):
create_or_update_monthly_billing(service_id, datetime(int(year), int(month), 1))
sms_res = get_monthly_billing_by_notification_type(
service_id, datetime(int(year), int(month), 1), SMS_TYPE
@@ -200,165 +200,203 @@ class PopulateMonthlyBilling(Command):
print('SMS: {}'.format(sms_res.monthly_totals))
print('Email: {}'.format(email_res.monthly_totals))
class BackfillProcessingTime(Command):
option_list = (
Option('-s', '--start_date', dest='start_date', help="Date (%Y-%m-%d) start date inclusive"),
Option('-e', '--end_date', dest='end_date', help="Date (%Y-%m-%d) end date inclusive"),
service_ids = get_service_ids_that_need_billing_populated(
start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16)
)
start, end = 1, 13
def run(self, start_date, end_date):
start_date = datetime.strptime(start_date, '%Y-%m-%d')
end_date = datetime.strptime(end_date, '%Y-%m-%d')
if year == '2016':
start = 4
delta = end_date - start_date
print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
for i in range(delta.days + 1):
# because the tz conversion funcs talk about midnight, and the midnight before last,
# we want to pretend we're running this from the next morning, so add one.
process_date = start_date + timedelta(days=i + 1)
process_start_date = get_midnight_for_day_before(process_date)
process_end_date = get_london_midnight_in_utc(process_date)
print('Sending notification processing-time for {} - {}'.format(
process_start_date.isoformat(),
process_end_date.isoformat()
))
send_processing_time_for_start_and_end(process_start_date, process_end_date)
for service_id in service_ids:
print('Starting to populate data for service {}'.format(str(service_id)))
print('Starting populating monthly billing for {}'.format(year))
for i in range(start, end):
print('Population for {}-{}'.format(i, year))
populate(service_id, year, i)
class PopulateServiceEmailReplyTo(Command):
@commands.command()
@click.option('-s', '--start_date', required=True, help="Date (%Y-%m-%d) start date inclusive")
@click.option('-e', '--end_date', required=True, help="Date (%Y-%m-%d) end date inclusive")
def backfill_processing_time(start_date, end_date):
"""
Send historical performance platform stats.
"""
start_date = datetime.strptime(start_date, '%Y-%m-%d')
end_date = datetime.strptime(end_date, '%Y-%m-%d')
def run(self):
services_to_update = """
INSERT INTO service_email_reply_to(id, service_id, email_address, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, reply_to_email_address, true, '{}'
FROM services
WHERE reply_to_email_address IS NOT NULL
AND id NOT IN(
SELECT service_id
FROM service_email_reply_to
)
""".format(datetime.utcnow())
delta = end_date - start_date
result = db.session.execute(services_to_update)
db.session.commit()
print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
print("Populated email reply to addresses for {}".format(result.rowcount))
for i in range(delta.days + 1):
# because the tz conversion funcs talk about midnight, and the midnight before last,
# we want to pretend we're running this from the next morning, so add one.
process_date = start_date + timedelta(days=i + 1)
process_start_date = get_midnight_for_day_before(process_date)
process_end_date = get_london_midnight_in_utc(process_date)
print('Sending notification processing-time for {} - {}'.format(
process_start_date.isoformat(),
process_end_date.isoformat()
))
send_processing_time_for_start_and_end(process_start_date, process_end_date)
class PopulateServiceSmsSender(Command):
@commands.command()
def populate_service_email_reply_to():
"""
Migrate reply to emails.
"""
services_to_update = """
INSERT INTO service_email_reply_to(id, service_id, email_address, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, reply_to_email_address, true, '{}'
FROM services
WHERE reply_to_email_address IS NOT NULL
AND id NOT IN(
SELECT service_id
FROM service_email_reply_to
)
""".format(datetime.utcnow())
def run(self):
services_to_update = """
INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), service_id, number, id, true, '{}'
FROM inbound_numbers
WHERE service_id NOT IN(
SELECT service_id
FROM service_sms_senders
)
""".format(datetime.utcnow())
result = db.session.execute(services_to_update)
db.session.commit()
services_to_update_from_services = """
INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, sms_sender, null, true, '{}'
print("Populated email reply to addresses for {}".format(result.rowcount))
@commands.command()
def populate_service_sms_sender():
"""
Migrate sms senders. Must be called when working on a fresh db!
"""
services_to_update = """
INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), service_id, number, id, true, '{}'
FROM inbound_numbers
WHERE service_id NOT IN(
SELECT service_id
FROM service_sms_senders
)
""".format(datetime.utcnow())
services_to_update_from_services = """
INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, sms_sender, null, true, '{}'
FROM services
WHERE id NOT IN(
SELECT service_id
FROM service_sms_senders
)
""".format(datetime.utcnow())
result = db.session.execute(services_to_update)
second_result = db.session.execute(services_to_update_from_services)
db.session.commit()
services_count_query = db.session.execute("Select count(*) from services").fetchall()[0][0]
service_sms_sender_count_query = db.session.execute("Select count(*) from service_sms_senders").fetchall()[0][0]
print("Populated sms sender {} services from inbound_numbers".format(result.rowcount))
print("Populated sms sender {} services from services".format(second_result.rowcount))
print("{} services in table".format(services_count_query))
print("{} service_sms_senders".format(service_sms_sender_count_query))
@commands.command()
def populate_service_letter_contact():
"""
Migrates letter contact blocks.
"""
services_to_update = """
INSERT INTO service_letter_contacts(id, service_id, contact_block, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, letter_contact_block, true, '{}'
FROM services
WHERE letter_contact_block IS NOT NULL
AND id NOT IN(
SELECT service_id
FROM service_letter_contacts
)
""".format(datetime.utcnow())
result = db.session.execute(services_to_update)
db.session.commit()
print("Populated letter contacts for {} services".format(result.rowcount))
@commands.command()
def populate_service_and_service_history_free_sms_fragment_limit():
"""
DEPRECATED. Set services to have 250k sms limit.
"""
services_to_update = """
UPDATE services
SET free_sms_fragment_limit = 250000
WHERE free_sms_fragment_limit IS NULL
"""
services_history_to_update = """
UPDATE services_history
SET free_sms_fragment_limit = 250000
WHERE free_sms_fragment_limit IS NULL
"""
services_result = db.session.execute(services_to_update)
services_history_result = db.session.execute(services_history_to_update)
db.session.commit()
print("Populated free sms fragment limits for {} services".format(services_result.rowcount))
print("Populated free sms fragment limits for {} services history".format(services_history_result.rowcount))
@commands.command()
def populate_annual_billing():
"""
add annual_billing for 2016, 2017 and 2018.
"""
financial_year = [2016, 2017, 2018]
for fy in financial_year:
populate_data = """
INSERT INTO annual_billing(id, service_id, free_sms_fragment_limit, financial_year_start,
created_at, updated_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, 250000, {}, '{}', '{}'
FROM services
WHERE id NOT IN(
SELECT service_id
FROM service_sms_senders
)
""".format(datetime.utcnow())
FROM annual_billing
WHERE financial_year_start={})
""".format(fy, datetime.utcnow(), datetime.utcnow(), fy)
result = db.session.execute(services_to_update)
second_result = db.session.execute(services_to_update_from_services)
services_result1 = db.session.execute(populate_data)
db.session.commit()
services_count_query = db.session.execute("Select count(*) from services").fetchall()[0][0]
service_sms_sender_count_query = db.session.execute("Select count(*) from service_sms_senders").fetchall()[0][0]
print("Populated sms sender {} services from inbound_numbers".format(result.rowcount))
print("Populated sms sender {} services from services".format(second_result.rowcount))
print("{} services in table".format(services_count_query))
print("{} service_sms_senders".format(service_sms_sender_count_query))
print("Populated annual billing {} for {} services".format(fy, services_result1.rowcount))
class PopulateServiceLetterContact(Command):
def run(self):
services_to_update = """
INSERT INTO service_letter_contacts(id, service_id, contact_block, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, letter_contact_block, true, '{}'
FROM services
WHERE letter_contact_block IS NOT NULL
AND id NOT IN(
SELECT service_id
FROM service_letter_contacts
)
""".format(datetime.utcnow())
result = db.session.execute(services_to_update)
db.session.commit()
print("Populated letter contacts for {} services".format(result.rowcount))
@commands.command()
@click.option('-j', '--job_id', required=True, help="Enter the job id to rebuild the dvla file for")
def re_run_build_dvla_file_for_job(job_id):
"""
Rebuild dvla file for a job.
"""
from app.celery.tasks import build_dvla_file
from app.config import QueueNames
build_dvla_file.apply_async([job_id], queue=QueueNames.JOBS)
class PopulateServiceAndServiceHistoryFreeSmsFragmentLimit(Command):
def run(self):
services_to_update = """
UPDATE services
SET free_sms_fragment_limit = 250000
WHERE free_sms_fragment_limit IS NULL
"""
services_history_to_update = """
UPDATE services_history
SET free_sms_fragment_limit = 250000
WHERE free_sms_fragment_limit IS NULL
"""
services_result = db.session.execute(services_to_update)
services_history_result = db.session.execute(services_history_to_update)
db.session.commit()
print("Populated free sms fragment limits for {} services".format(services_result.rowcount))
print("Populated free sms fragment limits for {} services history".format(services_history_result.rowcount))
@commands.command(name='list-routes')
@flask.cli.with_appcontext
def list_routes():
"""List URLs of all application routes."""
for rule in sorted(current_app.url_map.iter_rules(), key=lambda r: r.rule):
print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule))
class PopulateAnnualBilling(Command):
def run(self):
financial_year = [2016, 2017, 2018]
for fy in financial_year:
populate_data = """
INSERT INTO annual_billing(id, service_id, free_sms_fragment_limit, financial_year_start,
created_at, updated_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, 250000, {}, '{}', '{}'
FROM services
WHERE id NOT IN(
SELECT service_id
FROM annual_billing
WHERE financial_year_start={})
""".format(fy, datetime.utcnow(), datetime.utcnow(), fy)
services_result1 = db.session.execute(populate_data)
db.session.commit()
print("Populated annual billing {} for {} services".format(fy, services_result1.rowcount))
class ReRunBuildDvlaFileForJob(Command):
option_list = (
Option('-j', '--job_id', dest='job_id', help="Enter the job id to rebuild the dvla file for"),
)
def run(self, job_id):
from app.celery.tasks import build_dvla_file
from app.config import QueueNames
build_dvla_file.apply_async([job_id], queue=QueueNames.JOBS)
def setup_commands(application):
application.cli.add_command(commands)

38
application.py
View File

@@ -1,38 +1,10 @@
#!/usr/bin/env python
##!/usr/bin/env python
from __future__ import print_function
import os
from flask_script import Manager, Server
from flask_migrate import Migrate, MigrateCommand
from app import (create_app, db, commands)
application = create_app()
manager = Manager(application)
port = int(os.environ.get('PORT', 6011))
manager.add_command("runserver", Server(host='0.0.0.0', port=port))
from flask import Flask
migrate = Migrate(application, db)
manager.add_command('db', MigrateCommand)
manager.add_command('create_provider_rate', commands.CreateProviderRateCommand)
manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand)
manager.add_command('custom_db_script', commands.CustomDbScript)
manager.add_command('populate_monthly_billing', commands.PopulateMonthlyBilling)
manager.add_command('backfill_processing_time', commands.BackfillProcessingTime)
manager.add_command('populate_service_email_reply_to', commands.PopulateServiceEmailReplyTo)
manager.add_command('populate_service_sms_sender', commands.PopulateServiceSmsSender)
manager.add_command('populate_service_letter_contact', commands.PopulateServiceLetterContact)
manager.add_command('populate_service_and_service_history_free_sms_fragment_limit',
commands.PopulateServiceAndServiceHistoryFreeSmsFragmentLimit)
manager.add_command('populate_annual_billing', commands.PopulateAnnualBilling)
manager.add_command('rerun_build_dvla_file', commands.ReRunBuildDvlaFileForJob)
from app import create_app
application = Flask('app')
@manager.command
def list_routes():
"""List URLs of all application routes."""
for rule in sorted(application.url_map.iter_rules(), key=lambda r: r.rule):
print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule))
if __name__ == '__main__':
manager.run()
create_app(application)

4
manifest-api-base.yml
View File

@@ -1,7 +1,7 @@
---
buildpack: python_buildpack
command: scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py --error-logfile /home/vcap/logs/gunicorn_error.log -w 5 -b 0.0.0.0:$PORT wsgi
command: scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py --error-logfile /home/vcap/logs/gunicorn_error.log -w 5 -b 0.0.0.0:$PORT application
services:
- notify-aws
- notify-config
@@ -14,6 +14,8 @@ services:
env:
NOTIFY_APP_NAME: public-api
CW_APP_NAME: api
# required by cf run-task
FLASK_APP: application.py
instances: 1
memory: 1G

9
migrations/README
View File

@@ -1,9 +0,0 @@
Generic single-database configuration.
python application.py db migrate to generate migration script.
python application.py db upgrade to upgrade db with script.
python application.py db downgrade to rollback db changes.
python application.py db current to show current script.

9
migrations/README.md Normal file
View File

@@ -0,0 +1,9 @@
Generic single-database configuration.
flask db migrate to generate migration script.
flask db upgrade to upgrade db with script.
flask db downgrade to rollback db changes.
flask db current to show current script.

1
requirements.txt
View File

@@ -5,7 +5,6 @@ docopt==0.6.2
Flask-Bcrypt==0.7.1
Flask-Marshmallow==0.8.0
Flask-Migrate==2.1.1
Flask-Script==2.0.5
Flask-SQLAlchemy==2.3.2
Flask==0.12.2
gunicorn==19.7.1

6
run_celery.py
View File

@@ -1,6 +1,10 @@
#!/usr/bin/env python
# notify_celery is referenced from manifest_delivery_base.yml, and cannot be removed
from flask import Flask
from app import notify_celery, create_app
application = create_app('delivery')
application = Flask('delivery')
create_app(application)
application.app_context().push()

2
scripts/bootstrap.sh
View File

@@ -36,4 +36,4 @@ createdb notification_api
# Upgrade databases
source environment.sh
python application.py db upgrade
flask db upgrade

2
scripts/check_if_new_migration.py
View File

@@ -8,7 +8,7 @@ def get_latest_db_migration_to_apply():
project_dir = dirname(dirname(abspath(__file__))) # Get the main project directory
migrations_dir = '{}/migrations/versions/'.format(project_dir)
migration_files = [migration_file for migration_file in os.listdir(migrations_dir) if migration_file.endswith('py')]
# sometimes there's a trailing underscore, if script was created with `python app.py db migrate --rev-id=...`
# sometimes there's a trailing underscore, if script was created with `flask db migrate --rev-id=...`
latest_file = sorted(migration_files, reverse=True)[0].replace('_.py', '').replace('.py', '')
return latest_file

2
scripts/run_app.sh
View File

@@ -3,4 +3,4 @@
set -e
source environment.sh
python3 application.py runserver
flask run -p 6011

26
server_commands.py
View File

@@ -1,26 +0,0 @@
from flask_script import Manager, Server
from flask_migrate import Migrate, MigrateCommand
from app import (create_app, db, commands)
import os
default_env_file = '/home/ubuntu/environment'
environment = 'live'
if os.path.isfile(default_env_file):
with open(default_env_file, 'r') as environment_file:
environment = environment_file.readline().strip()
from app.config import configs
os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment]
application = create_app()
manager = Manager(application)
migrate = Migrate(application, db)
manager.add_command('db', MigrateCommand)
manager.add_command('purge_functional_test_data', commands.PurgeFunctionalTestDataCommand)
manager.add_command('custom_db_script', commands.CustomDbScript)
if __name__ == '__main__':
manager.run()

4
tests/app/test_commands.py
View File

@@ -1,12 +1,12 @@
from datetime import datetime
from app.commands import BackfillProcessingTime
from app.commands import backfill_processing_time
def test_backfill_processing_time_works_for_correct_dates(mocker):
send_mock = mocker.patch('app.commands.send_processing_time_for_start_and_end')
BackfillProcessingTime().run('2017-08-01', '2017-08-03')
backfill_processing_time.callback('2017-08-01', '2017-08-03')
assert send_mock.call_count == 3
send_mock.assert_any_call(datetime(2017, 7, 31, 23, 0), datetime(2017, 8, 1, 23, 0))

8
tests/conftest.py
View File

@@ -1,10 +1,9 @@
from contextlib import contextmanager
import os
from flask import Flask
from alembic.command import upgrade
from alembic.config import Config
from flask_migrate import Migrate, MigrateCommand
from flask_script import Manager
import boto3
import pytest
import sqlalchemy
@@ -14,7 +13,8 @@ from app import create_app, db
@pytest.fixture(scope='session')
def notify_api():
app = create_app()
app = Flask('test')
create_app(app)
# deattach server-error error handlers - error_handler_spec looks like:
# {'blueprint_name': {
@@ -76,8 +76,6 @@ def notify_db(notify_api, worker_id):
current_app.config['SQLALCHEMY_DATABASE_URI'] += '_{}'.format(worker_id)
create_test_db(current_app.config['SQLALCHEMY_DATABASE_URI'])
Migrate(notify_api, db)
Manager(db, MigrateCommand)
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
ALEMBIC_CONFIG = os.path.join(BASE_DIR, 'migrations')
config = Config(ALEMBIC_CONFIG + '/alembic.ini')

7
wsgi.py
View File

@@ -1,7 +0,0 @@
from app import create_app
application = create_app()
if __name__ == "__main__":
application.run()