darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-26 17:23:50 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into add-reply-to-notifications

Browse Source
This commit is contained in:
Rebecca Law
2017-11-27 10:40:02 +00:00
parent 049daa0cb8 b7d5e80509
commit 5450af661e
31 changed files with 664 additions and 527 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
app/__ Normal file
View File

14
app/__init__.py
View File

@@ -6,6 +6,7 @@ import uuid
from flask import Flask, _request_ctx_stack, request, g, jsonify
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow
from flask_migrate import Migrate
from monotonic import monotonic
from notifications_utils.clients.statsd.statsd_client import StatsdClient
from notifications_utils.clients.redis.redis_client import RedisClient
@@ -25,6 +26,7 @@ DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
DATE_FORMAT = "%Y-%m-%d"
db = SQLAlchemy()
migrate = Migrate()
ma = Marshmallow()
notify_celery = NotifyCelery()
firetext_client = FiretextClient()
@@ -42,21 +44,19 @@ api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service)
def create_app(app_name=None):
application = Flask(__name__)
def create_app(application):
from app.config import configs
notify_environment = os.environ['NOTIFY_ENVIRONMENT']
application.config.from_object(configs[notify_environment])
if app_name:
application.config['NOTIFY_APP_NAME'] = app_name
application.config['NOTIFY_APP_NAME'] = application.name
init_app(application)
request_helper.init_app(application)
db.init_app(application)
migrate.init_app(application, db=db)
ma.init_app(application)
statsd_client.init_app(application)
logging.init_app(application, statsd_client)
@@ -73,6 +73,10 @@ def create_app(app_name=None):
register_blueprint(application)
register_v2_blueprints(application)
# avoid circular imports by importing this file later 😬
from app.commands import setup_commands
setup_commands(application)
return application

1
app/celery/tasks.py
View File

@@ -488,6 +488,7 @@ def send_inbound_sms_to_service(self, inbound_sms_id, service_id):
inbound_id=inbound_sms_id)
data = {
"id": str(inbound_sms.id),
# TODO: should we be validating and formatting the phone number here?
"source_number": inbound_sms.user_number,
"destination_number": inbound_sms.notify_number,
"message": inbound_sms.content,

627
app/commands.py
View File

@@ -1,7 +1,11 @@
import uuid
from datetime import datetime, timedelta
from decimal import Decimal
from flask_script import Command, Option
import functools
import flask
from flask import current_app
import click
from app import db
from app.dao.monthly_billing_dao import (
@@ -14,181 +18,197 @@ from app.dao.services_dao import (
delete_service_and_all_associated_db_objects,
dao_fetch_all_services_by_user
)
from app.dao.provider_rates_dao import create_provider_rates
from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
from app.performance_platform.processing_time import send_processing_time_for_start_and_end
class CreateProviderRateCommand(Command):
@click.group(name='command', help='Additional commands')
def command_group():
pass
option_list = (
Option('-p', '--provider_name', dest="provider_name", help='Provider name'),
Option('-c', '--cost', dest="cost", help='Cost (pence) per message including decimals'),
Option('-d', '--valid_from', dest="valid_from", help="Date (%Y-%m-%dT%H:%M:%S) valid from")
)
def run(self, provider_name, cost, valid_from):
if provider_name not in PROVIDERS:
raise Exception("Invalid provider name, must be one of ({})".format(', '.join(PROVIDERS)))
class notify_command:
def __init__(self, name=None):
self.name = name
def __call__(self, func):
# we need to call the flask with_appcontext decorator to ensure the config is loaded, db connected etc etc.
# we also need to use functools.wraps to carry through the names and docstrings etc of the functions.
# Then we need to turn it into a click.Command - that's what command_group.add_command expects.
@click.command(name=self.name)
@functools.wraps(func)
@flask.cli.with_appcontext
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
command_group.add_command(wrapper)
return wrapper
@notify_command()
@click.option('-p', '--provider_name', required=True, help='Provider name')
@click.option('-c', '--cost', required=True, help='Cost (pence) per message including decimals')
@click.option('-d', '--valid_from', required=True, help="Date (%Y-%m-%dT%H:%M:%S) valid from")
def create_provider_rates(provider_name, cost, valid_from):
"""
Backfill rates for a given provider
"""
if provider_name not in PROVIDERS:
raise Exception("Invalid provider name, must be one of ({})".format(', '.join(PROVIDERS)))
try:
cost = Decimal(cost)
except:
raise Exception("Invalid cost value.")
try:
valid_from = datetime.strptime('%Y-%m-%dT%H:%M:%S', valid_from)
except:
raise Exception("Invalid valid_from date. Use the format %Y-%m-%dT%H:%M:%S")
dao_create_provider_rates(provider_name, valid_from, cost)
@notify_command()
@click.option('-u', '--user_email_prefix', required=True, help="""
Functional test user email prefix. eg "notify-test-preview"
""") # noqa
def purge_functional_test_data(user_email_prefix):
"""
Remove non-seeded functional test data
users, services, etc. Give an email prefix. Probably "notify-test-preview".
"""
users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all()
for usr in users:
# Make sure the full email includes a uuid in it
# Just in case someone decides to use a similar email address.
try:
cost = Decimal(cost)
except:
raise Exception("Invalid cost value.")
try:
valid_from = datetime.strptime('%Y-%m-%dT%H:%M:%S', valid_from)
except:
raise Exception("Invalid valid_from date. Use the format %Y-%m-%dT%H:%M:%S")
create_provider_rates(provider_name, valid_from, cost)
class PurgeFunctionalTestDataCommand(Command):
option_list = (
Option('-u', '-user-email-prefix', dest='user_email_prefix', help="Functional test user email prefix."),
)
def run(self, user_email_prefix=None):
if user_email_prefix:
users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all()
for usr in users:
# Make sure the full email includes a uuid in it
# Just in case someone decides to use a similar email address.
try:
uuid.UUID(usr.email_address.split("@")[0].split('+')[1])
except ValueError:
print("Skipping {} as the user email doesn't contain a UUID.".format(usr.email_address))
else:
services = dao_fetch_all_services_by_user(usr.id)
if services:
for service in services:
delete_service_and_all_associated_db_objects(service)
else:
delete_user_verify_codes(usr)
delete_model_user(usr)
class CustomDbScript(Command):
option_list = (
Option('-n', '-name-of-db-function', dest='name_of_db_function', help="Function name of the DB script to run"),
)
def run(self, name_of_db_function):
db_function = getattr(self, name_of_db_function, None)
if callable(db_function):
db_function()
uuid.UUID(usr.email_address.split("@")[0].split('+')[1])
except ValueError:
print("Skipping {} as the user email doesn't contain a UUID.".format(usr.email_address))
else:
print('The specified function does not exist.')
services = dao_fetch_all_services_by_user(usr.id)
if services:
for service in services:
delete_service_and_all_associated_db_objects(service)
else:
delete_user_verify_codes(usr)
delete_model_user(usr)
def backfill_notification_statuses(self):
"""
This will be used to populate the new `Notification._status_fkey` with the old
`Notification._status_enum`
"""
LIMIT = 250000
subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT)
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq)
@notify_command()
def backfill_notification_statuses():
"""
DEPRECATED. Populates notification_status.
This will be used to populate the new `Notification._status_fkey` with the old
`Notification._status_enum`
"""
LIMIT = 250000
subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT)
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq)
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit {} updates at {}'.format(LIMIT, datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit {} updates at {}'.format(LIMIT, datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
def update_notification_international_flag(self):
# 250,000 rows takes 30 seconds to update.
subq = "select id from notifications where international is null limit 250000"
update = "update notifications set international = False where id in ({})".format(subq)
@notify_command()
def update_notification_international_flag():
"""
DEPRECATED. Set notifications.international=false.
"""
# 250,000 rows takes 30 seconds to update.
subq = "select id from notifications where international is null limit 250000"
update = "update notifications set international = False where id in ({})".format(subq)
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
# Now update notification_history
subq_history = "select id from notification_history where international is null limit 250000"
update_history = "update notification_history set international = False where id in ({})".format(subq_history)
# Now update notification_history
subq_history = "select id from notification_history where international is null limit 250000"
update_history = "update notification_history set international = False where id in ({})".format(subq_history)
result_history = db.session.execute(subq_history).fetchall()
while len(result_history) > 0:
db.session.execute(update_history)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result_history = db.session.execute(subq_history).fetchall()
while len(result_history) > 0:
db.session.execute(update_history)
print('commit 250000 updates at {}'.format(datetime.utcnow()))
db.session.commit()
result_history = db.session.execute(subq_history).fetchall()
def fix_notification_statuses_not_in_sync(self):
"""
This will be used to correct an issue where Notification._status_enum and NotificationHistory._status_fkey
became out of sync. See 979e90a.
Notification._status_enum is the source of truth so NotificationHistory._status_fkey will be updated with
these values.
"""
MAX = 10000
@notify_command()
def fix_notification_statuses_not_in_sync():
"""
DEPRECATED.
This will be used to correct an issue where Notification._status_enum and NotificationHistory._status_fkey
became out of sync. See 979e90a.
subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)
update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq)
Notification._status_enum is the source of truth so NotificationHistory._status_fkey will be updated with
these values.
"""
MAX = 10000
subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX)
update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq)
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq).fetchall()
subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \
.format(MAX)
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist)
result = db.session.execute(subq_hist).fetchall()
subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}" \
.format(MAX)
update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist)
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq_hist).fetchall()
while len(result) > 0:
db.session.execute(update)
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq_hist).fetchall()
def link_inbound_numbers_to_service(self):
update = """
UPDATE inbound_numbers SET
service_id = services.id,
updated_at = now()
FROM services
WHERE services.sms_sender = inbound_numbers.number AND
inbound_numbers.service_id is null
"""
result = db.session.execute(update)
db.session.commit()
@notify_command()
def link_inbound_numbers_to_service():
"""
DEPRECATED.
print("Linked {} inbound numbers to service".format(result.rowcount))
Matches inbound numbers and service ids based on services.sms_sender
"""
update = """
UPDATE inbound_numbers SET
service_id = services.id,
updated_at = now()
FROM services
WHERE services.sms_sender = inbound_numbers.number AND
inbound_numbers.service_id is null
"""
result = db.session.execute(update)
db.session.commit()
print("Linked {} inbound numbers to service".format(result.rowcount))
class PopulateMonthlyBilling(Command):
option_list = (
Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017"),
)
def run(self, year):
service_ids = get_service_ids_that_need_billing_populated(
start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16)
)
start, end = 1, 13
if year == '2016':
start = 4
for service_id in service_ids:
print('Starting to populate data for service {}'.format(str(service_id)))
print('Starting populating monthly billing for {}'.format(year))
for i in range(start, end):
print('Population for {}-{}'.format(i, year))
self.populate(service_id, year, i)
def populate(self, service_id, year, month):
@notify_command()
@click.option('-y', '--year', required=True, help="Use for integer value for year, e.g. 2017")
def populate_monthly_billing(year):
"""
Populate monthly billing table for all services for a given year.
"""
def populate(service_id, year, month):
create_or_update_monthly_billing(service_id, datetime(int(year), int(month), 1))
sms_res = get_monthly_billing_by_notification_type(
service_id, datetime(int(year), int(month), 1), SMS_TYPE
@@ -200,165 +220,202 @@ class PopulateMonthlyBilling(Command):
print('SMS: {}'.format(sms_res.monthly_totals))
print('Email: {}'.format(email_res.monthly_totals))
class BackfillProcessingTime(Command):
option_list = (
Option('-s', '--start_date', dest='start_date', help="Date (%Y-%m-%d) start date inclusive"),
Option('-e', '--end_date', dest='end_date', help="Date (%Y-%m-%d) end date inclusive"),
service_ids = get_service_ids_that_need_billing_populated(
start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16)
)
start, end = 1, 13
def run(self, start_date, end_date):
start_date = datetime.strptime(start_date, '%Y-%m-%d')
end_date = datetime.strptime(end_date, '%Y-%m-%d')
if year == '2016':
start = 4
delta = end_date - start_date
print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
for i in range(delta.days + 1):
# because the tz conversion funcs talk about midnight, and the midnight before last,
# we want to pretend we're running this from the next morning, so add one.
process_date = start_date + timedelta(days=i + 1)
process_start_date = get_midnight_for_day_before(process_date)
process_end_date = get_london_midnight_in_utc(process_date)
print('Sending notification processing-time for {} - {}'.format(
process_start_date.isoformat(),
process_end_date.isoformat()
))
send_processing_time_for_start_and_end(process_start_date, process_end_date)
for service_id in service_ids:
print('Starting to populate data for service {}'.format(str(service_id)))
print('Starting populating monthly billing for {}'.format(year))
for i in range(start, end):
print('Population for {}-{}'.format(i, year))
populate(service_id, year, i)
class PopulateServiceEmailReplyTo(Command):
@notify_command()
@click.option('-s', '--start_date', required=True, help="Date (%Y-%m-%d) start date inclusive")
@click.option('-e', '--end_date', required=True, help="Date (%Y-%m-%d) end date inclusive")
def backfill_processing_time(start_date, end_date):
"""
Send historical performance platform stats.
"""
start_date = datetime.strptime(start_date, '%Y-%m-%d')
end_date = datetime.strptime(end_date, '%Y-%m-%d')
def run(self):
services_to_update = """
INSERT INTO service_email_reply_to(id, service_id, email_address, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, reply_to_email_address, true, '{}'
FROM services
WHERE reply_to_email_address IS NOT NULL
AND id NOT IN(
SELECT service_id
FROM service_email_reply_to
)
""".format(datetime.utcnow())
delta = end_date - start_date
result = db.session.execute(services_to_update)
db.session.commit()
print('Sending notification processing-time data for all days between {} and {}'.format(start_date, end_date))
print("Populated email reply to addresses for {}".format(result.rowcount))
for i in range(delta.days + 1):
# because the tz conversion funcs talk about midnight, and the midnight before last,
# we want to pretend we're running this from the next morning, so add one.
process_date = start_date + timedelta(days=i + 1)
process_start_date = get_midnight_for_day_before(process_date)
process_end_date = get_london_midnight_in_utc(process_date)
print('Sending notification processing-time for {} - {}'.format(
process_start_date.isoformat(),
process_end_date.isoformat()
))
send_processing_time_for_start_and_end(process_start_date, process_end_date)
class PopulateServiceSmsSender(Command):
@notify_command()
def populate_service_email_reply_to():
"""
Migrate reply to emails.
"""
services_to_update = """
INSERT INTO service_email_reply_to(id, service_id, email_address, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, reply_to_email_address, true, '{}'
FROM services
WHERE reply_to_email_address IS NOT NULL
AND id NOT IN(
SELECT service_id
FROM service_email_reply_to
)
""".format(datetime.utcnow())
def run(self):
services_to_update = """
INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), service_id, number, id, true, '{}'
FROM inbound_numbers
WHERE service_id NOT IN(
SELECT service_id
FROM service_sms_senders
)
""".format(datetime.utcnow())
result = db.session.execute(services_to_update)
db.session.commit()
services_to_update_from_services = """
INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, sms_sender, null, true, '{}'
print("Populated email reply to addresses for {}".format(result.rowcount))
@notify_command()
def populate_service_sms_sender():
"""
Migrate sms senders. Must be called when working on a fresh db!
"""
services_to_update = """
INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), service_id, number, id, true, '{}'
FROM inbound_numbers
WHERE service_id NOT IN(
SELECT service_id
FROM service_sms_senders
)
""".format(datetime.utcnow())
services_to_update_from_services = """
INSERT INTO service_sms_senders(id, service_id, sms_sender, inbound_number_id, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, sms_sender, null, true, '{}'
FROM services
WHERE id NOT IN(
SELECT service_id
FROM service_sms_senders
)
""".format(datetime.utcnow())
result = db.session.execute(services_to_update)
second_result = db.session.execute(services_to_update_from_services)
db.session.commit()
services_count_query = db.session.execute("Select count(*) from services").fetchall()[0][0]
service_sms_sender_count_query = db.session.execute("Select count(*) from service_sms_senders").fetchall()[0][0]
print("Populated sms sender {} services from inbound_numbers".format(result.rowcount))
print("Populated sms sender {} services from services".format(second_result.rowcount))
print("{} services in table".format(services_count_query))
print("{} service_sms_senders".format(service_sms_sender_count_query))
@notify_command()
def populate_service_letter_contact():
"""
Migrates letter contact blocks.
"""
services_to_update = """
INSERT INTO service_letter_contacts(id, service_id, contact_block, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, letter_contact_block, true, '{}'
FROM services
WHERE letter_contact_block IS NOT NULL
AND id NOT IN(
SELECT service_id
FROM service_letter_contacts
)
""".format(datetime.utcnow())
result = db.session.execute(services_to_update)
db.session.commit()
print("Populated letter contacts for {} services".format(result.rowcount))
@notify_command()
def populate_service_and_service_history_free_sms_fragment_limit():
"""
DEPRECATED. Set services to have 250k sms limit.
"""
services_to_update = """
UPDATE services
SET free_sms_fragment_limit = 250000
WHERE free_sms_fragment_limit IS NULL
"""
services_history_to_update = """
UPDATE services_history
SET free_sms_fragment_limit = 250000
WHERE free_sms_fragment_limit IS NULL
"""
services_result = db.session.execute(services_to_update)
services_history_result = db.session.execute(services_history_to_update)
db.session.commit()
print("Populated free sms fragment limits for {} services".format(services_result.rowcount))
print("Populated free sms fragment limits for {} services history".format(services_history_result.rowcount))
@notify_command()
def populate_annual_billing():
"""
add annual_billing for 2016, 2017 and 2018.
"""
financial_year = [2016, 2017, 2018]
for fy in financial_year:
populate_data = """
INSERT INTO annual_billing(id, service_id, free_sms_fragment_limit, financial_year_start,
created_at, updated_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, 250000, {}, '{}', '{}'
FROM services
WHERE id NOT IN(
SELECT service_id
FROM service_sms_senders
)
""".format(datetime.utcnow())
FROM annual_billing
WHERE financial_year_start={})
""".format(fy, datetime.utcnow(), datetime.utcnow(), fy)
result = db.session.execute(services_to_update)
second_result = db.session.execute(services_to_update_from_services)
services_result1 = db.session.execute(populate_data)
db.session.commit()
services_count_query = db.session.execute("Select count(*) from services").fetchall()[0][0]
service_sms_sender_count_query = db.session.execute("Select count(*) from service_sms_senders").fetchall()[0][0]
print("Populated sms sender {} services from inbound_numbers".format(result.rowcount))
print("Populated sms sender {} services from services".format(second_result.rowcount))
print("{} services in table".format(services_count_query))
print("{} service_sms_senders".format(service_sms_sender_count_query))
print("Populated annual billing {} for {} services".format(fy, services_result1.rowcount))
class PopulateServiceLetterContact(Command):
def run(self):
services_to_update = """
INSERT INTO service_letter_contacts(id, service_id, contact_block, is_default, created_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, letter_contact_block, true, '{}'
FROM services
WHERE letter_contact_block IS NOT NULL
AND id NOT IN(
SELECT service_id
FROM service_letter_contacts
)
""".format(datetime.utcnow())
result = db.session.execute(services_to_update)
db.session.commit()
print("Populated letter contacts for {} services".format(result.rowcount))
@notify_command()
@click.option('-j', '--job_id', required=True, help="Enter the job id to rebuild the dvla file for")
def re_run_build_dvla_file_for_job(job_id):
"""
Rebuild dvla file for a job.
"""
from app.celery.tasks import build_dvla_file
from app.config import QueueNames
build_dvla_file.apply_async([job_id], queue=QueueNames.JOBS)
class PopulateServiceAndServiceHistoryFreeSmsFragmentLimit(Command):
def run(self):
services_to_update = """
UPDATE services
SET free_sms_fragment_limit = 250000
WHERE free_sms_fragment_limit IS NULL
"""
services_history_to_update = """
UPDATE services_history
SET free_sms_fragment_limit = 250000
WHERE free_sms_fragment_limit IS NULL
"""
services_result = db.session.execute(services_to_update)
services_history_result = db.session.execute(services_history_to_update)
db.session.commit()
print("Populated free sms fragment limits for {} services".format(services_result.rowcount))
print("Populated free sms fragment limits for {} services history".format(services_history_result.rowcount))
@notify_command(name='list-routes')
def list_routes():
"""List URLs of all application routes."""
for rule in sorted(current_app.url_map.iter_rules(), key=lambda r: r.rule):
print("{:10} {}".format(", ".join(rule.methods - set(['OPTIONS', 'HEAD'])), rule.rule))
class PopulateAnnualBilling(Command):
def run(self):
financial_year = [2016, 2017, 2018]
for fy in financial_year:
populate_data = """
INSERT INTO annual_billing(id, service_id, free_sms_fragment_limit, financial_year_start,
created_at, updated_at)
SELECT uuid_in(md5(random()::text || now()::text)::cstring), id, 250000, {}, '{}', '{}'
FROM services
WHERE id NOT IN(
SELECT service_id
FROM annual_billing
WHERE financial_year_start={})
""".format(fy, datetime.utcnow(), datetime.utcnow(), fy)
services_result1 = db.session.execute(populate_data)
db.session.commit()
print("Populated annual billing {} for {} services".format(fy, services_result1.rowcount))
class ReRunBuildDvlaFileForJob(Command):
option_list = (
Option('-j', '--job_id', dest='job_id', help="Enter the job id to rebuild the dvla file for"),
)
def run(self, job_id):
from app.celery.tasks import build_dvla_file
from app.config import QueueNames
build_dvla_file.apply_async([job_id], queue=QueueNames.JOBS)
def setup_commands(application):
application.cli.add_command(command_group)

15
app/config.py
View File

@@ -389,6 +389,21 @@ class Staging(Config):
API_RATE_LIMIT_ENABLED = True
CHECK_PROXY_HEADER = True
API_KEY_LIMITS = {
KEY_TYPE_TEAM: {
"limit": 21000,
"interval": 60
},
KEY_TYPE_NORMAL: {
"limit": 21000,
"interval": 60
},
KEY_TYPE_TEST: {
"limit": 21000,
"interval": 60
}
}
class Live(Config):
NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk'

8
app/dao/templates_dao.py
View File

@@ -1,7 +1,7 @@
from datetime import datetime
import uuid
from sqlalchemy import desc
from sqlalchemy import asc, desc
from sqlalchemy.sql.expression import bindparam
from app import db
@@ -65,14 +65,16 @@ def dao_get_all_templates_for_service(service_id, template_type=None):
template_type=template_type,
archived=False
).order_by(
desc(Template.created_at)
asc(Template.name),
asc(Template.template_type),
).all()
return Template.query.filter_by(
service_id=service_id,
archived=False
).order_by(
desc(Template.created_at)
asc(Template.name),
asc(Template.template_type),
).all()

2
app/inbound_sms/inbound_sms_schemas.py
View File

@@ -3,7 +3,7 @@ get_inbound_sms_for_service_schema = {
"description": "schema for parameters allowed when searching for to field=",
"type": "object",
"properties": {
"phone_number": {"type": "string", "format": "phone_number"},
"phone_number": {"type": "string"},
"limit": {"type": ["integer", "null"], "minimum": 1}
}
}

47
app/inbound_sms/rest.py
View File

@@ -1,11 +1,10 @@
from flask import (
Blueprint,
jsonify,
request,
current_app, json)
from jsonschema import ValidationError
request
)
from notifications_utils.recipients import validate_and_format_phone_number
from notifications_utils.recipients import try_validate_and_format_phone_number
from app.dao.inbound_sms_dao import (
dao_get_inbound_sms_for_service,
@@ -26,25 +25,31 @@ inbound_sms = Blueprint(
register_errors(inbound_sms)
@inbound_sms.route('', methods=['POST', 'GET'])
def get_inbound_sms_for_service(service_id):
if request.method == 'GET':
limit = request.args.get('limit')
user_number = request.args.get('user_number')
if user_number:
# we use this to normalise to an international phone number
user_number = validate_and_format_phone_number(user_number, international=True)
results = dao_get_inbound_sms_for_service(service_id, limit, user_number)
return jsonify(data=[row.serialize() for row in results])
@inbound_sms.route('', methods=['POST'])
def post_query_inbound_sms_for_service(service_id):
form = validate(request.get_json(), get_inbound_sms_for_service_schema)
if 'phone_number' in form:
# we use this to normalise to an international phone number - but this may fail if it's an alphanumeric
user_number = try_validate_and_format_phone_number(form['phone_number'], international=True)
else:
form = validate(request.get_json(), get_inbound_sms_for_service_schema)
results = dao_get_inbound_sms_for_service(service_id, form.get('limit'), form.get('phone_number'))
user_number = None
results = dao_get_inbound_sms_for_service(service_id, form.get('limit'), user_number)
return jsonify(data=[row.serialize() for row in results])
return jsonify(data=[row.serialize() for row in results])
@inbound_sms.route('', methods=['GET'])
def get_inbound_sms_for_service(service_id):
limit = request.args.get('limit')
user_number = request.args.get('user_number')
if user_number:
# we use this to normalise to an international phone number - but this may fail if it's an alphanumeric
user_number = try_validate_and_format_phone_number(user_number, international=True)
results = dao_get_inbound_sms_for_service(service_id, limit, user_number)
return jsonify(data=[row.serialize() for row in results])
@inbound_sms.route('/summary')

8
app/notifications/receive_notifications.py
View File

@@ -2,7 +2,7 @@ from urllib.parse import unquote
import iso8601
from flask import jsonify, Blueprint, current_app, request, abort
from notifications_utils.recipients import validate_and_format_phone_number
from notifications_utils.recipients import try_validate_and_format_phone_number
from app import statsd_client, firetext_client, mmg_client
from app.celery import tasks
@@ -109,7 +109,11 @@ def format_mmg_datetime(date):
def create_inbound_sms_object(service, content, from_number, provider_ref, date_received, provider_name):
user_number = validate_and_format_phone_number(from_number, international=True)
user_number = try_validate_and_format_phone_number(
from_number,
international=True,
log_msg='Invalid from_number received'
)
provider_date = date_received
if provider_date: