darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-11 02:23:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/alphagov/notifications-api into sms_whitelist

Browse Source
This commit is contained in:
venusbb
2017-07-25 17:14:59 +01:00
parent b056d73706 3d3703750c
commit fb601daedc
56 changed files with 1113 additions and 418 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/__init__.py
+1
View File
@@ -49,6 +49,7 @@ def create_app(app_name=None):
from app.config import configs
notify_environment = os.environ['NOTIFY_ENVIRONMENT']
application.config.from_object(configs[notify_environment])
if app_name:
app/celery/__init__.py
-27
View File
@@ -1,27 +0,0 @@
class QueueNames(object):
PERIODIC = 'periodic-tasks'
PRIORITY = 'priority-tasks'
DATABASE = 'database-tasks'
SEND = 'send-tasks'
RESEARCH_MODE = 'research-mode-tasks'
STATISTICS = 'statistics-tasks'
JOBS = 'job-tasks'
RETRY = 'retry-tasks'
NOTIFY = 'notify-internal-tasks'
PROCESS_FTP = 'process-ftp-tasks'
@staticmethod
def all_queues():
return [
QueueNames.PRIORITY,
QueueNames.PERIODIC,
QueueNames.DATABASE,
QueueNames.SEND,
QueueNames.RESEARCH_MODE,
QueueNames.STATISTICS,
QueueNames.JOBS,
QueueNames.RETRY,
QueueNames.NOTIFY,
QueueNames.PROCESS_FTP
]
app/celery/celery.py
+3 -123
View File
@@ -1,130 +1,11 @@
from datetime import timedelta
from celery import Celery
from celery.schedules import crontab
from kombu import Queue, Exchange
from app.celery import QueueNames
class CeleryConfig:
def __init__(self, config):
self.broker_transport_options['queue_name_prefix'] = config['NOTIFICATION_QUEUE_PREFIX']
self.broker_url = config.get('BROKER_URL', 'sqs://')
broker_transport_options = {
'region': 'eu-west-1',
'polling_interval': 1, # 1 second
'visibility_timeout': 310,
'queue_name_prefix': None
}
enable_utc = True,
timezone = 'Europe/London'
accept_content = ['json']
task_serializer = 'json'
imports = ('app.celery.tasks', 'app.celery.scheduled_tasks')
beat_schedule = {
'run-scheduled-jobs': {
'task': 'run-scheduled-jobs',
'schedule': crontab(minute=1),
'options': {'queue': QueueNames.PERIODIC}
},
# 'send-scheduled-notifications': {
# 'task': 'send-scheduled-notifications',
# 'schedule': crontab(minute='*/15'),
# 'options': {'queue': 'periodic'}
# },
'delete-verify-codes': {
'task': 'delete-verify-codes',
'schedule': timedelta(minutes=63),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-invitations': {
'task': 'delete-invitations',
'schedule': timedelta(minutes=66),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-sms-notifications': {
'task': 'delete-sms-notifications',
'schedule': crontab(minute=0, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-email-notifications': {
'task': 'delete-email-notifications',
'schedule': crontab(minute=20, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-letter-notifications': {
'task': 'delete-letter-notifications',
'schedule': crontab(minute=40, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-inbound-sms': {
'task': 'delete-inbound-sms',
'schedule': crontab(minute=0, hour=1),
'options': {'queue': QueueNames.PERIODIC}
},
'send-daily-performance-platform-stats': {
'task': 'send-daily-performance-platform-stats',
'schedule': crontab(minute=0, hour=2),
'options': {'queue': QueueNames.PERIODIC}
},
'switch-current-sms-provider-on-slow-delivery': {
'task': 'switch-current-sms-provider-on-slow-delivery',
'schedule': crontab(), # Every minute
'options': {'queue': QueueNames.PERIODIC}
},
'timeout-sending-notifications': {
'task': 'timeout-sending-notifications',
'schedule': crontab(minute=0, hour=3),
'options': {'queue': QueueNames.PERIODIC}
},
'remove_sms_email_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=0, hour=4),
'options': {'queue': QueueNames.PERIODIC},
# TODO: Avoid duplication of keywords - ideally by moving definitions out of models.py
'kwargs': {'job_types': ['email', 'sms']}
},
'remove_letter_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=20, hour=4),
'options': {'queue': QueueNames.PERIODIC},
# TODO: Avoid duplication of keywords - ideally by moving definitions out of models.py
'kwargs': {'job_types': ['letter']}
},
'remove_transformed_dvla_files': {
'task': 'remove_transformed_dvla_files',
'schedule': crontab(minute=40, hour=4),
'options': {'queue': QueueNames.PERIODIC}
},
'delete_dvla_response_files': {
'task': 'delete_dvla_response_files',
'schedule': crontab(minute=10, hour=5),
'options': {'queue': QueueNames.PERIODIC}
},
'timeout-job-statistics': {
'task': 'timeout-job-statistics',
'schedule': crontab(minute=0, hour=5),
'options': {'queue': QueueNames.PERIODIC}
}
}
task_queues = []
class NotifyCelery(Celery):
def init_app(self, app):
celery_config = CeleryConfig(app.config)
super().__init__(app.import_name, broker=celery_config.broker_url)
if app.config['INITIALISE_QUEUES']:
for queue in QueueNames.all_queues():
CeleryConfig.task_queues.append(
Queue(queue, Exchange('default'), routing_key=queue)
)
self.config_from_object(celery_config)
super().__init__(app.import_name, broker=app.config['BROKER_URL'])
self.conf.update(app.config)
TaskBase = self.Task
class ContextTask(TaskBase):
@@ -133,5 +14,4 @@ class NotifyCelery(Celery):
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
self.Task = ContextTask
app/celery/provider_tasks.py
+1 -1
View File
@@ -3,7 +3,7 @@ from notifications_utils.recipients import InvalidEmailError
from sqlalchemy.orm.exc import NoResultFound
from app import notify_celery
from app.celery import QueueNames
from app.config import QueueNames
from app.dao import notifications_dao
from app.dao.notifications_dao import update_notification_status_by_id
from app.statsd_decorators import statsd
app/celery/research_mode_tasks.py
+1
View File
@@ -1,6 +1,7 @@
import json
from flask import current_app
from app import notify_celery
from requests import request, RequestException, HTTPError
from app.models import SMS_TYPE
app/celery/scheduled_tasks.py
+21 -2
View File
@@ -9,9 +9,17 @@ from sqlalchemy.exc import SQLAlchemyError
from app.aws import s3
from app import notify_celery
from app import performance_platform_client
from app.dao.date_util import get_month_start_end_date
from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than_limited_by
from app.dao.jobs_dao import (
dao_set_scheduled_jobs_to_pending,
dao_get_jobs_older_than_limited_by
)
from app.dao.monthly_billing_dao import (
get_service_ids_that_need_sms_billing_populated,
create_or_update_monthly_billing_sms
)
from app.dao.notifications_dao import (
dao_timeout_notifications,
is_delivery_slow_for_provider,
@@ -28,7 +36,7 @@ from app.models import LETTER_TYPE
from app.notifications.process_notifications import send_notification_to_queue
from app.statsd_decorators import statsd
from app.celery.tasks import process_job
from app.celery import QueueNames
from app.config import QueueNames
@notify_celery.task(name="remove_csv_files")
@@ -281,3 +289,14 @@ def delete_dvla_response_files_older_than_seven_days():
except SQLAlchemyError as e:
current_app.logger.exception("Failed to delete dvla response files")
raise
@notify_celery.task(name="populate_monthly_billing")
@statsd(namespace="tasks")
def populate_monthly_billing():
# for every service with billable units this month update billing totals for yesterday
# this will overwrite the existing amount.
yesterday = datetime.utcnow() - timedelta(days=1)
start_date, end_date = get_month_start_end_date(yesterday)
services = get_service_ids_that_need_sms_billing_populated(start_date, end_date=end_date)
[create_or_update_monthly_billing_sms(service_id=s.service_id, billing_month=start_date) for s in services]
app/celery/statistics_tasks.py
+1 -1
View File
@@ -10,7 +10,7 @@ from app.dao.statistics_dao import (
)
from app.dao.notifications_dao import get_notification_by_id
from app.models import NOTIFICATION_STATUS_TYPES_COMPLETED
from app.celery import QueueNames
from app.config import QueueNames
def create_initial_notification_statistic_tasks(notification):
app/celery/tasks.py
+3 -3
View File
@@ -19,8 +19,8 @@ from app import (
)
from app.aws import s3
from app.celery import provider_tasks
from app.config import QueueNames
from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
from app.celery import QueueNames
from app.dao.jobs_dao import (
dao_update_job,
dao_get_job_by_id,
@@ -182,7 +182,7 @@ def send_sms(self,
provider_tasks.deliver_sms.apply_async(
[str(saved_notification.id)],
queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE
queue=QueueNames.SEND_SMS if not service.research_mode else QueueNames.RESEARCH_MODE
)
current_app.logger.info(
@@ -227,7 +227,7 @@ def send_email(self,
provider_tasks.deliver_email.apply_async(
[str(saved_notification.id)],
queue=QueueNames.SEND if not service.research_mode else QueueNames.RESEARCH_MODE
queue=QueueNames.SEND_EMAIL if not service.research_mode else QueueNames.RESEARCH_MODE
)
current_app.logger.info("Email {} created at {}".format(saved_notification.id, created_at))
app/commands.py
+17 -1
View File
@@ -5,7 +5,8 @@ from flask.ext.script import Command, Manager, Option
from app import db
from app.models import (PROVIDERS, Service, User, NotificationHistory)
from app.dao.monthly_billing_dao import create_or_update_monthly_billing_sms, get_monthly_billing_sms
from app.models import (PROVIDERS, User)
from app.dao.services_dao import (
delete_service_and_all_associated_db_objects,
dao_fetch_all_services_by_user
@@ -146,3 +147,18 @@ class CustomDbScript(Command):
print('Committed {} updates at {}'.format(len(result), datetime.utcnow()))
db.session.commit()
result = db.session.execute(subq_hist).fetchall()
class PopulateMonthlyBilling(Command):
option_list = (
Option('-s', '-service-id', dest='service_id',
help="Service id to populate monthly billing for"),
Option('-m', '-month', dest="month", help="Use for integer value for month, e.g. 7 for July"),
Option('-y', '-year', dest="year", help="Use for integer value for year, e.g. 2017")
)
def run(self, service_id, month, year):
create_or_update_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
results = get_monthly_billing_sms(service_id, datetime(int(year), int(month), 1))
print("Finished populating data for {} for service id {}".format(month, service_id))
print(results.monthly_totals)
app/config.py
+145 -3
View File
@@ -1,6 +1,10 @@
from datetime import timedelta
import os
import json
from celery.schedules import crontab
from kombu import Exchange, Queue
from app.models import (
EMAIL_TYPE, SMS_TYPE, LETTER_TYPE,
KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST
@@ -14,6 +18,38 @@ if os.environ.get('VCAP_SERVICES'):
extract_cloudfoundry_config()
class QueueNames(object):
PERIODIC = 'periodic-tasks'
PRIORITY = 'priority-tasks'
DATABASE = 'database-tasks'
SEND_COMBINED = 'send-tasks'
SEND_SMS = 'send-sms-tasks'
SEND_EMAIL = 'send-email-tasks'
RESEARCH_MODE = 'research-mode-tasks'
STATISTICS = 'statistics-tasks'
JOBS = 'job-tasks'
RETRY = 'retry-tasks'
NOTIFY = 'notify-internal-tasks'
PROCESS_FTP = 'process-ftp-tasks'
@staticmethod
def all_queues():
return [
QueueNames.PRIORITY,
QueueNames.PERIODIC,
QueueNames.DATABASE,
QueueNames.SEND_COMBINED,
QueueNames.SEND_SMS,
QueueNames.SEND_EMAIL,
QueueNames.RESEARCH_MODE,
QueueNames.STATISTICS,
QueueNames.JOBS,
QueueNames.RETRY,
QueueNames.NOTIFY,
QueueNames.PROCESS_FTP
]
class Config(object):
# URL of admin app
ADMIN_BASE_URL = os.environ['ADMIN_BASE_URL']
@@ -94,6 +130,104 @@ class Config(object):
CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
BROKER_URL = 'sqs://'
BROKER_TRANSPORT_OPTIONS = {
'region': AWS_REGION,
'polling_interval': 1, # 1 second
'visibility_timeout': 310,
'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
}
CELERY_ENABLE_UTC = True,
CELERY_TIMEZONE = 'Europe/London'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_IMPORTS = ('app.celery.tasks', 'app.celery.scheduled_tasks')
CELERYBEAT_SCHEDULE = {
'run-scheduled-jobs': {
'task': 'run-scheduled-jobs',
'schedule': crontab(minute=1),
'options': {'queue': QueueNames.PERIODIC}
},
# 'send-scheduled-notifications': {
# 'task': 'send-scheduled-notifications',
# 'schedule': crontab(minute='*/15'),
# 'options': {'queue': 'periodic'}
# },
'delete-verify-codes': {
'task': 'delete-verify-codes',
'schedule': timedelta(minutes=63),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-invitations': {
'task': 'delete-invitations',
'schedule': timedelta(minutes=66),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-sms-notifications': {
'task': 'delete-sms-notifications',
'schedule': crontab(minute=0, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-email-notifications': {
'task': 'delete-email-notifications',
'schedule': crontab(minute=20, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-letter-notifications': {
'task': 'delete-letter-notifications',
'schedule': crontab(minute=40, hour=0),
'options': {'queue': QueueNames.PERIODIC}
},
'delete-inbound-sms': {
'task': 'delete-inbound-sms',
'schedule': crontab(minute=0, hour=1),
'options': {'queue': QueueNames.PERIODIC}
},
'send-daily-performance-platform-stats': {
'task': 'send-daily-performance-platform-stats',
'schedule': crontab(minute=0, hour=2),
'options': {'queue': QueueNames.PERIODIC}
},
'switch-current-sms-provider-on-slow-delivery': {
'task': 'switch-current-sms-provider-on-slow-delivery',
'schedule': crontab(), # Every minute
'options': {'queue': QueueNames.PERIODIC}
},
'timeout-sending-notifications': {
'task': 'timeout-sending-notifications',
'schedule': crontab(minute=0, hour=3),
'options': {'queue': QueueNames.PERIODIC}
},
'remove_sms_email_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=0, hour=4),
'options': {'queue': QueueNames.PERIODIC},
'kwargs': {'job_types': [EMAIL_TYPE, SMS_TYPE]}
},
'remove_letter_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=20, hour=4),
'options': {'queue': QueueNames.PERIODIC},
'kwargs': {'job_types': [LETTER_TYPE]}
},
'remove_transformed_dvla_files': {
'task': 'remove_transformed_dvla_files',
'schedule': crontab(minute=40, hour=4),
'options': {'queue': QueueNames.PERIODIC}
},
'delete_dvla_response_files': {
'task': 'delete_dvla_response_files',
'schedule': crontab(minute=10, hour=5),
'options': {'queue': QueueNames.PERIODIC}
},
'timeout-job-statistics': {
'task': 'timeout-job-statistics',
'schedule': crontab(minute=0, hour=5),
'options': {'queue': QueueNames.PERIODIC}
}
}
CELERY_QUEUES = []
NOTIFICATIONS_ALERT = 5 # five mins
FROM_NUMBER = 'development'
@@ -132,7 +266,6 @@ class Config(object):
}
FREE_SMS_TIER_FRAGMENT_COUNT = 250000
INITIALISE_QUEUES = False
SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]'))
@@ -142,20 +275,24 @@ class Config(object):
######################
class Development(Config):
INITIALISE_QUEUES = True
SQLALCHEMY_ECHO = False
NOTIFY_EMAIL_DOMAIN = 'notify.tools'
CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
NOTIFY_ENVIRONMENT = 'development'
NOTIFICATION_QUEUE_PREFIX = 'development'
DEBUG = True
for queue in QueueNames.all_queues():
Config.CELERY_QUEUES.append(
Queue(queue, Exchange('default'), routing_key=queue)
)
API_HOST_NAME = "http://localhost:6011"
API_RATE_LIMIT_ENABLED = True
class Test(Config):
INITIALISE_QUEUES = True
NOTIFY_EMAIL_DOMAIN = 'test.notify.com'
FROM_NUMBER = 'testing'
NOTIFY_ENVIRONMENT = 'test'
@@ -169,6 +306,11 @@ class Test(Config):
BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
for queue in QueueNames.all_queues():
Config.CELERY_QUEUES.append(
Queue(queue, Exchange('default'), routing_key=queue)
)
API_RATE_LIMIT_ENABLED = True
API_HOST_NAME = "http://localhost:6011"
app/dao/date_util.py
+13
View File
@@ -16,3 +16,16 @@ def get_april_fools(year):
"""
return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(
tzinfo=None)
def get_month_start_end_date(month_year):
"""
This function return the start and date of the month_year as UTC,
:param month_year: the datetime to calculate the start and end date for that month
:return: start_date, end_date, month
"""
import calendar
_, num_days = calendar.monthrange(month_year.year, month_year.month)
first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0)
last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999)
return first_day, last_day
app/dao/monthly_billing_dao.py
+58
View File
@@ -0,0 +1,58 @@
from datetime import datetime
from app import db
from app.dao.dao_utils import transactional
from app.dao.date_util import get_month_start_end_date
from app.dao.notification_usage_dao import get_billing_data_for_month
from app.models import MonthlyBilling, SMS_TYPE, NotificationHistory
from app.statsd_decorators import statsd
def get_service_ids_that_need_sms_billing_populated(start_date, end_date):
return db.session.query(
NotificationHistory.service_id
).filter(
NotificationHistory.created_at >= start_date,
NotificationHistory.created_at <= end_date,
NotificationHistory.notification_type == SMS_TYPE,
NotificationHistory.billable_units != 0
).distinct().all()
@transactional
def create_or_update_monthly_billing_sms(service_id, billing_month):
start_date, end_date = get_month_start_end_date(billing_month)
monthly = get_billing_data_for_month(service_id=service_id, start_date=start_date, end_date=end_date)
# update monthly
monthly_totals = _monthly_billing_data_to_json(monthly)
row = MonthlyBilling.query.filter_by(year=billing_month.year,
month=datetime.strftime(billing_month, "%B"),
notification_type='sms').first()
if row:
row.monthly_totals = monthly_totals
else:
row = MonthlyBilling(service_id=service_id,
notification_type=SMS_TYPE,
year=billing_month.year,
month=datetime.strftime(billing_month, "%B"),
monthly_totals=monthly_totals)
db.session.add(row)
@statsd(namespace="dao")
def get_monthly_billing_sms(service_id, billing_month):
monthly = MonthlyBilling.query.filter_by(service_id=service_id,
year=billing_month.year,
month=datetime.strftime(billing_month, "%B"),
notification_type=SMS_TYPE).first()
return monthly
def _monthly_billing_data_to_json(monthly):
# total cost must take into account the free allowance.
# might be a good idea to capture free allowance in this table
return [{"billing_units": x.billing_units,
"rate_multiplier": x.rate_multiplier,
"international": x.international,
"rate": x.rate,
"total_cost": (x.billing_units * x.rate_multiplier) * x.rate} for x in monthly]
app/dao/notification_usage_dao.py
+27 -11
View File
@@ -6,7 +6,7 @@ from sqlalchemy import func, case, cast
from sqlalchemy import literal_column
from app import db
from app.dao.date_util import get_financial_year
from app.dao.date_util import get_financial_year, get_month_start_end_date
from app.models import (NotificationHistory,
Rate,
NOTIFICATION_STATUS_TYPES_BILLABLE,
@@ -20,7 +20,7 @@ from app.utils import get_london_month_from_utc_column
@statsd(namespace="dao")
def get_yearly_billing_data(service_id, year):
start_date, end_date = get_financial_year(year)
rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
def get_valid_from(valid_from):
return start_date if valid_from < start_date else valid_from
@@ -35,10 +35,24 @@ def get_yearly_billing_data(service_id, year):
return sum(result, [])
@statsd(namespace="dao")
def get_billing_data_for_month(service_id, start_date, end_date):
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
result = []
# so the start end date in the query are the valid from the rate, not the month - this is going to take some thought
for r, n in zip(rates, rates[1:]):
result.extend(sms_billing_data_per_month_query(r.rate, service_id, max(r.valid_from, start_date),
min(n.valid_from, end_date)))
result.extend(
sms_billing_data_per_month_query(rates[-1].rate, service_id, max(rates[-1].valid_from, start_date), end_date))
return result
@statsd(namespace="dao")
def get_monthly_billing_data(service_id, year):
start_date, end_date = get_financial_year(year)
rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
result = []
for r, n in zip(rates, rates[1:]):
@@ -103,7 +117,7 @@ def sms_yearly_billing_data_query(rate, service_id, start_date, end_date):
return result
def get_rates_for_year(start_date, end_date, notification_type):
def get_rates_for_daterange(start_date, end_date, notification_type):
rates = Rate.query.filter(Rate.notification_type == notification_type).order_by(Rate.valid_from).all()
results = []
for current_rate, current_rate_expiry_date in zip(rates, rates[1:]):
@@ -115,8 +129,10 @@ def get_rates_for_year(start_date, end_date, notification_type):
results.append(rates[-1])
if not results:
if start_date >= rates[-1].valid_from:
results.append(rates[-1])
for x in reversed(rates):
if start_date >= x.valid_from:
results.append(x)
break
return results
@@ -128,12 +144,12 @@ def is_between(date, start_date, end_date):
def sms_billing_data_per_month_query(rate, service_id, start_date, end_date):
month = get_london_month_from_utc_column(NotificationHistory.created_at)
result = db.session.query(
month,
func.sum(NotificationHistory.billable_units),
rate_multiplier(),
month.label('month'),
func.sum(NotificationHistory.billable_units).label('billing_units'),
rate_multiplier().label('rate_multiplier'),
NotificationHistory.international,
NotificationHistory.notification_type,
cast(rate, Float())
cast(rate, Float()).label('rate')
).filter(
*billing_data_filter(SMS_TYPE, start_date, end_date, service_id)
).group_by(
@@ -193,7 +209,7 @@ def get_total_billable_units_for_sent_sms_notifications_in_date_range(start_date
def discover_rate_bounds_for_billing_query(start_date, end_date):
bounds = []
rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
rates = get_rates_for_daterange(start_date, end_date, SMS_TYPE)
def current_valid_from(index):
return rates[index].valid_from
app/delivery/rest.py
+9 -5
View File
@@ -1,6 +1,6 @@
from flask import Blueprint, jsonify
from app.celery import QueueNames
from app.config import QueueNames
from app.delivery import send_to_providers
from app.models import EMAIL_TYPE
from app.celery import provider_tasks
@@ -24,16 +24,20 @@ def send_notification_to_provider(notification_id):
send_response(
send_to_providers.send_email_to_provider,
provider_tasks.deliver_email,
notification)
notification,
QueueNames.SEND_EMAIL
)
else:
send_response(
send_to_providers.send_sms_to_provider,
provider_tasks.deliver_sms,
notification)
notification,
QueueNames.SEND_SMS
)
return jsonify({}), 204
def send_response(send_call, task_call, notification):
def send_response(send_call, task_call, notification, queue):
try:
send_call(notification)
except Exception as e:
@@ -42,4 +46,4 @@ def send_response(send_call, task_call, notification):
notification.id,
notification.notification_type),
e)
task_call.apply_async((str(notification.id)), queue=QueueNames.SEND)
task_call.apply_async((str(notification.id)), queue=queue)
app/delivery/send_to_providers.py
+1 -1
View File
@@ -18,7 +18,7 @@ from app.dao.templates_dao import dao_get_template_by_id
from app.models import SMS_TYPE, KEY_TYPE_TEST, BRANDING_ORG, EMAIL_TYPE, NOTIFICATION_TECHNICAL_FAILURE, \
NOTIFICATION_SENT, NOTIFICATION_SENDING
from app.celery.statistics_tasks import record_initial_job_statistics, create_initial_notification_statistic_tasks
from app.celery.statistics_tasks import create_initial_notification_statistic_tasks
def send_sms_to_provider(notification):
app/invite/rest.py
+1 -1
View File
@@ -4,7 +4,7 @@ from flask import (
jsonify,
current_app)
from app.celery import QueueNames
from app.config import QueueNames
from app.dao.invited_user_dao import (
save_invited_user,
get_invited_user,
app/job/rest.py
+1 -1
View File
@@ -36,7 +36,7 @@ from app.models import JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING, JOB_STATUS_CANC
from app.utils import pagination_links
from app.celery import QueueNames
from app.config import QueueNames
job_blueprint = Blueprint('job', __name__, url_prefix='/service/<uuid:service_id>/job')
app/letters/send_letter_jobs.py
+1 -1
View File
@@ -2,7 +2,7 @@ from flask import Blueprint, jsonify
from flask import request
from app import notify_celery
from app.celery import QueueNames
from app.config import QueueNames
from app.dao.jobs_dao import dao_get_all_letter_jobs
from app.schemas import job_schema
from app.v2.errors import register_errors
app/models.py
+26 -1
View File
@@ -4,7 +4,6 @@ import datetime
from flask import url_for, current_app
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.dialects.postgresql import (
UUID,
JSON
@@ -1246,3 +1245,29 @@ class LetterRateDetail(db.Model):
letter_rate = db.relationship('LetterRate', backref='letter_rates')
page_total = db.Column(db.Integer, nullable=False)
rate = db.Column(db.Numeric(), nullable=False)
class MonthlyBilling(db.Model):
__tablename__ = 'monthly_billing'
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
service = db.relationship('Service', backref='monthly_billing')
month = db.Column(db.String, nullable=False)
year = db.Column(db.Float(asdecimal=False), nullable=False)
notification_type = db.Column(notification_types, nullable=False)
monthly_totals = db.Column(JSON, nullable=False)
updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
__table_args__ = (
UniqueConstraint('service_id', 'month', 'year', 'notification_type', name='uix_monthly_billing'),
)
def serialized(self):
return {
"month": self.month,
"year": self.year,
"service_id": str(self.service_id),
"notification_type": self.notification_type,
"monthly_totals": self.monthly_totals
}
app/notifications/notifications_letter_callback.py
+1 -1
View File
@@ -13,7 +13,7 @@ from app.celery.tasks import update_letter_notifications_statuses
from app.v2.errors import register_errors
from app.notifications.utils import autoconfirm_subscription
from app.schema_validation import validate
from app.celery import QueueNames
from app.config import QueueNames
letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__)
register_errors(letter_callback_blueprint)
app/notifications/notifications_sms_callback.py
+13 -3
View File
@@ -1,4 +1,5 @@
from flask import Blueprint
from flask import current_app
from flask import json
from flask import request, jsonify
@@ -22,6 +23,12 @@ def process_mmg_response():
success, errors = process_sms_client_response(status=str(data.get('status')),
reference=data.get('CID'),
client_name=client_name)
safe_to_log = data.copy()
safe_to_log.pop("MSISDN")
current_app.logger.info(
"Full delivery response from {} for notification: {}\n{}".format(client_name, request.form.get('CID'),
safe_to_log))
if errors:
raise InvalidRequest(errors, status_code=400)
else:
@@ -36,9 +43,12 @@ def process_firetext_response():
client_name=client_name)
if errors:
raise InvalidRequest(errors, status_code=400)
status = request.form.get('status')
success, errors = process_sms_client_response(status=status,
safe_to_log = dict(request.form).copy()
safe_to_log.pop('mobile')
current_app.logger.info(
"Full delivery response from {} for notification: {}\n{}".format(client_name, request.form.get('reference'),
safe_to_log))
success, errors = process_sms_client_response(status=request.form.get('status'),
reference=request.form.get('reference'),
client_name=client_name)
if errors:
app/notifications/process_notifications.py
+9 -4
View File
@@ -1,3 +1,4 @@
import uuid
from datetime import datetime
from flask import current_app
@@ -12,7 +13,7 @@ from app import redis_store
from app.celery import provider_tasks
from notifications_utils.clients import redis
from app.celery import QueueNames
from app.config import QueueNames
from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE, ScheduledNotification
from app.dao.notifications_dao import (dao_create_notification,
dao_delete_notifications_and_history_by_id,
@@ -35,6 +36,7 @@ def check_placeholders(template_object):
def persist_notification(
*,
template_id,
template_version,
recipient,
@@ -53,7 +55,8 @@ def persist_notification(
created_by_id=None
):
notification_created_at = created_at or datetime.utcnow()
if not notification_id and simulated:
notification_id = uuid.uuid4()
notification = Notification(
id=notification_id,
template_id=template_id,
@@ -100,12 +103,14 @@ def persist_notification(
def send_notification_to_queue(notification, research_mode, queue=None):
if research_mode or notification.key_type == KEY_TYPE_TEST:
queue = QueueNames.RESEARCH_MODE
elif not queue:
queue = QueueNames.SEND
if notification.notification_type == SMS_TYPE:
if not queue:
queue = QueueNames.SEND_SMS
deliver_task = provider_tasks.deliver_sms
if notification.notification_type == EMAIL_TYPE:
if not queue:
queue = QueueNames.SEND_EMAIL
deliver_task = provider_tasks.deliver_email
try:
app/notifications/receive_notifications.py
+1 -1
View File
@@ -6,7 +6,7 @@ from notifications_utils.recipients import validate_and_format_phone_number
from app import statsd_client, firetext_client, mmg_client
from app.celery import tasks
from app.celery import QueueNames
from app.config import QueueNames
from app.dao.services_dao import dao_fetch_services_by_sms_sender
from app.dao.inbound_sms_dao import dao_create_inbound_sms
from app.models import InboundSms, INBOUND_SMS_TYPE, SMS_TYPE
app/notifications/rest.py
+1 -1
View File
@@ -6,7 +6,7 @@ from flask import (
)
from app import api_user, authenticated_service
from app.celery import QueueNames
from app.config import QueueNames
from app.dao import (
templates_dao,
notifications_dao
app/notifications/validators.py
+2 -2
View File
@@ -9,7 +9,7 @@ from notifications_utils.clients.redis import rate_limit_cache_key, daily_limit_
from app.dao import services_dao, templates_dao
from app.models import (
INTERNATIONAL_SMS_TYPE, SMS_TYPE,
INTERNATIONAL_SMS_TYPE, SMS_TYPE, EMAIL_TYPE,
KEY_TYPE_TEST, KEY_TYPE_TEAM, SCHEDULE_NOTIFICATIONS
)
from app.service.utils import service_allowed_to_send_to
@@ -104,7 +104,7 @@ def validate_and_format_recipient(send_to, key_type, service, notification_type)
number=send_to,
international=international_phone_info.international
)
else:
elif notification_type == EMAIL_TYPE:
return validate_and_format_email_address(email_address=send_to)
app/schema_validation/definitions.py
+3
View File
@@ -20,6 +20,9 @@ personalisation = {
}
letter_personalisation = dict(personalisation, required=["address_line_1", "postcode"])
https_url = {
"type": "string",
"format": "uri",
app/service/rest.py
-1
View File
@@ -68,7 +68,6 @@ from app.schemas import (
user_schema,
permission_schema,
notification_with_template_schema,
notification_with_personalisation_schema,
notifications_filter_schema,
detailed_service_schema
)
app/service/send_notification.py
+1 -1
View File
@@ -1,4 +1,4 @@
from app.celery import QueueNames
from app.config import QueueNames
from app.notifications.validators import (
check_service_over_daily_message_limit,
validate_and_format_recipient,
app/service/sender.py
+1 -1
View File
@@ -1,6 +1,6 @@
from flask import current_app
from app.celery import QueueNames
from app.config import QueueNames
from app.dao.services_dao import dao_fetch_service_by_id, dao_fetch_active_users_for_service
from app.dao.templates_dao import dao_get_template_by_id
from app.models import EMAIL_TYPE, KEY_TYPE_NORMAL
app/user/rest.py
+1 -1
View File
@@ -4,7 +4,7 @@ from datetime import datetime
from flask import (jsonify, request, Blueprint, current_app)
from app.celery import QueueNames
from app.config import QueueNames
from app.dao.users_dao import (
get_user_by_id,
save_model_user,
app/v2/notifications/create_response.py
+45
View File
@@ -0,0 +1,45 @@
def create_post_sms_response_from_notification(notification, content, from_number, url_root, scheduled_for):
noti = __create_notification_response(notification, url_root, scheduled_for)
noti['content'] = {
'from_number': from_number,
'body': content
}
return noti
def create_post_email_response_from_notification(notification, content, subject, email_from, url_root, scheduled_for):
noti = __create_notification_response(notification, url_root, scheduled_for)
noti['content'] = {
"from_email": email_from,
"body": content,
"subject": subject
}
return noti
def create_post_letter_response_from_notification(notification, content, subject, url_root, scheduled_for):
noti = __create_notification_response(notification, url_root, scheduled_for)
noti['content'] = {
"body": content,
"subject": subject
}
return noti
def __create_notification_response(notification, url_root, scheduled_for):
return {
"id": notification.id,
"reference": notification.client_reference,
"uri": "{}v2/notifications/{}".format(url_root, str(notification.id)),
'template': {
"id": notification.template_id,
"version": notification.template_version,
"uri": "{}services/{}/templates/{}".format(
url_root,
str(notification.service_id),
str(notification.template_id)
)
},
"scheduled_for": scheduled_for if scheduled_for else None
}
app/v2/notifications/notification_schemas.py
+39 -36
View File
@@ -1,5 +1,5 @@
from app.models import NOTIFICATION_STATUS_TYPES, TEMPLATE_TYPES
from app.schema_validation.definitions import (uuid, personalisation)
from app.schema_validation.definitions import (uuid, personalisation, letter_personalisation)
template = {
@@ -192,40 +192,43 @@ post_email_response = {
}
def create_post_sms_response_from_notification(notification, body, from_number, url_root, service_id, scheduled_for):
return {"id": notification.id,
"reference": notification.client_reference,
"content": {'body': body,
'from_number': from_number},
"uri": "{}v2/notifications/{}".format(url_root, str(notification.id)),
"template": __create_template_from_notification(notification=notification,
url_root=url_root,
service_id=service_id),
"scheduled_for": scheduled_for if scheduled_for else None
}
post_letter_request = {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "POST letter notification schema",
"type": "object",
"title": "POST v2/notifications/letter",
"properties": {
"reference": {"type": "string"},
"template_id": uuid,
"personalisation": letter_personalisation
},
"required": ["template_id", "personalisation"]
}
letter_content = {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Letter content for POST letter notification",
"type": "object",
"title": "notification letter content",
"properties": {
"body": {"type": "string"},
"subject": {"type": "string"}
},
"required": ["body", "subject"]
}
def create_post_email_response_from_notification(notification, content, subject, email_from, url_root, service_id,
scheduled_for):
return {
"id": notification.id,
"reference": notification.client_reference,
"content": {
"from_email": email_from,
"body": content,
"subject": subject
},
"uri": "{}v2/notifications/{}".format(url_root, str(notification.id)),
"template": __create_template_from_notification(notification=notification,
url_root=url_root,
service_id=service_id),
"scheduled_for": scheduled_for if scheduled_for else None
}
def __create_template_from_notification(notification, url_root, service_id):
return {
"id": notification.template_id,
"version": notification.template_version,
"uri": "{}services/{}/templates/{}".format(url_root, str(service_id), str(notification.template_id))
}
post_letter_response = {
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "POST sms notification response schema",
"type": "object",
"title": "response v2/notifications/letter",
"properties": {
"id": uuid,
"reference": {"type": ["string", "null"]},
"content": letter_content,
"uri": {"type": "string", "format": "uri"},
"template": template,
"scheduled_for": {"type": ["string", "null"]}
},
"required": ["id", "content", "uri", "template"]
}
app/v2/notifications/post_notifications.py
+92 -40
View File
@@ -1,8 +1,10 @@
from flask import request, jsonify, current_app
import functools
from flask import request, jsonify, current_app, abort
from app import api_user, authenticated_service
from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY
from app.celery import QueueNames
from app.config import QueueNames
from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, PRIORITY
from app.notifications.process_notifications import (
persist_notification,
send_notification_to_queue,
@@ -19,17 +21,26 @@ from app.schema_validation import validate
from app.v2.notifications import v2_notification_blueprint
from app.v2.notifications.notification_schemas import (
post_sms_request,
create_post_sms_response_from_notification,
post_email_request,
create_post_email_response_from_notification)
post_letter_request
)
from app.v2.notifications.create_response import (
create_post_sms_response_from_notification,
create_post_email_response_from_notification,
create_post_letter_response_from_notification
)
@v2_notification_blueprint.route('/<notification_type>', methods=['POST'])
def post_notification(notification_type):
if notification_type == EMAIL_TYPE:
form = validate(request.get_json(), post_email_request)
else:
elif notification_type == SMS_TYPE:
form = validate(request.get_json(), post_sms_request)
elif notification_type == LETTER_TYPE:
form = validate(request.get_json(), post_letter_request)
else:
abort(404)
check_service_has_permission(notification_type, authenticated_service.permissions)
@@ -38,12 +49,6 @@ def post_notification(notification_type):
check_rate_limiting(authenticated_service, api_user)
form_send_to = form['phone_number'] if notification_type == SMS_TYPE else form['email_address']
send_to = validate_and_format_recipient(send_to=form_send_to,
key_type=api_user.key_type,
service=authenticated_service,
notification_type=notification_type)
template, template_with_content = validate_template(
form['template_id'],
form.get('personalisation', {}),
@@ -51,20 +56,74 @@ def post_notification(notification_type):
notification_type,
)
if notification_type == LETTER_TYPE:
notification = process_letter_notification(
form=form,
api_key=api_user,
template=template,
service=authenticated_service,
)
else:
notification = process_sms_or_email_notification(
form=form,
notification_type=notification_type,
api_key=api_user,
template=template,
service=authenticated_service
)
if notification_type == SMS_TYPE:
sms_sender = authenticated_service.sms_sender or current_app.config.get('FROM_NUMBER')
create_resp_partial = functools.partial(
create_post_sms_response_from_notification,
from_number=sms_sender
)
elif notification_type == EMAIL_TYPE:
create_resp_partial = functools.partial(
create_post_email_response_from_notification,
subject=template_with_content.subject,
email_from=authenticated_service.email_from
)
elif notification_type == LETTER_TYPE:
create_resp_partial = functools.partial(
create_post_letter_response_from_notification,
subject=template_with_content.subject,
)
resp = create_resp_partial(
notification=notification,
content=str(template_with_content),
url_root=request.url_root,
scheduled_for=scheduled_for
)
return jsonify(resp), 201
def process_sms_or_email_notification(*, form, notification_type, api_key, template, service):
form_send_to = form['email_address'] if notification_type == EMAIL_TYPE else form['phone_number']
send_to = validate_and_format_recipient(send_to=form_send_to,
key_type=api_key.key_type,
service=service,
notification_type=notification_type)
# Do not persist or send notification to the queue if it is a simulated recipient
simulated = simulated_recipient(send_to, notification_type)
notification = persist_notification(template_id=template.id,
template_version=template.version,
recipient=form_send_to,
service=authenticated_service,
personalisation=form.get('personalisation', None),
notification_type=notification_type,
api_key_id=api_user.id,
key_type=api_user.key_type,
client_reference=form.get('reference', None),
simulated=simulated)
notification = persist_notification(
template_id=template.id,
template_version=template.version,
recipient=form_send_to,
service=service,
personalisation=form.get('personalisation', None),
notification_type=notification_type,
api_key_id=api_key.id,
key_type=api_key.key_type,
client_reference=form.get('reference', None),
simulated=simulated
)
scheduled_for = form.get("scheduled_for", None)
if scheduled_for:
persist_scheduled_notification(notification.id, form["scheduled_for"])
else:
@@ -72,26 +131,19 @@ def post_notification(notification_type):
queue_name = QueueNames.PRIORITY if template.process_type == PRIORITY else None
send_notification_to_queue(
notification=notification,
research_mode=authenticated_service.research_mode,
research_mode=service.research_mode,
queue=queue_name
)
else:
current_app.logger.info("POST simulated notification for id: {}".format(notification.id))
if notification_type == SMS_TYPE:
sms_sender = authenticated_service.sms_sender or current_app.config.get('FROM_NUMBER')
resp = create_post_sms_response_from_notification(notification=notification,
body=str(template_with_content),
from_number=sms_sender,
url_root=request.url_root,
service_id=authenticated_service.id,
scheduled_for=scheduled_for)
else:
resp = create_post_email_response_from_notification(notification=notification,
content=str(template_with_content),
subject=template_with_content.subject,
email_from=authenticated_service.email_from,
url_root=request.url_root,
service_id=authenticated_service.id,
scheduled_for=scheduled_for)
return jsonify(resp), 201
return notification
def process_letter_notification(*, form, api_key, template, service):
# create job
# create notification
# trigger build_dvla_file task
raise NotImplementedError