darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-18 13:38:53 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into service-not-found-returns-404

Browse Source 
Conflicts:
	app/errors.py
This commit is contained in:
Rebecca Law
2016-02-25 15:33:53 +00:00
parent 0f333de2cc ab495d983e
commit ff6cad1acf
36 changed files with 917 additions and 767 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/__init__.py
View File

@@ -1,3 +1,4 @@
import uuid
import os
import re
from flask import request, url_for
@@ -51,7 +52,7 @@ def create_app():
application.register_blueprint(user_blueprint, url_prefix='/user')
application.register_blueprint(template_blueprint)
application.register_blueprint(status_blueprint, url_prefix='/status')
application.register_blueprint(notifications_blueprint, url_prefix='/notifications')
application.register_blueprint(notifications_blueprint)
application.register_blueprint(job_blueprint)
application.register_blueprint(invite_blueprint)
@@ -101,3 +102,7 @@ def email_safe(string):
character.lower() if character.isalnum() or character == "." else ""
for character in re.sub("\s+", ".", string.strip())
])
def create_uuid():
return str(uuid.uuid4())

7
app/aws/s3.py Normal file
View File

@@ -0,0 +1,7 @@
from boto3 import resource
def get_job_from_s3(bucket_name, job_id):
s3 = resource('s3')
key = s3.Object(bucket_name, '{}.csv'.format(job_id))
return key.get()['Body'].read().decode('utf-8')

102
app/celery/tasks.py
View File

@@ -1,55 +1,125 @@
from app import create_uuid
from app import notify_celery, encryption, firetext_client, aws_ses_client
from app.clients.email.aws_ses import AwsSesClientException
from app.clients.sms.firetext import FiretextClientException
from app.dao.templates_dao import get_model_templates
from app.dao.notifications_dao import save_notification
from app.dao.templates_dao import dao_get_template_by_id
from app.dao.notifications_dao import dao_create_notification, dao_update_notification
from app.dao.jobs_dao import dao_update_job, dao_get_job_by_id
from app.models import Notification
from flask import current_app
from sqlalchemy.exc import SQLAlchemyError
from app.aws import s3
from app.csv import get_recipient_from_csv
from datetime import datetime
@notify_celery.task(name="process-job")
def process_job(job_id):
start = datetime.utcnow()
job = dao_get_job_by_id(job_id)
job.status = 'in progress'
dao_update_job(job)
file = s3.get_job_from_s3(job.bucket_name, job_id)
recipients = get_recipient_from_csv(file)
for recipient in recipients:
encrypted = encryption.encrypt({
'template': job.template_id,
'job': str(job.id),
'to': recipient
})
if job.template.template_type == 'sms':
send_sms.apply_async((
str(job.service_id),
str(create_uuid()),
encrypted,
str(datetime.utcnow())),
queue='bulk-sms'
)
if job.template.template_type == 'email':
send_email.apply_async((
str(job.service_id),
str(create_uuid()),
job.template.subject,
"{}@{}".format(job.service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']),
encrypted,
str(datetime.utcnow())),
queue='bulk-email')
finished = datetime.utcnow()
job.status = 'finished'
job.processing_started = start
job.processing_finished = finished
dao_update_job(job)
current_app.logger.info(
"Job {} created at {} started at {} finished at {}".format(job_id, job.created_at, start, finished)
)
@notify_celery.task(name="send-sms")
def send_sms(service_id, notification_id, encrypted_notification):
def send_sms(service_id, notification_id, encrypted_notification, created_at):
notification = encryption.decrypt(encrypted_notification)
template = get_model_templates(notification['template'])
template = dao_get_template_by_id(notification['template'])
client = firetext_client
try:
sent_at = datetime.utcnow()
notification_db_object = Notification(
id=notification_id,
template_id=notification['template'],
to=notification['to'],
service_id=service_id,
status='sent'
job_id=notification.get('job', None),
status='sent',
created_at=created_at,
sent_at=sent_at,
sent_by=client.get_name()
)
save_notification(notification_db_object)
dao_create_notification(notification_db_object)
try:
firetext_client.send_sms(notification['to'], template.content)
client.send_sms(notification['to'], template.content)
except FiretextClientException as e:
current_app.logger.debug(e)
save_notification(notification_db_object, {"status": "failed"})
notification_db_object.status = 'failed'
dao_update_notification(notification_db_object)
current_app.logger.info(
"SMS {} created at {} sent at {}".format(notification_id, created_at, sent_at)
)
except SQLAlchemyError as e:
current_app.logger.debug(e)
@notify_celery.task(name="send-email")
def send_email(service_id, notification_id, subject, from_address, encrypted_notification):
def send_email(service_id, notification_id, subject, from_address, encrypted_notification, created_at):
notification = encryption.decrypt(encrypted_notification)
template = get_model_templates(notification['template'])
template = dao_get_template_by_id(notification['template'])
client = aws_ses_client
try:
sent_at = datetime.utcnow()
notification_db_object = Notification(
id=notification_id,
template_id=notification['template'],
to=notification['to'],
service_id=service_id,
status='sent'
job_id=notification.get('job', None),
status='sent',
created_at=created_at,
sent_at=sent_at,
sent_by=client.get_name()
)
save_notification(notification_db_object)
dao_create_notification(notification_db_object)
try:
aws_ses_client.send_email(
client.send_email(
from_address,
notification['to'],
subject,
@@ -57,8 +127,12 @@ def send_email(service_id, notification_id, subject, from_address, encrypted_not
)
except AwsSesClientException as e:
current_app.logger.debug(e)
save_notification(notification_db_object, {"status": "failed"})
notification_db_object.status = 'failed'
dao_update_notification(notification_db_object)
current_app.logger.info(
"Email {} created at {} sent at {}".format(notification_id, created_at, sent_at)
)
except SQLAlchemyError as e:
current_app.logger.debug(e)

3
app/clients/email/__init__.py
View File

@@ -15,3 +15,6 @@ class EmailClient(Client):
def send_email(self, *args, **kwargs):
raise NotImplemented('TODO Need to implement.')
def get_name(self):
raise NotImplemented('TODO Need to implement.')

4
app/clients/email/aws_ses.py
View File

@@ -15,6 +15,10 @@ class AwsSesClient(EmailClient):
def init_app(self, region, *args, **kwargs):
self._client = boto3.client('ses', region_name=region)
super(AwsSesClient, self).__init__(*args, **kwargs)
self.name = 'ses'
def get_name(self):
return self.name
def send_email(self,
source,

3
app/clients/sms/__init__.py
View File

@@ -15,3 +15,6 @@ class SmsClient(Client):
def send_sms(self, *args, **kwargs):
raise NotImplemented('TODO Need to implement.')
def get_name(self):
raise NotImplemented('TODO Need to implement.')

4
app/clients/sms/firetext.py
View File

@@ -21,6 +21,10 @@ class FiretextClient(SmsClient):
super(SmsClient, self).__init__(*args, **kwargs)
self.api_key = config.config.get('FIRETEXT_API_KEY')
self.from_number = config.config.get('FIRETEXT_NUMBER')
self.name = 'firetext'
def get_name(self):
return self.name
def send_sms(self, to, content):

4
app/clients/sms/twilio.py
View File

@@ -22,6 +22,10 @@ class TwilioClient(SmsClient):
config.config.get('TWILIO_ACCOUNT_SID'),
config.config.get('TWILIO_AUTH_TOKEN'))
self.from_number = config.config.get('TWILIO_NUMBER')
self.name = 'twilio'
def get_name(self):
return self.name
def send_sms(self, to, content):
try:

12
app/csv.py Normal file
View File

@@ -0,0 +1,12 @@
import csv
def get_recipient_from_csv(file_data):
numbers = []
reader = csv.DictReader(
file_data.splitlines(),
lineterminator='\n',
quoting=csv.QUOTE_NONE)
for i, row in enumerate(reader):
numbers.append(row['to'].replace(' ', ''))
return numbers

31
app/dao/jobs_dao.py
View File

@@ -2,24 +2,23 @@ from app import db
from app.models import Job
def save_job(job, update_dict={}):
if update_dict:
update_dict.pop('id', None)
update_dict.pop('service', None)
update_dict.pop('template', None)
Job.query.filter_by(id=job.id).update(update_dict)
else:
db.session.add(job)
db.session.commit()
def dao_get_job_by_service_id_and_job_id(service_id, job_id):
return Job.query.filter_by(service_id=service_id, id=job_id).first()
def get_job(service_id, job_id):
return Job.query.filter_by(service_id=service_id, id=job_id).one()
def get_jobs_by_service(service_id):
def dao_get_jobs_by_service_id(service_id):
return Job.query.filter_by(service_id=service_id).all()
def _get_jobs():
return Job.query.all()
def dao_get_job_by_id(job_id):
return Job.query.filter_by(id=job_id).first()
def dao_create_job(job):
db.session.add(job)
db.session.commit()
def dao_update_job(job):
db.session.add(job)
db.session.commit()

16
app/dao/notifications_dao.py
View File

@@ -2,15 +2,13 @@ from app import db
from app.models import Notification
def save_notification(notification, update_dict={}):
if update_dict:
update_dict.pop('id', None)
update_dict.pop('job', None)
update_dict.pop('service', None)
update_dict.pop('template', None)
Notification.query.filter_by(id=notification.id).update(update_dict)
else:
db.session.add(notification)
def dao_create_notification(notification):
db.session.add(notification)
db.session.commit()
def dao_update_notification(notification):
db.session.add(notification)
db.session.commit()

32
app/dao/templates_dao.py
View File

@@ -3,34 +3,6 @@ from app.models import (Template, Service)
from sqlalchemy import asc
def save_model_template(template, update_dict=None):
if update_dict:
update_dict.pop('id', None)
service = update_dict.pop('service')
Template.query.filter_by(id=template.id).update(update_dict)
template.service = service
else:
db.session.add(template)
db.session.commit()
def delete_model_template(template):
db.session.delete(template)
db.session.commit()
def get_model_templates(template_id=None, service_id=None):
# TODO need better mapping from function params to sql query.
if template_id and service_id:
return Template.query.filter_by(
id=template_id, service_id=service_id).one()
elif template_id:
return Template.query.filter_by(id=template_id).one()
elif service_id:
return Template.query.filter_by(service=Service.query.get(service_id)).all()
return Template.query.all()
def dao_create_template(template):
db.session.add(template)
db.session.commit()
@@ -45,5 +17,9 @@ def dao_get_template_by_id_and_service_id(template_id, service_id):
return Template.query.filter_by(id=template_id, service_id=service_id).first()
def dao_get_template_by_id(template_id):
return Template.query.filter_by(id=template_id).first()
def dao_get_all_templates_for_service(service_id):
return Template.query.filter_by(service=Service.query.get(service_id)).order_by(asc(Template.created_at)).all()

159
app/job/rest.py
View File

@@ -1,148 +1,81 @@
import boto3
import json
from flask import (
Blueprint,
jsonify,
request,
current_app
request
)
from sqlalchemy.exc import DataError
from sqlalchemy.orm.exc import NoResultFound
from app.dao.jobs_dao import (
save_job,
get_job,
get_jobs_by_service
dao_create_job,
dao_get_job_by_service_id_and_job_id,
dao_get_jobs_by_service_id,
dao_update_job
)
from app.dao import notifications_dao
from app.dao.services_dao import (
dao_fetch_service_by_id
)
from app.schemas import (
job_schema,
jobs_schema,
job_schema_load_json,
notification_status_schema,
notifications_status_schema,
notification_status_schema_load_json
jobs_schema
)
from app.celery.tasks import process_job
job = Blueprint('job', __name__, url_prefix='/service/<service_id>/job')
from app.errors import register_errors
register_errors(job)
@job.route('/<job_id>', methods=['GET'])
def get_job_by_service_and_job_id(service_id, job_id):
job = dao_get_job_by_service_id_and_job_id(service_id, job_id)
if not job:
return jsonify(result="error", message="Job {} not found for service {}".format(job_id, service_id)), 404
data, errors = job_schema.dump(job)
return jsonify(data=data)
@job.route('', methods=['GET'])
def get_job_for_service(service_id, job_id=None):
if job_id:
try:
job = get_job(service_id, job_id)
data, errors = job_schema.dump(job)
return jsonify(data=data)
except DataError:
return jsonify(result="error", message="Invalid job id"), 400
except NoResultFound:
return jsonify(result="error", message="Job not found"), 404
else:
jobs = get_jobs_by_service(service_id)
data, errors = jobs_schema.dump(jobs)
return jsonify(data=data)
def get_jobs_by_service(service_id):
jobs = dao_get_jobs_by_service_id(service_id)
data, errors = jobs_schema.dump(jobs)
return jsonify(data=data)
@job.route('', methods=['POST'])
def create_job(service_id):
job, errors = job_schema.load(request.get_json())
service = dao_fetch_service_by_id(service_id)
if not service:
return jsonify(result="error", message="Service {} not found".format(service_id)), 404
data = request.get_json()
data.update({
"service": service_id
})
job, errors = job_schema.load(data)
if errors:
return jsonify(result="error", message=errors), 400
save_job(job)
_enqueue_job(job)
dao_create_job(job)
process_job.apply_async([str(job.id)], queue="process-job")
return jsonify(data=job_schema.dump(job).data), 201
@job.route('/<job_id>', methods=['PUT'])
@job.route('/<job_id>', methods=['POST'])
def update_job(service_id, job_id):
fetched_job = dao_get_job_by_service_id_and_job_id(service_id, job_id)
if not fetched_job:
return jsonify(result="error", message="Job {} not found for service {}".format(job_id, service_id)), 404
job = get_job(service_id, job_id)
update_dict, errors = job_schema_load_json.load(request.get_json())
current_data = dict(job_schema.dump(fetched_job).data.items())
current_data.update(request.get_json())
update_dict, errors = job_schema.load(current_data)
if errors:
return jsonify(result="error", message=errors), 400
save_job(job, update_dict=update_dict)
return jsonify(data=job_schema.dump(job).data), 200
@job.route('/<job_id>/notification', methods=['POST'])
def create_notification_for_job(service_id, job_id):
# TODO assert service_id == payload service id
# and same for job id
notification, errors = notification_status_schema.load(request.get_json())
if errors:
return jsonify(result="error", message=errors), 400
notifications_dao.save_notification(notification)
return jsonify(data=notification_status_schema.dump(notification).data), 201
@job.route('/<job_id>/notification', methods=['GET'])
@job.route('/<job_id>/notification/<notification_id>')
def get_notification_for_job(service_id, job_id, notification_id=None):
if notification_id:
try:
notification = notifications_dao.get_notification_for_job(service_id, job_id, notification_id)
data, errors = notification_status_schema.dump(notification)
return jsonify(data=data)
except DataError:
return jsonify(result="error", message="Invalid notification id"), 400
except NoResultFound:
return jsonify(result="error", message="Notification not found"), 404
else:
notifications = notifications_dao.get_notifications_for_job(service_id, job_id)
data, errors = notifications_status_schema.dump(notifications)
return jsonify(data=data)
@job.route('/<job_id>/notification/<notification_id>', methods=['PUT'])
def update_notification_for_job(service_id, job_id, notification_id):
notification = notifications_dao.get_notification_for_job(service_id, job_id, notification_id)
update_dict, errors = notification_status_schema_load_json.load(request.get_json())
if errors:
return jsonify(result="error", message=errors), 400
notifications_dao.save_notification(notification, update_dict=update_dict)
return jsonify(data=job_schema.dump(notification).data), 200
def _enqueue_job(job):
aws_region = current_app.config['AWS_REGION']
queue_name = current_app.config['NOTIFY_JOB_QUEUE']
queue = boto3.resource('sqs', region_name=aws_region).create_queue(QueueName=queue_name)
data = {
'id': str(job.id),
'service': str(job.service.id),
'template': job.template.id,
'bucket_name': job.bucket_name,
'file_name': job.file_name,
'original_file_name': job.original_file_name
}
job_json = json.dumps(data)
queue.send_message(MessageBody=job_json,
MessageAttributes={'id': {'StringValue': str(job.id), 'DataType': 'String'},
'service': {'StringValue': str(job.service.id), 'DataType': 'String'},
'template': {'StringValue': str(job.template.id), 'DataType': 'String'},
'bucket_name': {'StringValue': job.bucket_name, 'DataType': 'String'},
'file_name': {'StringValue': job.file_name, 'DataType': 'String'},
'original_file_name': {'StringValue': job.original_file_name,
'DataType': 'String'}})
dao_update_job(update_dict)
return jsonify(data=job_schema.dump(update_dict).data), 200

19
app/models.py
View File

@@ -163,6 +163,16 @@ class Job(db.Model):
onupdate=datetime.datetime.now)
status = db.Column(db.Enum(*JOB_STATUS_TYPES, name='job_status_types'), nullable=False, default='pending')
notification_count = db.Column(db.Integer, nullable=False)
processing_started = db.Column(
db.DateTime,
index=False,
unique=False,
nullable=True)
processing_finished = db.Column(
db.DateTime,
index=False,
unique=False,
nullable=True)
VERIFY_CODE_TYPES = ['email', 'sms']
@@ -217,8 +227,13 @@ class Notification(db.Model):
db.DateTime,
index=False,
unique=False,
nullable=False,
default=datetime.datetime.now)
nullable=False)
sent_at = db.Column(
db.DateTime,
index=False,
unique=False,
nullable=True)
sent_by = db.Column(db.String, nullable=True)
updated_at = db.Column(
db.DateTime,
index=False,

124
app/notifications/rest.py
View File

@@ -1,4 +1,4 @@
import uuid
from datetime import datetime
from flask import (
Blueprint,
@@ -7,11 +7,9 @@ from flask import (
current_app
)
from app import api_user, encryption
from app.aws_sqs import add_notification_to_queue
from app import api_user, encryption, create_uuid
from app.dao import (
templates_dao,
users_dao,
services_dao,
notifications_dao
)
@@ -29,12 +27,11 @@ from app.errors import register_errors
register_errors(notifications)
def create_notification_id():
return str(uuid.uuid4())
SMS_NOTIFICATION = 'sms'
EMAIL_NOTIFICATION = 'email'
@notifications.route('/<string:notification_id>', methods=['GET'])
@notifications.route('/notifications/<string:notification_id>', methods=['GET'])
def get_notifications(notification_id):
try:
notification = notifications_dao.get_notification(api_user['client'], notification_id)
@@ -43,87 +40,68 @@ def get_notifications(notification_id):
return jsonify(result="error", message="not found"), 404
@notifications.route('/sms', methods=['POST'])
@notifications.route('/notifications/sms', methods=['POST'])
def create_sms_notification():
notification, errors = sms_template_notification_schema.load(request.get_json())
if errors:
return jsonify(result="error", message=errors), 400
template = templates_dao.dao_get_template_by_id_and_service_id(
template_id=notification['template'],
service_id=api_user['client']
)
if not template:
return jsonify(result="error", message={'template': ['Template not found']}), 400
service = services_dao.dao_fetch_service_by_id(api_user['client'])
if service.restricted:
if notification['to'] not in [user.email_address for user in service.users]:
return jsonify(result="error", message={'to': ['Invalid phone number for restricted service']}), 400
notification_id = create_notification_id()
send_sms.apply_async((
api_user['client'],
notification_id,
encryption.encrypt(notification)),
queue='sms')
return jsonify({'notification_id': notification_id}), 201
return send_notification(notification_type=SMS_NOTIFICATION)
@notifications.route('/email', methods=['POST'])
@notifications.route('/notifications/email', methods=['POST'])
def create_email_notification():
notification, errors = email_notification_schema.load(request.get_json())
return send_notification(notification_type=EMAIL_NOTIFICATION)
def send_notification(notification_type):
assert notification_type
service_id = api_user['client']
schema = sms_template_notification_schema if notification_type is SMS_NOTIFICATION else email_notification_schema
notification, errors = schema.load(request.get_json())
if errors:
return jsonify(result="error", message=errors), 400
template = templates_dao.dao_get_template_by_id_and_service_id(
template_id=notification['template'],
service_id=api_user['client']
service_id=service_id
)
if not template:
return jsonify(result="error", message={'template': ['Template not found']}), 400
return jsonify(
result="error",
message={
'template': ['Template {} not found for service {}'.format(notification['template'], service_id)]
}
), 404
service = services_dao.dao_fetch_service_by_id(api_user['client'])
if service.restricted:
if notification['to'] not in [user.email_address for user in service.users]:
return jsonify(result="error", message={'to': ['Email address not permitted for restricted service']}), 400
if notification_type is SMS_NOTIFICATION:
if notification['to'] not in [user.mobile_number for user in service.users]:
return jsonify(
result="error", message={'to': ['Invalid phone number for restricted service']}), 400
else:
if notification['to'] not in [user.email_address for user in service.users]:
return jsonify(
result="error", message={'to': ['Email address not permitted for restricted service']}), 400
notification_id = create_notification_id()
notification_id = create_uuid()
send_email.apply_async((
api_user['client'],
notification_id,
template.subject,
"{}@{}".format(service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']),
encryption.encrypt(notification)),
queue='email')
return jsonify({'notification_id': notification_id}), 201
@notifications.route('/sms/service/<service_id>', methods=['POST'])
def create_sms_for_service(service_id):
resp_json = request.get_json()
notification, errors = sms_template_notification_schema.load(resp_json)
if errors:
return jsonify(result="error", message=errors), 400
template_id = notification['template']
job_id = notification['job']
# TODO: job/job_id is in notification and can used to update job status
# TODO: remove once beta is reading notifications from the queue
template = templates_dao.get_model_templates(template_id)
if template.service.id != uuid.UUID(service_id):
message = "Invalid template: id {} for service id: {}".format(template.id, service_id)
return jsonify(result="error", message=message), 400
notification_id = add_notification_to_queue(service_id, template_id, 'sms', notification)
if notification_type is SMS_NOTIFICATION:
send_sms.apply_async((
service_id,
notification_id,
encryption.encrypt(notification),
str(datetime.utcnow())),
queue='sms')
else:
send_email.apply_async((
service_id,
notification_id,
template.subject,
"{}@{}".format(service.email_from, current_app.config['NOTIFY_EMAIL_DOMAIN']),
encryption.encrypt(notification),
str(datetime.utcnow())),
queue='email')
return jsonify({'notification_id': notification_id}), 201

13
app/schemas.py
View File

@@ -108,6 +108,16 @@ class SmsTemplateNotificationSchema(SmsNotificationSchema):
job = fields.String()
class JobSmsTemplateNotificationSchema(SmsNotificationSchema):
template = fields.Int(required=True)
job = fields.String(required=True)
class JobEmailTemplateNotificationSchema(EmailNotificationSchema):
template = fields.Int(required=True)
job = fields.String(required=True)
class SmsAdminNotificationSchema(SmsNotificationSchema):
content = fields.Str(required=True)
@@ -143,12 +153,13 @@ api_keys_schema = ApiKeySchema(many=True)
job_schema = JobSchema()
job_schema_load_json = JobSchema(load_json=True)
jobs_schema = JobSchema(many=True)
# TODO: Remove this schema once the admin app has stopped using the /user/<user_id>code endpoint
old_request_verify_code_schema = OldRequestVerifyCodeSchema()
request_verify_code_schema = RequestVerifyCodeSchema()
sms_admin_notification_schema = SmsAdminNotificationSchema()
sms_template_notification_schema = SmsTemplateNotificationSchema()
job_sms_template_notification_schema = JobSmsTemplateNotificationSchema()
email_notification_schema = EmailNotificationSchema()
job_email_template_notification_schema = JobEmailTemplateNotificationSchema()
notification_status_schema = NotificationStatusSchema()
notifications_status_schema = NotificationStatusSchema(many=True)
notification_status_schema_load_json = NotificationStatusSchema(load_json=True)