Merge branch 'master' into celery-spike

Conflicts:
	app/__init__.py
	app/notifications/rest.py
	config.py
	wsgi.py

app/__init__.py

@@ -18,6 +18,7 @@ db = SQLAlchemy()
 ma = Marshmallow()
 notify_alpha_client = NotifyAPIClient()
 celery = NotifyCelery()
+
 api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
 
 
@@ -31,7 +32,6 @@ def create_app(config_name, config_overrides=None):
     ma.init_app(application)
     init_app(application, config_overrides)
     logging.init_app(application)
-    notify_alpha_client.init_app(application)
 
     celery.init_app(application)
 
@@ -78,7 +78,6 @@ def init_app(app, config_overrides):
         return response
 
 
-
 def convert_to_boolean(value):
     """Turn strings to bools if they look like them
 

app/authentication/auth.py

@@ -1,10 +1,11 @@
-from flask import request, jsonify, _request_ctx_stack
-from client.authentication import decode_jwt_token, get_token_issuer
-from client.errors import TokenDecodeError, TokenRequestError, TokenExpiredError, TokenPayloadError
+from flask import request, jsonify, _request_ctx_stack, current_app
+from notifications_python_client.authentication import decode_jwt_token, get_token_issuer
+from notifications_python_client.errors import TokenDecodeError, TokenRequestError, TokenExpiredError, TokenPayloadError
 from app.dao.api_key_dao import get_unsigned_secrets
 
 
 def authentication_response(message, code):
+    current_app.logger.info(message)
     return jsonify(
         error=message
     ), code
@@ -27,8 +28,7 @@ def requires_auth():
         return authentication_response("Invalid token: signature", 403)
     if api_client is None:
         authentication_response("Invalid credentials", 403)
-    # If the api_client does not have any secrets return response saying that
-    errors_resp = authentication_response("Invalid token: api client has no secrets", 403)
+
     for secret in api_client['secret']:
         try:
             decode_jwt_token(
@@ -53,12 +53,16 @@ def requires_auth():
 
 
 def fetch_client(client):
-    from flask import current_app
     if client == current_app.config.get('ADMIN_CLIENT_USER_NAME'):
         return {
             "client": client,
             "secret": [current_app.config.get('ADMIN_CLIENT_SECRET')]
         }
+    elif client == current_app.config.get('DELIVERY_CLIENT_USER_NAME'):
+        return {
+            "client": client,
+            "secret": [current_app.config.get('DELIVERY_CLIENT_SECRET')]
+        }
     else:
         return {
             "client": client,

app/aws_sqs.py

@@ -0,0 +1,20 @@
+import uuid
+import boto3
+from itsdangerous import URLSafeSerializer
+from flask import current_app
+
+
+def add_notification_to_queue(service_id, template_id, type_, notification):
+    q = boto3.resource(
+        'sqs', region_name=current_app.config['AWS_REGION']
+    ).create_queue(QueueName="{}_{}".format(
+        current_app.config['NOTIFICATION_QUEUE_PREFIX'],
+        str(service_id)))
+    message_id = str(uuid.uuid4())
+    serializer = URLSafeSerializer(current_app.config.get('SECRET_KEY'))
+    encrypted = serializer.dumps(notification, current_app.config.get('DANGEROUS_SALT'))
+    q.send_message(MessageBody=encrypted,
+                   MessageAttributes={'type': {'StringValue': type_, 'DataType': 'String'},
+                                      'message_id': {'StringValue': message_id, 'DataType': 'String'},
+                                      'service_id': {'StringValue': str(service_id), 'DataType': 'String'},
+                                      'template_id': {'StringValue': str(template_id), 'DataType': 'String'}})

app/celery/celery.py

@@ -15,7 +15,3 @@ class NotifyCelery(Celery):
                 with app.app_context():
                     return TaskBase.__call__(self, *args, **kwargs)
         self.Task = ContextTask
-
-
-
-

app/dao/jobs_dao.py

@@ -2,8 +2,14 @@ from app import db
 from app.models import Job
 
 
-def save_job(job):
-    db.session.add(job)
+def save_job(job, update_dict={}):
+    if update_dict:
+        update_dict.pop('id', None)
+        update_dict.pop('service', None)
+        update_dict.pop('template', None)
+        Job.query.filter_by(id=job.id).update(update_dict)
+    else:
+        db.session.add(job)
     db.session.commit()
 
 

app/dao/notifications_dao.py

@@ -0,0 +1,22 @@
+from app import db
+from app.models import Notification
+
+
+def save_notification(notification, update_dict={}):
+    if update_dict:
+        update_dict.pop('id', None)
+        update_dict.pop('job', None)
+        update_dict.pop('service', None)
+        update_dict.pop('template', None)
+        Notification.query.filter_by(id=notification.id).update(update_dict)
+    else:
+        db.session.add(notification)
+    db.session.commit()
+
+
+def get_notification(service_id, job_id, notification_id):
+    return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one()
+
+
+def get_notifications(service_id, job_id):
+    return Notification.query.filter_by(service_id=service_id, job_id=job_id).all()

app/dao/templates_dao.py

@@ -23,7 +23,6 @@ def delete_model_template(template):
 
 
 def get_model_templates(template_id=None, service_id=None):
-    temp = Template.query.first()
     # TODO need better mapping from function params to sql query.
     if template_id and service_id:
         return Template.query.filter_by(

app/job/rest.py

@@ -17,9 +17,19 @@ from app.dao.jobs_dao import (
     get_jobs_by_service
 )
 
+from app.dao.notifications_dao import (
+    save_notification,
+    get_notification,
+    get_notifications
+)
+
 from app.schemas import (
     job_schema,
-    jobs_schema
+    jobs_schema,
+    job_schema_load_json,
+    notification_status_schema,
+    notifications_status_schema,
+    notification_status_schema_load_json
 )
 
 job = Blueprint('job', __name__, url_prefix='/service/<service_id>/job')
@@ -56,12 +66,88 @@ def create_job(service_id):
     return jsonify(data=job_schema.dump(job).data), 201
 
 
+@job.route('/<job_id>', methods=['PUT'])
+def update_job(service_id, job_id):
+
+    job = get_job(service_id, job_id)
+    update_dict, errors = job_schema_load_json.load(request.get_json())
+    if errors:
+        return jsonify(result="error", message=errors), 400
+    try:
+        save_job(job, update_dict=update_dict)
+    except Exception as e:
+        return jsonify(result="error", message=str(e)), 400
+    return jsonify(data=job_schema.dump(job).data), 200
+
+
+@job.route('/<job_id>/notification', methods=['POST'])
+def create_notification_for_job(service_id, job_id):
+
+    # TODO assert service_id == payload service id
+    # and same for job id
+    notification, errors = notification_status_schema.load(request.get_json())
+    if errors:
+        return jsonify(result="error", message=errors), 400
+    try:
+        save_notification(notification)
+    except Exception as e:
+        return jsonify(result="error", message=str(e)), 500
+    return jsonify(data=notification_status_schema.dump(notification).data), 201
+
+
+@job.route('/<job_id>/notification', methods=['GET'])
+@job.route('/<job_id>/notification/<notification_id>')
+def get_notification_for_job(service_id, job_id, notification_id=None):
+    if notification_id:
+        try:
+            notification = get_notification(service_id, job_id, notification_id)
+            data, errors = notification_status_schema.dump(notification)
+            return jsonify(data=data)
+        except DataError:
+            return jsonify(result="error", message="Invalid notification id"), 400
+        except NoResultFound:
+            return jsonify(result="error", message="Notification not found"), 404
+    else:
+        notifications = get_notifications(service_id, job_id)
+        data, errors = notifications_status_schema.dump(notifications)
+        return jsonify(data=data)
+
+
+@job.route('/<job_id>/notification/<notification_id>', methods=['PUT'])
+def update_notification_for_job(service_id, job_id, notification_id):
+
+    notification = get_notification(service_id, job_id, notification_id)
+    update_dict, errors = notification_status_schema_load_json.load(request.get_json())
+
+    if errors:
+        return jsonify(result="error", message=errors), 400
+    try:
+        save_notification(notification, update_dict=update_dict)
+    except Exception as e:
+        return jsonify(result="error", message=str(e)), 400
+
+    return jsonify(data=job_schema.dump(notification).data), 200
+
+
 def _enqueue_job(job):
     aws_region = current_app.config['AWS_REGION']
     queue_name = current_app.config['NOTIFY_JOB_QUEUE']
 
     queue = boto3.resource('sqs', region_name=aws_region).create_queue(QueueName=queue_name)
-    job_json = json.dumps({'job_id': str(job.id),  'service_id': str(job.service.id)})
+    data = {
+        'id': str(job.id),
+        'service': str(job.service.id),
+        'template': job.template.id,
+        'bucket_name': job.bucket_name,
+        'file_name': job.file_name,
+        'original_file_name': job.original_file_name
+    }
+    job_json = json.dumps(data)
     queue.send_message(MessageBody=job_json,
-                       MessageAttributes={'job_id': {'StringValue': str(job.id), 'DataType': 'String'},
-                                          'service_id': {'StringValue': str(job.service.id), 'DataType': 'String'}})
+                       MessageAttributes={'id': {'StringValue': str(job.id), 'DataType': 'String'},
+                                          'service': {'StringValue': str(job.service.id), 'DataType': 'String'},
+                                          'template': {'StringValue': str(job.template.id), 'DataType': 'String'},
+                                          'bucket_name': {'StringValue': job.bucket_name, 'DataType': 'String'},
+                                          'file_name': {'StringValue': job.file_name, 'DataType': 'String'},
+                                          'original_file_name': {'StringValue': job.original_file_name,
+                                                                 'DataType': 'String'}})

app/models.py

@@ -189,3 +189,34 @@ class VerifyCode(db.Model):
 
     def check_code(self, cde):
         return check_hash(cde, self._code)
+
+
+NOTIFICATION_STATUS_TYPES = ['sent', 'failed']
+
+
+class Notification(db.Model):
+
+    __tablename__ = 'notifications'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    to = db.Column(db.String, nullable=False)
+    job_id = db.Column(UUID(as_uuid=True), db.ForeignKey('jobs.id'), index=True, unique=False)
+    job = db.relationship('Job', backref=db.backref('notifications', lazy='dynamic'))
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False)
+    service = db.relationship('Service')
+    template_id = db.Column(db.BigInteger, db.ForeignKey('templates.id'), index=True, unique=False)
+    template = db.relationship('Template')
+    created_at = db.Column(
+        db.DateTime,
+        index=False,
+        unique=False,
+        nullable=False,
+        default=datetime.datetime.now)
+    updated_at = db.Column(
+        db.DateTime,
+        index=False,
+        unique=False,
+        nullable=True,
+        onupdate=datetime.datetime.now)
+    status = db.Column(
+        db.Enum(*NOTIFICATION_STATUS_TYPES, name='notification_status_types'), nullable=False, default='sent')

app/notifications/rest.py

@@ -1,165 +1,71 @@
-import json
+import uuid
 
-import boto3
 from flask import (
     Blueprint,
     jsonify,
-    request,
-    current_app
+    request
 )
-from itsdangerous import URLSafeSerializer
-from app import notify_alpha_client
-from app import api_user
-from app.dao import (templates_dao, services_dao)
-import re
-from app import celery
 
-mobile_regex = re.compile("^\\+44[\\d]{10}$")
+from app import api_user
+from app.aws_sqs import add_notification_to_queue
+from app.dao import (templates_dao)
+from app.schemas import (
+    email_notification_schema, sms_template_notification_schema)
 
 notifications = Blueprint('notifications', __name__)
 
 
 @notifications.route('/<notification_id>', methods=['GET'])
 def get_notifications(notification_id):
-    return jsonify(notify_alpha_client.fetch_notification_by_id(notification_id)), 200
-
-
-@celery.task(name="make-sms", bind="True")
-def send_sms(self, to, template):
-    print('Executing task id {0.id}, args: {0.args!r} kwargs: {0.kwargs!r}'.format(self.request))
-    from time import sleep
-    sleep(0.5)
-    print('finished')
-    #notify_alpha_client.send_sms(mobile_number=to, message=template)
+    # TODO return notification id details
+    return jsonify({'id': notification_id}), 200
 
 
 @notifications.route('/sms', methods=['POST'])
 def create_sms_notification():
-    notification = request.get_json()['notification']
-    errors = {}
-    to, to_errors = validate_to(notification)
-    if to_errors['to']:
-        errors.update(to_errors)
+    resp_json = request.get_json()
 
-    # TODO: should create a different endpoint for the admin client to send verify codes.
-    if api_user['client'] == current_app.config.get('ADMIN_CLIENT_USER_NAME'):
-        content, content_errors = validate_content_for_admin_client(notification)
-        if content_errors['content']:
-            errors.update(content_errors)
-        if errors:
-            return jsonify(result="error", message=errors), 400
+    notification, errors = sms_template_notification_schema.load(resp_json)
+    if errors:
+        return jsonify(result="error", message=errors), 400
 
-        return jsonify(notify_alpha_client.send_sms(mobile_number=to, message=content)), 200
-
-    else:
-        to, restricted_errors = validate_to_for_service(to, api_user['client'])
-        if restricted_errors['restricted']:
-            errors.update(restricted_errors)
-
-        template, template_errors = validate_template(notification, api_user['client'])
-        if template_errors['template']:
-            errors.update(template_errors)
-        if errors:
-            return jsonify(result="error", message=errors), 400
-
-        # add notification to the queue
-        service = services_dao.get_model_services(api_user['client'], _raise=False)
-        #_add_notification_to_queue(template.id, service, 'sms', to)
-        send_sms.apply_async((to, template.content), queue=str(service.id))
-        return jsonify(success=True)  # notify_alpha_client.send_sms(mobile_number=to, message=template.content)), 200
+    add_notification_to_queue(api_user['client'], notification['template'], 'sms', notification)
+    # TODO data to be returned
+    return jsonify({}), 204
 
 
 @notifications.route('/email', methods=['POST'])
 def create_email_notification():
-    notification = request.get_json()['notification']
-    errors = {}
-    for k in ['to', 'from', 'subject', 'message']:
-        k_error = validate_required_and_something(notification, k)
-        if k_error:
-            errors.update(k_error)
+    resp_json = request.get_json()
+    notification, errors = email_notification_schema.load(resp_json)
+    if errors:
+        return jsonify(result="error", message=errors), 400
+    add_notification_to_queue(api_user['client'], "admin", 'email', notification)
+    # TODO data to be returned
+    return jsonify({}), 204
 
+
+@notifications.route('/sms/service/<service_id>', methods=['POST'])
+def create_sms_for_service(service_id):
+
+    resp_json = request.get_json()
+
+    notification, errors = sms_template_notification_schema.load(resp_json)
     if errors:
         return jsonify(result="error", message=errors), 400
 
-    return jsonify(notify_alpha_client.send_email(
-        notification['to'],
-        notification['message'],
-        notification['from'],
-        notification['subject']))
+    template_id = notification['template']
+    job_id = notification['job']
 
+    # TODO: job/job_id is in notification and can used to update job status
 
-def validate_to_for_service(mob, service_id):
-    errors = {"restricted": []}
-    service = services_dao.get_model_services(service_id=service_id)
-    if service.restricted:
-        valid = False
-        for usr in service.users:
-            if mob == usr.mobile_number:
-                valid = True
-                break
-        if not valid:
-            errors['restricted'].append('Invalid phone number for restricted service')
-    return mob, errors
+    # TODO: remove once beta is reading notifications from the queue
+    template = templates_dao.get_model_templates(template_id)
 
+    if template.service.id != uuid.UUID(service_id):
+        message = "Invalid template: id {} for service id: {}".format(template.id, service_id)
+        return jsonify(result="error", message=message), 400
 
-def validate_to(json_body):
-    errors = {"to": []}
-    mob = json_body.get('to', None)
-    if not mob:
-        errors['to'].append('Required data missing')
-    else:
-        if not mobile_regex.match(mob):
-            errors['to'].append('invalid phone number, must be of format +441234123123')
-    return mob, errors
-
-
-def validate_template(json_body, service_id):
-    errors = {"template": []}
-    template_id = json_body.get('template', None)
-    template = ''
-    if not template_id:
-        errors['template'].append('Required data missing')
-    else:
-        try:
-            template = templates_dao.get_model_templates(
-                template_id=json_body['template'],
-                service_id=service_id)
-        except:
-            errors['template'].append("Unable to load template.")
-    return template, errors
-
-
-def validate_content_for_admin_client(json_body):
-    errors = {"content": []}
-    content = json_body.get('template', None)
-    if not content:
-        errors['content'].append('Required content')
-
-    return content, errors
-
-
-def validate_required_and_something(json_body, field):
-    errors = []
-    if field not in json_body and json_body[field]:
-        errors.append('Required data for field.')
-    return {field: errors} if errors else None
-
-
-def _add_notification_to_queue(template_id, service, msg_type, to):
-    q = boto3.resource('sqs', region_name=current_app.config['AWS_REGION']).create_queue(
-        QueueName=str(service.id))
-    import uuid
-    message_id = str(uuid.uuid4())
-    notification = json.dumps({'message_id': message_id,
-                               'service_id': str(service.id),
-                               'to': to,
-                               'message_type': msg_type,
-                               'template_id': template_id})
-    serializer = URLSafeSerializer(current_app.config.get('SECRET_KEY'))
-    encrypted = serializer.dumps(notification, current_app.config.get('DANGEROUS_SALT'))
-    q.send_message(MessageBody=encrypted,
-                   MessageAttributes={'type': {'StringValue': msg_type, 'DataType': 'String'},
-                                      'message_id': {'StringValue': message_id, 'DataType': 'String'},
-                                      'service_id': {'StringValue': str(service.id), 'DataType': 'String'},
-                                      'template_id': {'StringValue': str(template_id), 'DataType': 'String'}})
-
+    add_notification_to_queue(service_id, template_id, 'sms', notification)
+    # TODO data to be returned
+    return jsonify({}), 204

app/schemas.py

@@ -1,7 +1,10 @@
+import re
 from flask_marshmallow.fields import fields
 from . import ma
 from . import models
-from marshmallow import post_load, ValidationError
+from marshmallow import (post_load, ValidationError, validates, validates_schema)
+
+mobile_regex = re.compile("^\\+44[\\d]{10}$")
 
 
 # TODO I think marshmallow provides a better integration and error handling.
@@ -59,12 +62,77 @@ class JobSchema(BaseSchema):
 
 
 class RequestVerifyCodeSchema(ma.Schema):
-    def verify_code_type(self):
-        if self not in models.VERIFY_CODE_TYPES:
+
+    code_type = fields.Str(required=True)
+    to = fields.Str(required=False)
+
+    @validates('code_type')
+    def validate_code_type(self, code):
+        if code not in models.VERIFY_CODE_TYPES:
             raise ValidationError('Invalid code type')
 
-    code_type = fields.Str(required=True, validate=verify_code_type)
-    to = fields.Str(required=False)
+
+# TODO main purpose to be added later
+# when processing templates, template will be
+# common for all notifications.
+class NotificationSchema(ma.Schema):
+    pass
+
+
+class SmsNotificationSchema(NotificationSchema):
+    to = fields.Str(required=True)
+
+    @validates('to')
+    def validate_to(self, value):
+        if not mobile_regex.match(value):
+            raise ValidationError('Invalid phone number, must be of format +441234123123')
+
+
+class SmsTemplateNotificationSchema(SmsNotificationSchema):
+    template = fields.Int(required=True)
+    job = fields.String()
+
+    @validates('template')
+    def validate_template(self, value):
+        if not models.Template.query.filter_by(id=value).first():
+            # TODO is this message consistent with what marshmallow
+            # would normally produce.
+            raise ValidationError('Template not found')
+
+    @validates_schema
+    def validate_schema(self, data):
+        """
+        Validate the to field is valid for this template
+        """
+        template_id = data.get('template', None)
+        template = models.Template.query.filter_by(id=template_id).first()
+        if template:
+            service = template.service
+            if service.restricted:
+                valid = False
+                for usr in service.users:
+                    if data['to'] == usr.mobile_number:
+                        valid = True
+                        break
+                if not valid:
+                    raise ValidationError('Invalid phone number for restricted service', 'restricted')
+
+
+class SmsAdminNotificationSchema(SmsNotificationSchema):
+    content = fields.Str(required=True)
+
+
+class EmailNotificationSchema(NotificationSchema):
+    to_address = fields.Str(load_from="to", dump_to='to', required=True)
+    from_address = fields.Str(load_from="from", dump_to='from', required=True)
+    subject = fields.Str(required=True)
+    body = fields.Str(load_from="message", dump_to='message', required=True)
+
+
+class NotificationStatusSchema(BaseSchema):
+
+    class Meta:
+        model = models.Notification
 
 
 user_schema = UserSchema()
@@ -83,3 +151,9 @@ job_schema = JobSchema()
 job_schema_load_json = JobSchema(load_json=True)
 jobs_schema = JobSchema(many=True)
 request_verify_code_schema = RequestVerifyCodeSchema()
+sms_admin_notification_schema = SmsAdminNotificationSchema()
+sms_template_notification_schema = SmsTemplateNotificationSchema()
+email_notification_schema = EmailNotificationSchema()
+notification_status_schema = NotificationStatusSchema()
+notifications_status_schema = NotificationStatusSchema(many=True)
+notification_status_schema_load_json = NotificationStatusSchema(load_json=True)

app/user/rest.py

@@ -1,8 +1,9 @@
 from datetime import datetime
-from flask import (jsonify, request, abort)
+from flask import (jsonify, request, abort, Blueprint)
 from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
 from app.dao.services_dao import get_model_services
+from app.aws_sqs import add_notification_to_queue
 from app.dao.users_dao import (
     get_model_users,
     save_model_user,
@@ -16,8 +17,7 @@ from app.dao.users_dao import (
 from app.schemas import (
     user_schema, users_schema, service_schema, services_schema,
     request_verify_code_schema, user_schema_load_json)
-from app import notify_alpha_client
-from flask import Blueprint
+from app import api_user
 
 
 user = Blueprint('user', __name__)
@@ -133,20 +133,18 @@ def send_user_code(user_id):
     from app.dao.users_dao import create_secret_code
     secret_code = create_secret_code()
     create_user_code(user, secret_code, verify_code.get('code_type'))
-    # TODO this will need to fixed up when we stop using
-    # notify_alpha_client
     if verify_code.get('code_type') == 'sms':
         mobile = user.mobile_number if verify_code.get('to', None) is None else verify_code.get('to')
-        notify_alpha_client.send_sms(
-            mobile_number=mobile,
-            message=secret_code)
+        notification = {'to': mobile, 'content': secret_code}
+        add_notification_to_queue(api_user['client'], 'admin', 'sms', notification)
     elif verify_code.get('code_type') == 'email':
         email = user.email_address if verify_code.get('to', None) is None else verify_code.get('to')
-        notify_alpha_client.send_email(
-            email,
-            secret_code,
-            'notify@digital.cabinet-office.gov.uk',
-            'Verification code')
+        notification = {
+            'to_address': email,
+            'from_address': 'notify@digital.cabinet-office.gov.uk',
+            'subject': 'Verification code',
+            'body': secret_code}
+        add_notification_to_queue(api_user['client'], 'admin', 'email', notification)
     else:
         abort(500)
     return jsonify({}), 204