Merge branch 'master' into schedule-api-notification

app/aws/s3.py

@@ -4,7 +4,12 @@ from flask import current_app
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
-def get_s3_job_object(bucket_name, file_location):
+def get_s3_file(bucket_name, file_location):
+    s3_file = get_s3_object(bucket_name, file_location)
+    return s3_file.get()['Body'].read().decode('utf-8')
+
+
+def get_s3_object(bucket_name, file_location):
     s3 = resource('s3')
     return s3.Object(bucket_name, file_location)
 
@@ -12,12 +17,12 @@ def get_s3_job_object(bucket_name, file_location):
 def get_job_from_s3(service_id, job_id):
     bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME']
     file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id)
-    obj = get_s3_job_object(bucket_name, file_location)
+    obj = get_s3_object(bucket_name, file_location)
     return obj.get()['Body'].read().decode('utf-8')
 
 
 def remove_job_from_s3(service_id, job_id):
     bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME']
     file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id)
-    obj = get_s3_job_object(bucket_name, file_location)
+    obj = get_s3_object(bucket_name, file_location)
     return obj.delete()

app/celery/tasks.py

@@ -1,6 +1,7 @@
 import random
 
 from datetime import (datetime)
+from collections import namedtuple
 
 from flask import current_app
 from notifications_utils.recipients import (
@@ -354,3 +355,27 @@ def get_template_class(template_type):
         # since we don't need rendering capabilities (we only need to extract placeholders) both email and letter can
         # use the same base template
         return WithSubjectTemplate
+
+
+@notify_celery.task(bind=True, name='update-letter-notifications-statuses')
+@statsd(namespace="tasks")
+def update_letter_notifications_statuses(self, filename):
+    bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
+    response_file = s3.get_s3_file(bucket_location, filename)
+
+    try:
+        NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold'])
+        notification_updates = [NotificationUpdate(*line.split('|')) for line in response_file.splitlines()]
+
+    except TypeError:
+        current_app.logger.exception('DVLA response file: {} has an invalid format'.format(filename))
+        raise
+
+    else:
+        if notification_updates:
+            for update in notification_updates:
+                current_app.logger.info('DVLA update: {}'.format(str(update)))
+                # TODO: Update notifications with desired status
+            return notification_updates
+        else:
+            current_app.logger.exception('DVLA response file contained no updates')

app/config.py

@@ -89,6 +89,7 @@ class Config(object):
     PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201'
     ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb'
     CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
+    SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
 
     BROKER_URL = 'sqs://'
     BROKER_TRANSPORT_OPTIONS = {

app/dao/services_dao.py

@@ -384,3 +384,12 @@ def dao_suspend_service(service_id):
 def dao_resume_service(service_id):
     service = Service.query.get(service_id)
     service.active = True
+
+
+def dao_fetch_active_users_for_service(service_id):
+    query = User.query.filter(
+        User.user_to_service.any(id=service_id),
+        User.state == 'active'
+    )
+
+    return query.all()

app/models.py

@@ -31,6 +31,18 @@ from app import (
 from app.history_meta import Versioned
 from app.utils import get_utc_time_in_bst
 
+SMS_TYPE = 'sms'
+EMAIL_TYPE = 'email'
+LETTER_TYPE = 'letter'
+
+TEMPLATE_TYPES = [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]
+
+template_types = db.Enum(*TEMPLATE_TYPES, name='template_type')
+
+NORMAL = 'normal'
+PRIORITY = 'priority'
+TEMPLATE_PROCESS_TYPE = [NORMAL, PRIORITY]
+
 
 def filter_null_value_fields(obj):
     return dict(
@@ -183,6 +195,30 @@ class Service(db.Model, Versioned):
     )
 
 
+INTERNATIONAL_SMS_TYPE = 'international_sms'
+INCOMING_SMS_TYPE = 'incoming_sms'
+
+SERVICE_PERMISSION_TYPES = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INCOMING_SMS_TYPE]
+
+
+class ServicePermissionTypes(db.Model):
+    __tablename__ = 'service_permission_types'
+
+    name = db.Column(db.String(255), primary_key=True)
+
+
+class ServicePermission(db.Model):
+    __tablename__ = "service_permissions"
+
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'),
+                           primary_key=True, index=True, nullable=False)
+    service = db.relationship('Service')
+    permission = db.Column(db.String(255), db.ForeignKey('service_permission_types.name'),
+                           index=True, primary_key=True, nullable=False)
+    created_at = db.Column(db.DateTime, default=datetime.datetime.utcnow, nullable=False)
+    updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
+
+
 MOBILE_TYPE = 'mobile'
 EMAIL_TYPE = 'email'
 
@@ -293,19 +329,6 @@ class TemplateProcessTypes(db.Model):
     name = db.Column(db.String(255), primary_key=True)
 
 
-SMS_TYPE = 'sms'
-EMAIL_TYPE = 'email'
-LETTER_TYPE = 'letter'
-
-TEMPLATE_TYPES = [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]
-
-template_types = db.Enum(*TEMPLATE_TYPES, name='template_type')
-
-NORMAL = 'normal'
-PRIORITY = 'priority'
-TEMPLATE_PROCESS_TYPE = [NORMAL, PRIORITY]
-
-
 class Template(db.Model):
     __tablename__ = 'templates'
 

app/notifications/notifications_letter_callback.py

@@ -1,39 +1,57 @@
-from datetime import datetime
+from functools import wraps
 
 from flask import (
     Blueprint,
     jsonify,
     request,
-    current_app,
-    json
+    current_app
 )
 
-from app import statsd_client
-from app.clients.email.aws_ses import get_aws_responses
-from app.dao import (
-    notifications_dao
-)
+from app.celery.tasks import update_letter_notifications_statuses
+from app.v2.errors import register_errors
+from app.notifications.utils import autoconfirm_subscription
+from app.schema_validation import validate
 
-from app.notifications.process_client_response import validate_callback_data
 
 letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__)
-
-from app.errors import (
-    register_errors,
-    InvalidRequest
-)
-
 register_errors(letter_callback_blueprint)
 
 
+dvla_sns_callback_schema = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "sns callback received on s3 update",
+    "type": "object",
+    "title": "dvla internal sns callback",
+    "properties": {
+        "Type": {"enum": ["Notification", "SubscriptionConfirmation"]},
+        "MessageId": {"type": "string"},
+        "Message": {"type": ["string", "object"]}
+    },
+    "required": ["Type", "MessageId", "Message"]
+}
+
+
+def validate_schema(schema):
+    def decorator(f):
+        @wraps(f)
+        def wrapper(*args, **kw):
+            validate(request.json, schema)
+            return f(*args, **kw)
+        return wrapper
+    return decorator
+
+
 @letter_callback_blueprint.route('/notifications/letter/dvla', methods=['POST'])
+@validate_schema(dvla_sns_callback_schema)
 def process_letter_response():
-    try:
-        dvla_request = json.loads(request.data)
-        current_app.logger.info(dvla_request)
-        return jsonify(
-            result="success", message="DVLA callback succeeded"
-        ), 200
-    except ValueError:
-        error = "DVLA callback failed: invalid json"
-        raise InvalidRequest(error, status_code=400)
+    req_json = request.json
+    if not autoconfirm_subscription(req_json):
+        # The callback should have one record for an S3 Put Event.
+        filename = req_json['Message']['Records'][0]['s3']['object']['key']
+        current_app.logger.info('Received file from DVLA: {}'.format(filename))
+        current_app.logger.info('DVLA callback: Calling task to update letter notifications')
+        update_letter_notifications_statuses.apply_async([filename], queue='notify')
+
+    return jsonify(
+        result="success", message="DVLA callback succeeded"
+    ), 200

app/notifications/notifications_ses_callback.py

@@ -13,9 +13,8 @@ from app.clients.email.aws_ses import get_aws_responses
 from app.dao import (
     notifications_dao
 )
-
 from app.notifications.process_client_response import validate_callback_data
-from app.notifications.utils import confirm_subscription
+from app.notifications.utils import autoconfirm_subscription
 
 ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
 
@@ -32,14 +31,12 @@ def process_ses_response():
     try:
         ses_request = json.loads(request.data)
 
-        if ses_request.get('Type') == 'SubscriptionConfirmation':
-            current_app.logger.info("SNS subscription confirmation url: {}".format(ses_request['SubscribeURL']))
-            subscribed_topic = confirm_subscription(ses_request)
-            if subscribed_topic:
-                current_app.logger.info("Automatically subscribed to topic: {}".format(subscribed_topic))
-                return jsonify(
-                    result="success", message="SES callback succeeded"
-                ), 200
+        subscribed_topic = autoconfirm_subscription(ses_request)
+        if subscribed_topic:
+            current_app.logger.info("Automatically subscribed to topic: {}".format(subscribed_topic))
+            return jsonify(
+                result="success", message="SES callback succeeded"
+            ), 200
 
         errors = validate_callback_data(data=ses_request, fields=['Message'], client_name=client_name)
         if errors:

app/notifications/utils.py

@@ -16,3 +16,10 @@ def confirm_subscription(confirmation_request):
         raise e
 
     return confirmation_request['TopicArn']
+
+
+def autoconfirm_subscription(req_json):
+    if req_json.get('Type') == 'SubscriptionConfirmation':
+        current_app.logger.info("SNS subscription confirmation url: {}".format(req_json['SubscribeURL']))
+        subscribed_topic = confirm_subscription(req_json)
+        return subscribed_topic

app/service/rest.py

@@ -46,6 +46,7 @@ from app.errors import (
     InvalidRequest, register_errors)
 from app.service import statistics
 from app.service.utils import get_whitelist_objects
+from app.service.sender import send_notification_to_service_users
 from app.schemas import (
     service_schema,
     api_key_schema,
@@ -117,10 +118,25 @@ def create_service():
 @service_blueprint.route('/<uuid:service_id>', methods=['POST'])
 def update_service(service_id):
     fetched_service = dao_fetch_service_by_id(service_id)
+    # Capture the status change here as Marshmallow changes this later
+    service_going_live = fetched_service.restricted and not request.get_json().get('restricted', True)
+
     current_data = dict(service_schema.dump(fetched_service).data.items())
     current_data.update(request.get_json())
     update_dict = service_schema.load(current_data).data
     dao_update_service(update_dict)
+
+    if service_going_live:
+        send_notification_to_service_users(
+            service_id=service_id,
+            template_id=current_app.config['SERVICE_NOW_LIVE_TEMPLATE_ID'],
+            personalisation={
+                'service_name': current_data['name'],
+                'message_limit': current_data['message_limit']
+            },
+            include_user_fields=['name']
+        )
+
     return jsonify(data=service_schema.dump(fetched_service).data), 200
 
 

app/service/sender.py

@@ -0,0 +1,33 @@
+from flask import current_app
+
+from app.dao.services_dao import dao_fetch_service_by_id, dao_fetch_active_users_for_service
+from app.dao.templates_dao import dao_get_template_by_id
+from app.models import EMAIL_TYPE, KEY_TYPE_NORMAL
+from app.notifications.process_notifications import persist_notification, send_notification_to_queue
+
+
+def send_notification_to_service_users(service_id, template_id, personalisation={}, include_user_fields=[]):
+    template = dao_get_template_by_id(template_id)
+    service = dao_fetch_service_by_id(service_id)
+    active_users = dao_fetch_active_users_for_service(service.id)
+    notify_service = dao_fetch_service_by_id(current_app.config['NOTIFY_SERVICE_ID'])
+
+    for user in active_users:
+        personalisation = _add_user_fields(user, personalisation, include_user_fields)
+        notification = persist_notification(
+            template_id=template.id,
+            template_version=template.version,
+            recipient=user.email_address if template.template_type == EMAIL_TYPE else user.mobile_number,
+            service=notify_service,
+            personalisation=personalisation,
+            notification_type=template.template_type,
+            api_key_id=None,
+            key_type=KEY_TYPE_NORMAL
+        )
+        send_notification_to_queue(notification, False, queue='notify')
+
+
+def _add_user_fields(user, personalisation, fields):
+    for field in fields:
+        personalisation[field] = getattr(user, field)
+    return personalisation