Merge pull request #954 from alphagov/imdad-feat-process-dvla-sns-callback

Add callback to process DVLA response file

app/aws/s3.py

@@ -4,7 +4,12 @@ from flask import current_app
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
-def get_s3_job_object(bucket_name, file_location):
+def get_s3_file(bucket_name, file_location):
+    s3_file = get_s3_object(bucket_name, file_location)
+    return s3_file.get()['Body'].read().decode('utf-8')
+
+
+def get_s3_object(bucket_name, file_location):
     s3 = resource('s3')
     return s3.Object(bucket_name, file_location)
 
@@ -12,12 +17,12 @@ def get_s3_job_object(bucket_name, file_location):
 def get_job_from_s3(service_id, job_id):
     bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME']
     file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id)
-    obj = get_s3_job_object(bucket_name, file_location)
+    obj = get_s3_object(bucket_name, file_location)
     return obj.get()['Body'].read().decode('utf-8')
 
 
 def remove_job_from_s3(service_id, job_id):
     bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME']
     file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id)
-    obj = get_s3_job_object(bucket_name, file_location)
+    obj = get_s3_object(bucket_name, file_location)
     return obj.delete()

app/celery/tasks.py

@@ -1,6 +1,7 @@
 import random
 
 from datetime import (datetime)
+from collections import namedtuple
 
 from flask import current_app
 from notifications_utils.recipients import (
@@ -354,3 +355,27 @@ def get_template_class(template_type):
         # since we don't need rendering capabilities (we only need to extract placeholders) both email and letter can
         # use the same base template
         return WithSubjectTemplate
+
+
+@notify_celery.task(bind=True, name='update-letter-notifications-statuses')
+@statsd(namespace="tasks")
+def update_letter_notifications_statuses(self, filename):
+    bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
+    response_file = s3.get_s3_file(bucket_location, filename)
+
+    try:
+        NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold'])
+        notification_updates = [NotificationUpdate(*line.split('|')) for line in response_file.splitlines()]
+
+    except TypeError:
+        current_app.logger.exception('DVLA response file: {} has an invalid format'.format(filename))
+        raise
+
+    else:
+        if notification_updates:
+            for update in notification_updates:
+                current_app.logger.info('DVLA update: {}'.format(str(update)))
+                # TODO: Update notifications with desired status
+            return notification_updates
+        else:
+            current_app.logger.exception('DVLA response file contained no updates')

app/notifications/notifications_letter_callback.py

@@ -1,39 +1,57 @@
-from datetime import datetime
+from functools import wraps
 
 from flask import (
     Blueprint,
     jsonify,
     request,
-    current_app,
-    json
+    current_app
 )
 
-from app import statsd_client
-from app.clients.email.aws_ses import get_aws_responses
-from app.dao import (
-    notifications_dao
-)
+from app.celery.tasks import update_letter_notifications_statuses
+from app.v2.errors import register_errors
+from app.notifications.utils import autoconfirm_subscription
+from app.schema_validation import validate
 
-from app.notifications.process_client_response import validate_callback_data
 
 letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__)
-
-from app.errors import (
-    register_errors,
-    InvalidRequest
-)
-
 register_errors(letter_callback_blueprint)
 
 
+dvla_sns_callback_schema = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "sns callback received on s3 update",
+    "type": "object",
+    "title": "dvla internal sns callback",
+    "properties": {
+        "Type": {"enum": ["Notification", "SubscriptionConfirmation"]},
+        "MessageId": {"type": "string"},
+        "Message": {"type": ["string", "object"]}
+    },
+    "required": ["Type", "MessageId", "Message"]
+}
+
+
+def validate_schema(schema):
+    def decorator(f):
+        @wraps(f)
+        def wrapper(*args, **kw):
+            validate(request.json, schema)
+            return f(*args, **kw)
+        return wrapper
+    return decorator
+
+
 @letter_callback_blueprint.route('/notifications/letter/dvla', methods=['POST'])
+@validate_schema(dvla_sns_callback_schema)
 def process_letter_response():
-    try:
-        dvla_request = json.loads(request.data)
-        current_app.logger.info(dvla_request)
-        return jsonify(
-            result="success", message="DVLA callback succeeded"
-        ), 200
-    except ValueError:
-        error = "DVLA callback failed: invalid json"
-        raise InvalidRequest(error, status_code=400)
+    req_json = request.json
+    if not autoconfirm_subscription(req_json):
+        # The callback should have one record for an S3 Put Event.
+        filename = req_json['Message']['Records'][0]['s3']['object']['key']
+        current_app.logger.info('Received file from DVLA: {}'.format(filename))
+        current_app.logger.info('DVLA callback: Calling task to update letter notifications')
+        update_letter_notifications_statuses.apply_async([filename], queue='notify')
+
+    return jsonify(
+        result="success", message="DVLA callback succeeded"
+    ), 200

app/notifications/notifications_ses_callback.py

@@ -13,9 +13,8 @@ from app.clients.email.aws_ses import get_aws_responses
 from app.dao import (
     notifications_dao
 )
-
 from app.notifications.process_client_response import validate_callback_data
-from app.notifications.utils import confirm_subscription
+from app.notifications.utils import autoconfirm_subscription
 
 ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
 
@@ -32,14 +31,12 @@ def process_ses_response():
     try:
         ses_request = json.loads(request.data)
 
-        if ses_request.get('Type') == 'SubscriptionConfirmation':
-            current_app.logger.info("SNS subscription confirmation url: {}".format(ses_request['SubscribeURL']))
-            subscribed_topic = confirm_subscription(ses_request)
-            if subscribed_topic:
-                current_app.logger.info("Automatically subscribed to topic: {}".format(subscribed_topic))
-                return jsonify(
-                    result="success", message="SES callback succeeded"
-                ), 200
+        subscribed_topic = autoconfirm_subscription(ses_request)
+        if subscribed_topic:
+            current_app.logger.info("Automatically subscribed to topic: {}".format(subscribed_topic))
+            return jsonify(
+                result="success", message="SES callback succeeded"
+            ), 200
 
         errors = validate_callback_data(data=ses_request, fields=['Message'], client_name=client_name)
         if errors:

app/notifications/utils.py

@@ -16,3 +16,10 @@ def confirm_subscription(confirmation_request):
         raise e
 
     return confirmation_request['TopicArn']
+
+
+def autoconfirm_subscription(req_json):
+    if req_json.get('Type') == 'SubscriptionConfirmation':
+        current_app.logger.info("SNS subscription confirmation url: {}".format(req_json['SubscribeURL']))
+        subscribed_topic = confirm_subscription(req_json)
+        return subscribed_topic