Revert "Process SNS request triggered by a DVLA S3 update"

app/aws/s3.py

@@ -4,24 +4,20 @@ from flask import current_app
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
-def get_s3_object(bucket_name, file_location):
+def get_s3_job_object(bucket_name, file_location):
     s3 = resource('s3')
-    s3_object = s3.Object(bucket_name, file_location)
-    return s3_object.get()['Body'].read()
+    return s3.Object(bucket_name, file_location)
 
 
 def get_job_from_s3(service_id, job_id):
-    job = _job_from_s3(service_id, job_id)
-    return job
+    bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME']
+    file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id)
+    obj = get_s3_job_object(bucket_name, file_location)
+    return obj.get()['Body'].read().decode('utf-8')
 
 
 def remove_job_from_s3(service_id, job_id):
-    job = _job_from_s3(service_id, job_id)
-    return job.delete()
-
-
-def _job_from_s3():
     bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME']
     file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id)
-    obj = get_s3_object(bucket_name, file_location).decode('utf-8')
-    return obj
+    obj = get_s3_job_object(bucket_name, file_location)
+    return obj.delete()

app/celery/tasks.py

@@ -1,7 +1,6 @@
 import random
 
 from datetime import (datetime)
-from collections import namedtuple
 
 from flask import current_app
 from notifications_utils.recipients import (
@@ -24,10 +23,7 @@ from app.dao.jobs_dao import (
     all_notifications_are_created_for_job,
     dao_get_all_notifications_for_job,
     dao_update_job_status)
-from app.dao.notifications_dao import (
-    get_notification_by_id,
-    dao_update_notifications_sent_to_dvla
-)
+from app.dao.notifications_dao import get_notification_by_id, dao_update_notifications_sent_to_dvla
 from app.dao.provider_details_dao import get_current_provider
 from app.dao.services_dao import dao_fetch_service_by_id, fetch_todays_total_message_count
 from app.dao.templates_dao import dao_get_template_by_id
@@ -358,27 +354,3 @@ def get_template_class(template_type):
         # since we don't need rendering capabilities (we only need to extract placeholders) both email and letter can
         # use the same base template
         return WithSubjectTemplate
-
-
-@notify_celery.task(bind=True, name='update-letter-notifications-statuses')
-@statsd(namespace="tasks")
-def update_letter_notifications_statuses(self, filename):
-    bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
-    response_file = s3.get_s3_object(bucket_location, filename).decode('utf-8')
-
-    try:
-        NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold'])
-        notification_updates = [NotificationUpdate(*line.split('|')) for line in response_file.splitlines()]
-
-    except TypeError:
-        current_app.logger.exception('DVLA response file: {} has an invalid format'.format(filename))
-        raise
-
-    else:
-        if notification_updates:
-            for update in notification_updates:
-                current_app.logger.info('DVLA update: {}'.format(str(update)))
-                # TODO: Update notifications with desired status
-            return notification_updates
-        else:
-            current_app.logger.exception('DVLA response file contained no updates')

app/dao/notifications_dao.py

@@ -27,8 +27,7 @@ from app.models import (
     NOTIFICATION_PERMANENT_FAILURE,
     KEY_TYPE_NORMAL, KEY_TYPE_TEST,
     LETTER_TYPE,
-    NOTIFICATION_SENT
-)
+    NOTIFICATION_SENT)
 
 from app.dao.dao_utils import transactional
 from app.statsd_decorators import statsd

app/notifications/notifications_letter_callback.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from functools import wraps
 
 from flask import (
     Blueprint,
@@ -10,54 +9,31 @@ from flask import (
 )
 
 from app import statsd_client
-from app.celery.tasks import update_letter_notifications_statuses
 from app.clients.email.aws_ses import get_aws_responses
-from app.dao import notifications_dao
-from app.v2.errors import register_errors
-from app.notifications.process_client_response import validate_callback_data
-from app.notifications.utils import autoconfirm_subscription
-from app.schema_validation import validate
+from app.dao import (
+    notifications_dao
+)
 
+from app.notifications.process_client_response import validate_callback_data
 
 letter_callback_blueprint = Blueprint('notifications_letter_callback', __name__)
+
+from app.errors import (
+    register_errors,
+    InvalidRequest
+)
+
 register_errors(letter_callback_blueprint)
 
 
-dvla_sns_callback_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
-    "description": "sns callback received on s3 update",
-    "type": "object",
-    "title": "dvla internal sns callback",
-    "properties": {
-        "Type": {"enum": ["Notification", "SubscriptionConfirmation"]},
-        "MessageId": {"type": "string"},
-        "Message": {"type": ["string", "object"]}
-    },
-    "required": ["Type", "MessageId", "Message"]
-}
-
-
-def validate_schema(schema):
-    def decorator(f):
-        @wraps(f)
-        def wrapper(*args, **kw):
-            validate(request.json, schema)
-            return f(*args, **kw)
-        return wrapper
-    return decorator
-
-
 @letter_callback_blueprint.route('/notifications/letter/dvla', methods=['POST'])
-@validate_schema(dvla_sns_callback_schema)
 def process_letter_response():
-    req_json = request.json
-    if not autoconfirm_subscription(req_json):
-        # The callback should have one record for an S3 Put Event.
-        filename = req_json['Message']['Records'][0]['s3']['object']['key']
-        current_app.logger.info('Received file from DVLA: {}'.format(filename))
-        current_app.logger.info('DVLA callback: Calling task to update letter notifications')
-        update_letter_notifications_statuses.apply_async([filename], queue='notify')
-
-    return jsonify(
-        result="success", message="DVLA callback succeeded"
-    ), 200
+    try:
+        dvla_request = json.loads(request.data)
+        current_app.logger.info(dvla_request)
+        return jsonify(
+            result="success", message="DVLA callback succeeded"
+        ), 200
+    except ValueError:
+        error = "DVLA callback failed: invalid json"
+        raise InvalidRequest(error, status_code=400)

app/notifications/notifications_ses_callback.py

@@ -13,8 +13,9 @@ from app.clients.email.aws_ses import get_aws_responses
 from app.dao import (
     notifications_dao
 )
+
 from app.notifications.process_client_response import validate_callback_data
-from app.notifications.utils import autoconfirm_subscription
+from app.notifications.utils import confirm_subscription
 
 ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
 
@@ -31,12 +32,14 @@ def process_ses_response():
     try:
         ses_request = json.loads(request.data)
 
-        subscribed_topic = autoconfirm_subscription(ses_request)
-        if subscribed_topic:
-            current_app.logger.info("Automatically subscribed to topic: {}".format(subscribed_topic))
-            return jsonify(
-                result="success", message="SES callback succeeded"
-            ), 200
+        if ses_request.get('Type') == 'SubscriptionConfirmation':
+            current_app.logger.info("SNS subscription confirmation url: {}".format(ses_request['SubscribeURL']))
+            subscribed_topic = confirm_subscription(ses_request)
+            if subscribed_topic:
+                current_app.logger.info("Automatically subscribed to topic: {}".format(subscribed_topic))
+                return jsonify(
+                    result="success", message="SES callback succeeded"
+                ), 200
 
         errors = validate_callback_data(data=ses_request, fields=['Message'], client_name=client_name)
         if errors:

app/notifications/utils.py

@@ -16,10 +16,3 @@ def confirm_subscription(confirmation_request):
         raise e
 
     return confirmation_request['TopicArn']
-
-
-def autoconfirm_subscription(req_json):
-    if req_json.get('Type') == 'SubscriptionConfirmation':
-        current_app.logger.info("SNS subscription confirmation url: {}".format(req_json['SubscribeURL']))
-        subscribed_topic = confirm_subscription(req_json)
-        return subscribed_topic