Add a task to process sms callback from our providers

This runs on the new `sms-callbacks` queue. The function
`process_sms_client_response` has been replaced with a task called
`process_sms_client_response`. This involved some reorganisation of the
existing code and tests.

app/celery/process_sms_client_response_tasks.py

@@ -1,20 +1,17 @@
 import uuid
-
 from datetime import datetime
+
 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
 
-from app import statsd_client
+from app import notify_celery, statsd_client
 from app.clients import ClientException
-from app.dao import notifications_dao
 from app.clients.sms.firetext import get_firetext_responses
 from app.clients.sms.mmg import get_mmg_responses
-from app.celery.service_callback_tasks import (
-    send_delivery_status_to_service,
-    create_delivery_status_callback_data,
-)
+from app.celery.service_callback_tasks import send_delivery_status_to_service, create_delivery_status_callback_data
 from app.config import QueueNames
-from app.dao.notifications_dao import dao_update_notification
+from app.dao import notifications_dao
 from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.dao.templates_dao import dao_get_template_by_id
 from app.models import NOTIFICATION_PENDING
@@ -25,39 +22,23 @@ sms_response_mapper = {
 }
 
 
-def validate_callback_data(data, fields, client_name):
-    errors = []
-    for f in fields:
-        if not str(data.get(f, '')):
-            error = "{} callback failed: {} missing".format(client_name, f)
-            errors.append(error)
-    return errors if len(errors) > 0 else None
-
-
-def process_sms_client_response(status, provider_reference, client_name):
-    success = None
-    errors = None
+@notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
+def process_sms_client_response(self, status, provider_reference, client_name):
     # validate reference
-    if provider_reference == 'send-sms-code':
-        success = "{} callback succeeded: send-sms-code".format(client_name)
-        return success, errors
-
     try:
         uuid.UUID(provider_reference, version=4)
-    except ValueError:
-        errors = "{} callback with invalid reference {}".format(client_name, provider_reference)
-        return success, errors
+    except ValueError as e:
+        current_app.logger.exception(f'{client_name} callback with invalid reference {provider_reference}')
+        raise e
 
-    try:
-        response_parser = sms_response_mapper[client_name]
-    except KeyError:
-        return success, 'unknown sms client: {}'.format(client_name)
+    response_parser = sms_response_mapper[client_name]
 
-    # validate  status
+    # validate status
     try:
         notification_status = response_parser(status)
-        current_app.logger.info('{} callback return status of {} for reference: {}'.format(
-            client_name, status, provider_reference)
+        current_app.logger.info(
+            f'{client_name} callback returned status of {status} for reference: {provider_reference}'
         )
     except KeyError:
         _process_for_status(
@@ -65,14 +46,13 @@ def process_sms_client_response(status, provider_reference, client_name):
             client_name=client_name,
             provider_reference=provider_reference
         )
-        raise ClientException("{} callback failed: status {} not found.".format(client_name, status))
+        raise ClientException(f'{client_name} callback failed: status {status} not found.')
 
-    success = _process_for_status(
+    _process_for_status(
         notification_status=notification_status,
         client_name=client_name,
         provider_reference=provider_reference
     )
-    return success, errors
 
 
 def _process_for_status(notification_status, client_name, provider_reference):
@@ -114,6 +94,3 @@ def _process_for_status(notification_status, client_name, provider_reference):
             encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
             send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
                                                         queue=QueueNames.CALLBACKS)
-
-    success = "{} callback succeeded. reference {} updated".format(client_name, provider_reference)
-    return success

app/notifications/notifications_sms_callback.py

@@ -3,8 +3,9 @@ from flask import current_app
 from flask import json
 from flask import request, jsonify
 
+from app.celery.process_sms_client_response_tasks import process_sms_client_response
+from app.config import QueueNames
 from app.errors import InvalidRequest, register_errors
-from app.notifications.process_client_response import validate_callback_data, process_sms_client_response
 
 sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms")
 register_errors(sms_callback_blueprint)
@@ -20,19 +21,21 @@ def process_mmg_response():
     if errors:
         raise InvalidRequest(errors, status_code=400)
 
-    success, errors = process_sms_client_response(status=str(data.get('status')),
-                                                  provider_reference=data.get('CID'),
-                                                  client_name=client_name)
+    status = str(data.get('status'))
+    provider_reference = data.get('CID')
+
+    process_sms_client_response.apply_async(
+        [status, provider_reference, client_name],
+        queue=QueueNames.SMS_CALLBACKS,
+    )
 
     safe_to_log = data.copy()
     safe_to_log.pop("MSISDN")
     current_app.logger.debug(
-        "Full delivery response from {} for notification: {}\n{}".format(client_name, request.form.get('CID'),
-                                                                         safe_to_log))
-    if errors:
-        raise InvalidRequest(errors, status_code=400)
-    else:
-        return jsonify(result='success', message=success), 200
+        f"Full delivery response from {client_name} for notification: {provider_reference}\n{safe_to_log}"
+    )
+
+    return jsonify(result='success'), 200
 
 
 @sms_callback_blueprint.route('/firetext', methods=['POST'])
@@ -43,15 +46,28 @@ def process_firetext_response():
                                     client_name=client_name)
     if errors:
         raise InvalidRequest(errors, status_code=400)
+
+    status = request.form.get('status')
+    provider_reference = request.form.get('reference')
+
     safe_to_log = dict(request.form).copy()
     safe_to_log.pop('mobile')
     current_app.logger.debug(
-        "Full delivery response from {} for notification: {}\n{}".format(client_name, request.form.get('reference'),
-                                                                         safe_to_log))
-    success, errors = process_sms_client_response(status=request.form.get('status'),
-                                                  provider_reference=request.form.get('reference'),
-                                                  client_name=client_name)
-    if errors:
-        raise InvalidRequest(errors, status_code=400)
-    else:
-        return jsonify(result='success', message=success), 200
+        f"Full delivery response from {client_name} for notification: {provider_reference}\n{safe_to_log}"
+    )
+
+    process_sms_client_response.apply_async(
+        [status, provider_reference, client_name],
+        queue=QueueNames.SMS_CALLBACKS,
+    )
+
+    return jsonify(result='success'), 200
+
+
+def validate_callback_data(data, fields, client_name):
+    errors = []
+    for f in fields:
+        if not str(data.get(f, '')):
+            error = "{} callback failed: {} missing".format(client_name, f)
+            errors.append(error)
+    return errors if len(errors) > 0 else None