Add a task to process returned letter lists

Adds an API endpoint `/letters/returned` that accepts a list of
notification references and creates a task to update their status.

Adds a new task that uses the list of references to update the status
of notifications to 'returned-letter'.

The update is currently done using a single query and logs the
number of changed records (including notification history records).
This could potentially be done within the `/letters/returned` endpoint,
but creating a job right away allows us to extend this more easily
in the future (e.g. logging missing notifications or adding callbacks).

The job is using the database tasks queue.

app/celery/tasks.py

@@ -64,6 +64,7 @@ from app.models import (
     NOTIFICATION_SENDING,
     NOTIFICATION_TEMPORARY_FAILURE,
     NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_RETURNED_LETTER,
     SMS_TYPE,
     DailySortedLetter,
 )
@@ -591,3 +592,18 @@ def process_incomplete_job(job_id):
             process_row(row, template, job, job.service)
 
     job_complete(job, resumed=True)
+
+
+@notify_celery.task(name='process-returned-letters-list')
+@statsd(namespace="tasks")
+def process_returned_letters_list(notification_references):
+    updated, updated_history = dao_update_notifications_by_reference(
+        notification_references,
+        {"status": NOTIFICATION_RETURNED_LETTER}
+    )
+
+    current_app.logger.info(
+        "Updated {} letter notifications ({} history notifications, from {} references) to returned-letter".format(
+            updated, updated_history, len(notification_references)
+        )
+    )

app/letters/letter_schemas.py

@@ -13,3 +13,22 @@ letter_job_ids = {
     },
     "required": ["job_ids"]
 }
+
+
+letter_references = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "list of letter notification references",
+    "type": "object",
+    "title": "references",
+    "properties": {
+        "references": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                "pattern": "^[0-9A-Z]{16}$"
+            },
+            "minItems": 1
+        },
+    },
+    "required": ["references"]
+}

app/letters/rest.py

@@ -2,11 +2,12 @@ from flask import Blueprint, jsonify
 from flask import request
 
 from app import notify_celery
+from app.celery.tasks import process_returned_letters_list
 from app.config import QueueNames, TaskNames
 from app.dao.jobs_dao import dao_get_all_letter_jobs
 from app.schemas import job_schema
 from app.v2.errors import register_errors
-from app.letters.letter_schemas import letter_job_ids
+from app.letters.letter_schemas import letter_job_ids, letter_references
 from app.schema_validation import validate
 
 letter_job = Blueprint("letter-job", __name__)
@@ -27,3 +28,12 @@ def get_letter_jobs():
     data = job_schema.dump(letter_jobs, many=True).data
 
     return jsonify(data=data), 200
+
+
+@letter_job.route('/letters/returned', methods=['POST'])
+def create_process_returned_letters_job():
+    references = validate(request.get_json(), letter_references)
+
+    process_returned_letters_list.apply_async([references['references']], queue=QueueNames.DATABASE)
+
+    return jsonify(references=references['references']), 200

app/models.py

@@ -1309,7 +1309,7 @@ class Notification(db.Model):
 
         if self.status in [NOTIFICATION_CREATED, NOTIFICATION_SENDING]:
             return NOTIFICATION_STATUS_LETTER_ACCEPTED
-        elif self.status == NOTIFICATION_DELIVERED:
+        elif self.status in [NOTIFICATION_DELIVERED, NOTIFICATION_RETURNED_LETTER]:
             return NOTIFICATION_STATUS_LETTER_RECEIVED
         else:
             # Currently can only be technical-failure

tests/app/celery/test_tasks.py

@@ -27,6 +27,7 @@ from app.celery.tasks import (
     get_template_class,
     s3,
     send_inbound_sms_to_service,
+    process_returned_letters_list,
 )
 from app.config import QueueNames
 from app.dao import jobs_dao, services_dao
@@ -1551,3 +1552,14 @@ def test_process_incomplete_jobs_sets_status_to_in_progress_and_resets_processin
     assert job2.processing_started == datetime.utcnow()
 
     assert mock_process_incomplete_job.mock_calls == [call(str(job1.id)), call(str(job2.id))]
+
+
+def test_process_returned_letters_list(mocker, sample_letter_template):
+    create_notification(sample_letter_template, reference='ref1')
+    create_notification(sample_letter_template, reference='ref2')
+
+    process_returned_letters_list(['ref1', 'ref2', 'unknown-ref'])
+
+    assert [
+        n.status for n in Notification.query.all()
+    ] == ['returned-letter', 'returned-letter']

tests/app/letters/test_returned_letters.py

@@ -0,0 +1,23 @@
+import pytest
+
+
+@pytest.mark.parametrize('status, references', [
+    (200, ["1234567890ABCDEF", "1234567890ABCDEG"]),
+    (400, ["1234567890ABCDEFG", "1234567890ABCDEG"]),
+    (400, ["1234567890ABCDE", "1234567890ABCDEG"]),
+    (400, ["1234567890ABCDE\u26d4", "1234567890ABCDEG"]),
+    (400, ["NOTIFY0001234567890ABCDEF", "1234567890ABCDEG"]),
+])
+def test_process_returned_letters(status, references, admin_request, mocker):
+    mock_celery = mocker.patch("app.letters.rest.process_returned_letters_list.apply_async")
+
+    response = admin_request.post(
+        'letter-job.create_process_returned_letters_job',
+        _data={"references": references},
+        _expected_status=status
+    )
+
+    if status != 200:
+        assert '{} does not match'.format(references[0]) in response['errors'][0]['message']
+    else:
+        mock_celery.assert_called_once_with([references], queue='database-tasks')