When we sent the service the status callback for a notification, we have all the information we need.

Which means we can remove the need to request the data from the database.
In order for the PR to be backwards compatible I have added an optional parameter "encrypted_status_update".
If this is not None then the new code is called.

The next PR will send the encrypted data to this task.

A final PR will remove the code that uses the database to get the notification and service callback api.

app/celery/service_callback_tasks.py

@@ -6,6 +6,7 @@ from app import (
     db,
     DATETIME_FORMAT,
     notify_celery,
+    encryption
 )
 from app.dao.notifications_dao import (
     get_notification_by_id,
@@ -23,10 +24,61 @@ from app.config import QueueNames
 
 @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
 @statsd(namespace="tasks")
-def send_delivery_status_to_service(self, notification_id):
+def send_delivery_status_to_service(self, notification_id,
+                                    encrypted_status_update=None
+                                    ):
+    if not encrypted_status_update:
+        process_update_with_notification_id(self, notification_id=notification_id)
+    else:
+        try:
+            status_update = encryption.decrypt(encrypted_status_update)
+
+            data = {
+                "id": str(notification_id),
+                "reference": status_update['notification_client_reference'],
+                "to": status_update['notification_to'],
+                "status": status_update['notification_status'],
+                "created_at": status_update['notification_created_at'],
+                "completed_at": status_update['notification_updated_at'],
+                "sent_at": status_update['notification_sent_at'],
+                "notification_type": status_update['notification_type']
+            }
+
+            response = request(
+                method="POST",
+                url=status_update['service_callback_api_url'],
+                data=json.dumps(data),
+                headers={
+                    'Content-Type': 'application/json',
+                    'Authorization': 'Bearer {}'.format(status_update['service_callback_api_bearer_token'])
+                },
+                timeout=60
+            )
+            current_app.logger.info('send_delivery_status_to_service sending {} to {}, response {}'.format(
+                notification_id,
+                status_update['service_callback_api_url'],
+                response.status_code
+            ))
+            response.raise_for_status()
+        except RequestException as e:
+            current_app.logger.warning(
+                "send_delivery_status_to_service request failed for service_id: {} and url: {}. exc: {}".format(
+                    notification_id,
+                    status_update['service_callback_api_url'],
+                    e
+                )
+            )
+            if not isinstance(e, HTTPError) or e.response.status_code >= 500:
+                try:
+                    self.retry(queue=QueueNames.RETRY)
+                except self.MaxRetriesExceededError:
+                    current_app.logger.exception(
+                        'Retry: send_delivery_status_to_service has retried the max num of times')
+
+
+def process_update_with_notification_id(self, notification_id):
     retry = False
     try:
-        # TODO: do we need to do rate limit this?
         notification = get_notification_by_id(notification_id)
         service_callback_api = get_service_callback_api_for_service(service_id=notification.service_id)
         if not service_callback_api:
@@ -41,9 +93,9 @@ def send_delivery_status_to_service(self, notification_id):
             "reference": str(notification.client_reference),
             "to": notification.to,
             "status": notification.status,
-            "created_at": notification.created_at.strftime(DATETIME_FORMAT),     # the time service sent the request
+            "created_at": notification.created_at.strftime(DATETIME_FORMAT),
-            "completed_at": notification.updated_at.strftime(DATETIME_FORMAT),   # the last time the status was updated
+            "completed_at": notification.updated_at.strftime(DATETIME_FORMAT),
-            "sent_at": notification.sent_at.strftime(DATETIME_FORMAT),           # the time the email was sent
+            "sent_at": notification.sent_at.strftime(DATETIME_FORMAT),
             "notification_type": notification.notification_type
         }
 

tests/app/celery/test_service_callback_tasks.py

@@ -7,16 +7,13 @@ import pytest
 import requests_mock
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import (DATETIME_FORMAT)
+from app import (DATETIME_FORMAT, encryption)
 
-from tests.app.conftest import (
-    sample_service as create_sample_service,
-    sample_template as create_sample_template,
-)
 from tests.app.db import (
     create_notification,
-    create_user,
+    create_service_callback_api,
-    create_service_callback_api
+    create_service,
+    create_template
 )
 from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.config import QueueNames
@@ -24,18 +21,127 @@ from app.config import QueueNames
 
 @pytest.mark.parametrize("notification_type",
                          ["email", "letter", "sms"])
-def test_send_delivery_status_to_service_post_https_request_to_service(notify_db,
+def test_send_delivery_status_to_service_post_https_request_to_service_with_encrypted_data(
-                                                                       notify_db_session,
+        notify_db_session, notification_type):
-                                                                       notification_type):
-    user = create_user()
-    service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True)
 
+    callback_api, template = _set_up_test_data(notification_type)
+    datestr = datetime(2017, 6, 20)
+
+    notification = create_notification(template=template,
+                                       created_at=datestr,
+                                       updated_at=datestr,
+                                       sent_at=datestr,
+                                       status='sent'
+                                       )
+    encrypted_status_update = _set_up_encrypted_data(callback_api, notification)
+    with requests_mock.Mocker() as request_mock:
+        request_mock.post(callback_api.url,
+                          json={},
+                          status_code=200)
+        send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_status_update)
+
+    mock_data = {
+        "id": str(notification.id),
+        "reference": notification.client_reference,
+        "to": notification.to,
+        "status": notification.status,
+        "created_at": datestr.strftime(DATETIME_FORMAT),
+        "completed_at": datestr.strftime(DATETIME_FORMAT),
+        "sent_at": datestr.strftime(DATETIME_FORMAT),
+        "notification_type": notification_type
+    }
+
+    assert request_mock.call_count == 1
+    assert request_mock.request_history[0].url == callback_api.url
+    assert request_mock.request_history[0].method == 'POST'
+    assert request_mock.request_history[0].text == json.dumps(mock_data)
+    assert request_mock.request_history[0].headers["Content-type"] == "application/json"
+    assert request_mock.request_history[0].headers["Authorization"] == "Bearer {}".format(callback_api.bearer_token)
+
+
+@pytest.mark.parametrize("notification_type",
+                         ["email", "letter", "sms"])
+def test_send_delivery_status_to_service_retries_if_request_returns_500_with_encrypted_data(
+        notify_db_session, mocker, notification_type
+):
+    callback_api, template = _set_up_test_data(notification_type)
+    datestr = datetime(2017, 6, 20)
+    notification = create_notification(template=template,
+                                       created_at=datestr,
+                                       updated_at=datestr,
+                                       sent_at=datestr,
+                                       status='sent'
+                                       )
+    encrypted_data = _set_up_encrypted_data(callback_api, notification)
+    mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
+    with requests_mock.Mocker() as request_mock:
+        request_mock.post(callback_api.url,
+                          json={},
+                          status_code=500)
+        send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_data)
+
+    assert mocked.call_count == 1
+    assert mocked.call_args[1]['queue'] == 'retry-tasks'
+
+
+@pytest.mark.parametrize("notification_type",
+                         ["email", "letter", "sms"])
+def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404_with_encrypted_data(
+        notify_db_session,
+        mocker,
+        notification_type
+):
+    callback_api, template = _set_up_test_data(notification_type)
+    datestr = datetime(2017, 6, 20)
+    notification = create_notification(template=template,
+                                       created_at=datestr,
+                                       updated_at=datestr,
+                                       sent_at=datestr,
+                                       status='sent'
+                                       )
+    encrypted_data = _set_up_encrypted_data(callback_api, notification)
+    mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
+    with requests_mock.Mocker() as request_mock:
+        request_mock.post(callback_api.url,
+                          json={},
+                          status_code=404)
+        send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_data)
+
+    assert mocked.call_count == 0
+
+
+def _set_up_test_data(notification_type):
+    service = create_service(restricted=True)
+    template = create_template(service=service, template_type=notification_type, subject='Hello')
     callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/",
                                                bearer_token="something_unique")
-    template = create_sample_template(
+    return callback_api, template
-        notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
-    )
 
+
+def _set_up_encrypted_data(callback_api, notification):
+    data = {
+        "notification_id": str(notification.id),
+        "notification_client_reference": notification.client_reference,
+        "notification_to": notification.to,
+        "notification_status": notification.status,
+        "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
+        "notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT),
+        "notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT),
+        "notification_type": notification.notification_type,
+        "service_callback_api_url": callback_api.url,
+        "service_callback_api_bearer_token": callback_api.bearer_token,
+    }
+    encrypted_status_update = encryption.encrypt(data)
+    return encrypted_status_update
+
+
+# We are updating the task to take everything it needs so that there are no db calls.
+# The following tests will be deleted once that is complete.
+@pytest.mark.parametrize("notification_type",
+                         ["email", "letter", "sms"])
+def test_send_delivery_status_to_service_post_https_request_to_service(
+        notify_db_session, notification_type):
+    callback_api, template = _set_up_test_data(notification_type)
     datestr = datetime(2017, 6, 20)
 
     notification = create_notification(template=template,
@@ -73,12 +179,9 @@ def test_send_delivery_status_to_service_post_https_request_to_service(notify_db
 @pytest.mark.parametrize("notification_type",
                          ["email", "letter", "sms"])
 def test_send_delivery_status_to_service_does_not_sent_request_when_service_callback_api_does_not_exist(
-        notify_db, notify_db_session, mocker, notification_type):
+        notify_db_session, mocker, notification_type):
-    service = create_sample_service(notify_db, notify_db_session, restricted=True)
+    service = create_service(restricted=True)
-
+    template = create_template(service=service, template_type=notification_type, subject='Hello')
-    template = create_sample_template(
-        notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
-    )
     datestr = datetime(2017, 6, 20)
 
     notification = create_notification(template=template,
@@ -95,18 +198,10 @@ def test_send_delivery_status_to_service_does_not_sent_request_when_service_call
 
 @pytest.mark.parametrize("notification_type",
                          ["email", "letter", "sms"])
-def test_send_delivery_status_to_service_retries_if_request_returns_500(notify_db,
+def test_send_delivery_status_to_service_retries_if_request_returns_500(notify_db_session,
-                                                                        notify_db_session,
                                                                         mocker,
                                                                         notification_type):
-    user = create_user()
+    callback_api, template = _set_up_test_data(notification_type)
-    service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True)
-
-    template = create_sample_template(
-        notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
-    )
-    callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/",
-                                               bearer_token="something_unique")
     datestr = datetime(2017, 6, 20)
     notification = create_notification(template=template,
                                        created_at=datestr,
@@ -127,18 +222,11 @@ def test_send_delivery_status_to_service_retries_if_request_returns_500(notify_d
 
 @pytest.mark.parametrize("notification_type",
                          ["email", "letter", "sms"])
-def test_send_delivery_status_to_service_retries_if_request_throws_unknown(notify_db,
+def test_send_delivery_status_to_service_retries_if_request_throws_unknown(notify_db_session,
-                                                                           notify_db_session,
                                                                            mocker,
                                                                            notification_type):
-    user = create_user()
-    service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True)
 
-    template = create_sample_template(
+    callback_api, template = _set_up_test_data(notification_type)
-        notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
-    )
-    create_service_callback_api(service=service, url="https://some.service.gov.uk/",
-                                bearer_token="something_unique")
     datestr = datetime(2017, 6, 20)
     notification = create_notification(template=template,
                                        created_at=datestr,
@@ -158,18 +246,12 @@ def test_send_delivery_status_to_service_retries_if_request_throws_unknown(notif
 
 @pytest.mark.parametrize("notification_type",
                          ["email", "letter", "sms"])
-def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404(notify_db,
+def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404(
-                                                                                 notify_db_session,
+        notify_db_session,
-                                                                                 mocker,
+        mocker,
-                                                                                 notification_type):
+        notification_type
-    user = create_user()
+):
-    service = create_sample_service(notify_db, notify_db_session, user=user, restricted=True)
+    callback_api, template = _set_up_test_data(notification_type)
-
-    template = create_sample_template(
-        notify_db, notify_db_session, service=service, template_type=notification_type, subject_line='Hello'
-    )
-    callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/",
-                                               bearer_token="something_unique")
     datestr = datetime(2017, 6, 20)
     notification = create_notification(template=template,
                                        created_at=datestr,