Merge pull request #1754 from alphagov/revert-service-callback-eventlets

Revert service callback eventlets

app/celery/service_callback_tasks.py

@@ -24,6 +24,8 @@ from app.config import QueueNames
 @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
 @statsd(namespace="tasks")
 def send_delivery_status_to_service(self, notification_id):
+    retry = False
+    try:
         # TODO: do we need to do rate limit this?
         notification = get_notification_by_id(notification_id)
         service_callback_api = get_service_callback_api_for_service(service_id=notification.service_id)
@@ -45,7 +47,6 @@ def send_delivery_status_to_service(self, notification_id):
             "notification_type": notification.notification_type
         }
 
-    try:
         response = request(
             method="POST",
             url=service_callback_api.url,
@@ -71,6 +72,14 @@ def send_delivery_status_to_service(self, notification_id):
             )
         )
         if not isinstance(e, HTTPError) or e.response.status_code >= 500:
+            retry = True
+    except Exception as e:
+        current_app.logger.exception(
+            'Unhandled exception when sending callback for notification {}'.format(notification_id)
+        )
+        retry = True
+
+    if retry:
         try:
             self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:

manifest-delivery-base.yml

@@ -95,6 +95,6 @@ applications:
       NOTIFY_APP_NAME: delivery-worker-receipts
 
   - name: notify-delivery-worker-service-callbacks
-    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO -P eventlet -c 1000 -Q service-callbacks
+    command: scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q service-callbacks
     env:
       NOTIFY_APP_NAME: delivery-worker-service-callbacks

tests/app/celery/test_service_callback_tasks.py

@@ -1,10 +1,11 @@
+import uuid
 import json
 from datetime import datetime
 
+from requests import RequestException
 import pytest
 import requests_mock
-
+from sqlalchemy.exc import SQLAlchemyError
-from requests import RequestException
 
 from app import (DATETIME_FORMAT)
 
@@ -18,6 +19,7 @@ from tests.app.db import (
     create_service_callback_api
 )
 from app.celery.service_callback_tasks import send_delivery_status_to_service
+from app.config import QueueNames
 
 
 @pytest.mark.parametrize("notification_type",
@@ -88,7 +90,7 @@ def test_send_delivery_status_to_service_does_not_sent_request_when_service_call
     mocked = mocker.patch("requests.request")
     send_delivery_status_to_service(notification.id)
 
-    mocked.call_count == 0
+    assert mocked.call_count == 0
 
 
 @pytest.mark.parametrize("notification_type",
@@ -182,4 +184,15 @@ def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404
                           status_code=404)
         send_delivery_status_to_service(notification.id)
 
-    mocked.call_count == 0
+    assert mocked.call_count == 0
+
+
+def test_send_delivery_status_to_service_retries_if_database_error(client, mocker):
+    notification_id = uuid.uuid4()
+    db_call = mocker.patch('app.celery.service_callback_tasks.get_notification_by_id', side_effect=SQLAlchemyError)
+    retry = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
+
+    send_delivery_status_to_service(notification_id)
+
+    db_call.assert_called_once_with(notification_id)
+    retry.assert_called_once_with(queue=QueueNames.RETRY)