Revert "Merge pull request #3123 from alphagov/retry-loop-fix"

This reverts commit 541a765811, reversing
changes made to 6a9ac654a6.

app/celery/broadcast_message_tasks.py

@@ -28,8 +28,8 @@ def get_retry_delay(retry_count):
 
     # 2 to the power of x. 1, 2, 4, 8, 16, 32, ...
     delay = 2**retry_count
-    # never wait longer than 4 minutes
+    # never wait longer than 5 minutes
-    return min(delay, 240)
+    return min(delay, 300)
 
 
 def check_provider_message_should_send(broadcast_event, provider):
@@ -51,15 +51,6 @@ def check_provider_message_should_send(broadcast_event, provider):
     4. If you need to re-send this task off again, you'll need to run the following command on paas:
        `send_broadcast_provider_message.apply_async(args=(broadcast_event_id, provider), queue=QueueNames.BROADCASTS)`
     """
-    current_provider_message = broadcast_event.get_provider_message(provider)
-    # if this is the first time a task is being executed, it won't have a provider message yet
-    if current_provider_message and current_provider_message.status == BroadcastProviderMessageStatus.TECHNICAL_FAILURE:
-        raise CBCProxyFatalException(
-            f'Cannot send broadcast_event {broadcast_event.id} ' +
-            f'to provider {provider}: ' +
-            f'It is already in status technical-failure'
-        )
-
     if broadcast_event.transmitted_finishes_at < datetime.utcnow():
         # TODO: This should be a different kind of exception to distinguish "We should know something went wrong, but
         # no immediate action" from "We need to fix this immediately"

manifest.yml.j2

@@ -64,9 +64,7 @@
   'notify-delivery-worker-letters': {'memory': '2G'},
   'notify-delivery-worker-retry-tasks': {},
   'notify-delivery-worker-internal': {},
-  'notify-delivery-worker-broadcasts': {
+  'notify-delivery-worker-broadcasts': {},
-      'CELERYD_PREFETCH_MULTIPLIER': 1,
-  },
   'notify-delivery-worker-receipts': {},
   'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'},
   'notify-delivery-worker-save-api-notifications': {'disk_quota': '2G'},

tests/app/celery/test_broadcast_message_tasks.py

@@ -505,7 +505,7 @@ def test_send_broadcast_provider_message_errors(mocker, sample_service, provider
 @pytest.mark.parametrize('num_retries, expected_countdown', [
     (0, 1),
     (5, 32),
-    (20, 240),
+    (20, 300),
 ])
 def test_send_broadcast_provider_message_delays_retry_exponentially(
     mocker,
@@ -587,10 +587,10 @@ def test_trigger_link_tests_invokes_cbc_proxy_client(
     (0, 1),
     (1, 2),
     (2, 4),
-    (7, 128),
+    (8, 256),
-    (8, 240),
+    (9, 300),
-    (9, 240),
+    (10, 300),
-    (1000, 240),
+    (1000, 300),
 ])
 def test_get_retry_delay_has_capped_backoff(retry_count, expected_delay):
     assert get_retry_delay(retry_count) == expected_delay
@@ -709,10 +709,8 @@ def test_check_provider_message_should_send_doesnt_raise_if_newer_event_not_acke
     BroadcastProviderMessageStatus.ACK,
     BroadcastProviderMessageStatus.ERR,
 
-    pytest.param(
+    # TODO: Make this case fail - so we have a way of aborting a send if it's stuck in retry loop
     BroadcastProviderMessageStatus.TECHNICAL_FAILURE,
-        marks=pytest.mark.xfail(raises=CBCProxyFatalException)
-    ),
 ])
 def test_check_provider_message_should_send_doesnt_raise_if_current_event_already_has_provider_message(
     sample_template,