diff --git a/app/celery/broadcast_message_tasks.py b/app/celery/broadcast_message_tasks.py index e253d6e2c..18ca0225a 100644 --- a/app/celery/broadcast_message_tasks.py +++ b/app/celery/broadcast_message_tasks.py @@ -28,8 +28,8 @@ def get_retry_delay(retry_count): # 2 to the power of x. 1, 2, 4, 8, 16, 32, ... delay = 2**retry_count - # never wait longer than 4 minutes - return min(delay, 240) + # never wait longer than 5 minutes + return min(delay, 300) def check_provider_message_should_send(broadcast_event, provider): @@ -51,15 +51,6 @@ def check_provider_message_should_send(broadcast_event, provider): 4. If you need to re-send this task off again, you'll need to run the following command on paas: `send_broadcast_provider_message.apply_async(args=(broadcast_event_id, provider), queue=QueueNames.BROADCASTS)` """ - current_provider_message = broadcast_event.get_provider_message(provider) - # if this is the first time a task is being executed, it won't have a provider message yet - if current_provider_message and current_provider_message.status == BroadcastProviderMessageStatus.TECHNICAL_FAILURE: - raise CBCProxyFatalException( - f'Cannot send broadcast_event {broadcast_event.id} ' + - f'to provider {provider}: ' + - f'It is already in status technical-failure' - ) - if broadcast_event.transmitted_finishes_at < datetime.utcnow(): # TODO: This should be a different kind of exception to distinguish "We should know something went wrong, but # no immediate action" from "We need to fix this immediately" diff --git a/manifest.yml.j2 b/manifest.yml.j2 index 8a7cf8832..6f7d16062 100644 --- a/manifest.yml.j2 +++ b/manifest.yml.j2 @@ -64,9 +64,7 @@ 'notify-delivery-worker-letters': {'memory': '2G'}, 'notify-delivery-worker-retry-tasks': {}, 'notify-delivery-worker-internal': {}, - 'notify-delivery-worker-broadcasts': { - 'CELERYD_PREFETCH_MULTIPLIER': 1, - }, + 'notify-delivery-worker-broadcasts': {}, 'notify-delivery-worker-receipts': {}, 'notify-delivery-worker-service-callbacks': {'disk_quota': '2G'}, 'notify-delivery-worker-save-api-notifications': {'disk_quota': '2G'}, diff --git a/tests/app/celery/test_broadcast_message_tasks.py b/tests/app/celery/test_broadcast_message_tasks.py index c1e7aab5b..28c22d88f 100644 --- a/tests/app/celery/test_broadcast_message_tasks.py +++ b/tests/app/celery/test_broadcast_message_tasks.py @@ -505,7 +505,7 @@ def test_send_broadcast_provider_message_errors(mocker, sample_service, provider @pytest.mark.parametrize('num_retries, expected_countdown', [ (0, 1), (5, 32), - (20, 240), + (20, 300), ]) def test_send_broadcast_provider_message_delays_retry_exponentially( mocker, @@ -587,10 +587,10 @@ def test_trigger_link_tests_invokes_cbc_proxy_client( (0, 1), (1, 2), (2, 4), - (7, 128), - (8, 240), - (9, 240), - (1000, 240), + (8, 256), + (9, 300), + (10, 300), + (1000, 300), ]) def test_get_retry_delay_has_capped_backoff(retry_count, expected_delay): assert get_retry_delay(retry_count) == expected_delay @@ -709,10 +709,8 @@ def test_check_provider_message_should_send_doesnt_raise_if_newer_event_not_acke BroadcastProviderMessageStatus.ACK, BroadcastProviderMessageStatus.ERR, - pytest.param( - BroadcastProviderMessageStatus.TECHNICAL_FAILURE, - marks=pytest.mark.xfail(raises=CBCProxyFatalException) - ), + # TODO: Make this case fail - so we have a way of aborting a send if it's stuck in retry loop + BroadcastProviderMessageStatus.TECHNICAL_FAILURE, ]) def test_check_provider_message_should_send_doesnt_raise_if_current_event_already_has_provider_message( sample_template,