From d582e35471a8b3c0e17fd9cdbca629c122b271de Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Fri, 5 Feb 2021 12:46:23 +0000 Subject: [PATCH] dont try and send broadccast event if it's already in technical-failure this gives us an option to manually set a status in the database and avoid things being stuck in a retry loop forever --- app/celery/broadcast_message_tasks.py | 9 +++++++++ tests/app/celery/test_broadcast_message_tasks.py | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/celery/broadcast_message_tasks.py b/app/celery/broadcast_message_tasks.py index 31de633cf..e253d6e2c 100644 --- a/app/celery/broadcast_message_tasks.py +++ b/app/celery/broadcast_message_tasks.py @@ -51,6 +51,15 @@ def check_provider_message_should_send(broadcast_event, provider): 4. If you need to re-send this task off again, you'll need to run the following command on paas: `send_broadcast_provider_message.apply_async(args=(broadcast_event_id, provider), queue=QueueNames.BROADCASTS)` """ + current_provider_message = broadcast_event.get_provider_message(provider) + # if this is the first time a task is being executed, it won't have a provider message yet + if current_provider_message and current_provider_message.status == BroadcastProviderMessageStatus.TECHNICAL_FAILURE: + raise CBCProxyFatalException( + f'Cannot send broadcast_event {broadcast_event.id} ' + + f'to provider {provider}: ' + + f'It is already in status technical-failure' + ) + if broadcast_event.transmitted_finishes_at < datetime.utcnow(): # TODO: This should be a different kind of exception to distinguish "We should know something went wrong, but # no immediate action" from "We need to fix this immediately" diff --git a/tests/app/celery/test_broadcast_message_tasks.py b/tests/app/celery/test_broadcast_message_tasks.py index 93de72c0e..c1e7aab5b 100644 --- a/tests/app/celery/test_broadcast_message_tasks.py +++ b/tests/app/celery/test_broadcast_message_tasks.py @@ -709,8 +709,10 @@ def test_check_provider_message_should_send_doesnt_raise_if_newer_event_not_acke BroadcastProviderMessageStatus.ACK, BroadcastProviderMessageStatus.ERR, - # TODO: Make this case fail - so we have a way of aborting a send if it's stuck in retry loop - BroadcastProviderMessageStatus.TECHNICAL_FAILURE, + pytest.param( + BroadcastProviderMessageStatus.TECHNICAL_FAILURE, + marks=pytest.mark.xfail(raises=CBCProxyFatalException) + ), ]) def test_check_provider_message_should_send_doesnt_raise_if_current_event_already_has_provider_message( sample_template,