only retry if the broadcast message task is in sending

previously we would retry if the task was queued up for retry but the
status is in "received-ack" or "received-err". We don't expect that a
task will be retried after getting this status, but if there are
duplicate tasks that could happen. Lets plan for the worst by saying
"only process a retry if the task is currently in sending".

this way, if a duplicate task is on retry and the first task goes
through succesfully, the duplicate task will give up.

app/__init__.py

@@ -381,7 +381,7 @@ def setup_sqlalchemy_events(app):
                         'host': current_app.config['NOTIFY_APP_NAME'],  # worker name
                         'url_rule': current_task.name,  # task name
                     }
-                # anything else. migrations possibly.
+                # anything else. migrations possibly, or flask cli commands.
                 else:
                     current_app.logger.warning('Checked out sqlalchemy connection from outside of request/task')
                     connection_record.info['request_data'] = {

app/celery/broadcast_message_tasks.py

@@ -52,11 +52,11 @@ def check_provider_message_should_send(broadcast_event, provider):
     """
     current_provider_message = broadcast_event.get_provider_message(provider)
     # if this is the first time a task is being executed, it won't have a provider message yet
-    if current_provider_message and current_provider_message.status == BroadcastProviderMessageStatus.TECHNICAL_FAILURE:
+    if current_provider_message and current_provider_message.status != BroadcastProviderMessageStatus.SENDING:
         raise CBCProxyFatalException(
             f'Cannot send broadcast_event {broadcast_event.id} ' +
             f'to provider {provider}: ' +
-            'It is already in status technical-failure'
+            f'It is in status {current_provider_message.status}'
         )
 
     if broadcast_event.transmitted_finishes_at < datetime.utcnow():

tests/app/celery/test_broadcast_message_tasks.py

@@ -701,15 +701,20 @@ def test_check_provider_message_should_send_doesnt_raise_if_newer_event_not_acke
 
 @pytest.mark.parametrize('existing_message_status', [
     BroadcastProviderMessageStatus.SENDING,
-    BroadcastProviderMessageStatus.ACK,
-    BroadcastProviderMessageStatus.ERR,
-
+    pytest.param(
+        BroadcastProviderMessageStatus.ACK,
+        marks=pytest.mark.xfail(raises=CBCProxyFatalException)
+    ),
+    pytest.param(
+        BroadcastProviderMessageStatus.ERR,
+        marks=pytest.mark.xfail(raises=CBCProxyFatalException)
+    ),
     pytest.param(
         BroadcastProviderMessageStatus.TECHNICAL_FAILURE,
         marks=pytest.mark.xfail(raises=CBCProxyFatalException)
     ),
 ])
-def test_check_provider_message_should_send_doesnt_raise_if_current_event_already_has_provider_message(
+def test_check_provider_message_should_send_raises_if_current_event_already_has_provider_message_not_in_sending(
     sample_template,
     existing_message_status
 ):