In event of a task retry ensure the log message is identifier as such

- "RETRY" prefixes the messages

In event of the retry attempts completing without successfully completing the task identify message as such

- "RETRY FAILED" prefixes the messages

Applies to the send_sms|send_email and send_sms_to_provider|send_email_to_provider tasks

These are there to try and ensure we can alert on these events so that we know if we have started retrying messages

Retry messages also contain notification ids to aid debugging.

app/celery/provider_tasks.py

@@ -92,11 +92,15 @@ def send_sms_to_provider(self, service_id, notification_id):
         except SmsClientException as e:
             try:
                 current_app.logger.error(
-                    "SMS notification {} failed".format(notification_id)
+                    "RETRY: SMS notification {} failed".format(notification_id)
                 )
                 current_app.logger.exception(e)
                 self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
             except self.MaxRetriesExceededError:
+                current_app.logger.error(
+                    "RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification.id),
+                    e
+                )
                 update_notification_status_by_id(notification.id, 'technical-failure', 'failure')
 
         current_app.logger.info(
@@ -173,11 +177,15 @@ def send_email_to_provider(self, service_id, notification_id):
         except EmailClientException as e:
             try:
                 current_app.logger.error(
-                    "Email notification {} failed".format(notification_id)
+                    "RETRY: Email notification {} failed".format(notification_id)
                 )
                 current_app.logger.exception(e)
                 self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
             except self.MaxRetriesExceededError:
+                current_app.logger.error(
+                    "RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification.id),
+                    e
+                )
                 update_notification_status_by_id(notification.id, 'technical-failure', 'failure')
 
         current_app.logger.info(

app/celery/tasks.py

@@ -154,8 +154,14 @@ def send_sms(self,
         )
 
     except SQLAlchemyError as e:
-        current_app.logger.exception(e)
+        current_app.logger.exception("RETRY: send_sms notification {}".format(notification_id), e)
+        try:
             raise self.retry(queue="retry", exc=e)
+        except self.MaxRetriesExceededError:
+            current_app.logger.exception(
+                "RETRY FAILED: task send_sms failed for notification {}".format(notification.id),
+                e
+            )
 
 
 @notify_celery.task(bind=True, name="send-email", max_retries=5, default_retry_delay=300)
@@ -180,8 +186,14 @@ def send_email(self, service_id,
 
         current_app.logger.info("Email {} created at {}".format(notification_id, created_at))
     except SQLAlchemyError as e:
-        current_app.logger.exception(e)
+        current_app.logger.exception("RETRY: send_email notification {}".format(notification_id), e)
+        try:
             raise self.retry(queue="retry", exc=e)
+        except self.MaxRetriesExceededError:
+            current_app.logger.error(
+                "RETRY FAILED: task send_email failed for notification {}".format(notification.id),
+                e
+            )
 
 
 def _save_notification(created_at, notification, notification_id, service_id, notification_type, api_key_id, key_type):