diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py
index aec055246..7878461b9 100644
--- a/app/celery/provider_tasks.py
+++ b/app/celery/provider_tasks.py
@@ -26,7 +26,10 @@ def deliver_sms(self, notification_id):
             current_app.logger.exception(
                 "SMS notification delivery for id: {} failed".format(notification_id)
             )
-            self.retry(queue=QueueNames.RETRY)
+            if self.request.retries == 0:
+                self.retry(queue=QueueNames.RETRY, countdown=0)
+            else:
+                self.retry(queue=QueueNames.RETRY)
         except self.MaxRetriesExceededError:
             message = "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. " \
                       "Notification has been updated to technical-failure".format(notification_id)
diff --git a/tests/app/celery/test_provider_tasks.py b/tests/app/celery/test_provider_tasks.py
index 65b63682a..8ff6d6298 100644
--- a/tests/app/celery/test_provider_tasks.py
+++ b/tests/app/celery/test_provider_tasks.py
@@ -34,7 +34,7 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task
 
     deliver_sms(notification_id)
     app.delivery.send_to_providers.send_sms_to_provider.assert_not_called()
-    app.celery.provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks")
+    app.celery.provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=0)
 
 
 def test_should_call_send_email_to_provider_from_deliver_email_task(
@@ -66,7 +66,7 @@ def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(s
     with pytest.raises(NotificationTechnicalFailureException) as e:
         deliver_sms(sample_notification.id)
 
-    provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks")
+    provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=0)
 
     assert sample_notification.status == 'technical-failure'
     assert str(sample_notification.id) in e.value.message