Throw an exception whenever we updated a notification to technical failure.

If this is happening we want to know about it.

tests/app/celery/test_ftp_update_tasks.py

@@ -5,7 +5,7 @@ import pytest
 from freezegun import freeze_time
 from flask import current_app
 
-from app.exceptions import DVLAException
+from app.exceptions import DVLAException, NotificationTechnicalFailureException
 from app.models import (
     Job,
     Notification,
@@ -276,7 +276,9 @@ def test_update_letter_notifications_to_error_updates_based_on_notification_refe
     create_service_callback_api(service=sample_letter_template.service, url="https://original_url.com")
     dt = datetime.utcnow()
     with freeze_time(dt):
-        update_letter_notifications_to_error([first.reference])
+        with pytest.raises(NotificationTechnicalFailureException) as e:
+            update_letter_notifications_to_error([first.reference])
+    assert first.reference in e.value.message
 
     assert first.status == NOTIFICATION_TECHNICAL_FAILURE
     assert first.sent_by is None

tests/app/celery/test_provider_tasks.py

@@ -1,9 +1,11 @@
+import pytest
 from celery.exceptions import MaxRetriesExceededError
 from notifications_utils.recipients import InvalidEmailError
 
 import app
 from app.celery import provider_tasks
 from app.celery.provider_tasks import deliver_sms, deliver_email
+from app.exceptions import NotificationTechnicalFailureException
 
 
 def test_should_have_decorated_tasks_functions():
@@ -59,21 +61,25 @@ def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(s
     mocker.patch('app.delivery.send_to_providers.send_sms_to_provider', side_effect=Exception("EXPECTED"))
     mocker.patch('app.celery.provider_tasks.deliver_sms.retry', side_effect=MaxRetriesExceededError())
 
-    deliver_sms(sample_notification.id)
+    with pytest.raises(NotificationTechnicalFailureException) as e:
+        deliver_sms(sample_notification.id)
 
     provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks")
 
     assert sample_notification.status == 'technical-failure'
+    assert str(sample_notification.id) in e.value.message
 
 
 def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_email_task(sample_notification, mocker):
     mocker.patch('app.delivery.send_to_providers.send_email_to_provider', side_effect=Exception("EXPECTED"))
     mocker.patch('app.celery.provider_tasks.deliver_email.retry', side_effect=MaxRetriesExceededError())
 
-    deliver_email(sample_notification.id)
+    with pytest.raises(NotificationTechnicalFailureException) as e:
+        deliver_email(sample_notification.id)
 
     provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks")
     assert sample_notification.status == 'technical-failure'
+    assert str(sample_notification.id) in e.value.message
 
 
 def test_should_technical_error_and_not_retry_if_invalid_email(sample_notification, mocker):

tests/app/celery/test_scheduled_tasks.py

@@ -44,6 +44,7 @@ from app.dao.provider_details_dao import (
     dao_update_provider_details,
     get_current_provider
 )
+from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
     MonthlyBilling,
     NotificationHistory,
@@ -181,8 +182,10 @@ def test_update_status_of_notifications_after_timeout(notify_api, sample_templat
             status='pending',
             created_at=datetime.utcnow() - timedelta(
                 seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10))
-        timeout_notifications()
-
+        with pytest.raises(NotificationTechnicalFailureException) as e:
+            timeout_notifications()
+        print(e.value.message)
+        assert str(not2.id) in e.value.message
         assert not1.status == 'temporary-failure'
         assert not2.status == 'technical-failure'
         assert not3.status == 'temporary-failure'

tests/app/dao/notification_dao/test_notification_dao.py

@@ -1222,7 +1222,7 @@ def test_dao_timeout_notifications(sample_template):
     assert Notification.query.get(sending.id).status == 'sending'
     assert Notification.query.get(pending.id).status == 'pending'
     assert Notification.query.get(delivered.id).status == 'delivered'
-    updated_ids = dao_timeout_notifications(1)
+    technical_failure_notifications, temporary_failure_notifications = dao_timeout_notifications(1)
     assert Notification.query.get(created.id).status == 'technical-failure'
     assert Notification.query.get(sending.id).status == 'temporary-failure'
     assert Notification.query.get(pending.id).status == 'temporary-failure'
@@ -1231,7 +1231,7 @@ def test_dao_timeout_notifications(sample_template):
     assert NotificationHistory.query.get(sending.id).status == 'temporary-failure'
     assert NotificationHistory.query.get(pending.id).status == 'temporary-failure'
     assert NotificationHistory.query.get(delivered.id).status == 'delivered'
-    assert len(updated_ids) == 3
+    assert len(technical_failure_notifications + temporary_failure_notifications) == 3
 
 
 def test_dao_timeout_notifications_only_updates_for_older_notifications(sample_template):
@@ -1245,12 +1245,12 @@ def test_dao_timeout_notifications_only_updates_for_older_notifications(sample_t
     assert Notification.query.get(sending.id).status == 'sending'
     assert Notification.query.get(pending.id).status == 'pending'
     assert Notification.query.get(delivered.id).status == 'delivered'
-    updated_ids = dao_timeout_notifications(1)
+    technical_failure_notifications, temporary_failure_notifications = dao_timeout_notifications(1)
     assert NotificationHistory.query.get(created.id).status == 'created'
     assert NotificationHistory.query.get(sending.id).status == 'sending'
     assert NotificationHistory.query.get(pending.id).status == 'pending'
     assert NotificationHistory.query.get(delivered.id).status == 'delivered'
-    assert len(updated_ids) == 0
+    assert len(technical_failure_notifications + temporary_failure_notifications) == 0
 
 
 def test_dao_timeout_notifications_doesnt_affect_letters(sample_letter_template):
@@ -1265,13 +1265,13 @@ def test_dao_timeout_notifications_doesnt_affect_letters(sample_letter_template)
     assert Notification.query.get(pending.id).status == 'pending'
     assert Notification.query.get(delivered.id).status == 'delivered'
 
-    updated_ids = dao_timeout_notifications(1)
+    technical_failure_notifications, temporary_failure_notifications = dao_timeout_notifications(1)
 
     assert NotificationHistory.query.get(created.id).status == 'created'
     assert NotificationHistory.query.get(sending.id).status == 'sending'
     assert NotificationHistory.query.get(pending.id).status == 'pending'
     assert NotificationHistory.query.get(delivered.id).status == 'delivered'
-    assert len(updated_ids) == 0
+    assert len(technical_failure_notifications + temporary_failure_notifications) == 0
 
 
 def test_should_return_notifications_excluding_jobs_by_default(sample_template, sample_job, sample_api_key):

tests/app/delivery/test_send_to_providers.py

@@ -13,6 +13,7 @@ from app import mmg_client, firetext_client
 from app.dao import (provider_details_dao, notifications_dao)
 from app.dao.provider_details_dao import dao_switch_sms_provider_to_provider_with_identifier
 from app.delivery import send_to_providers
+from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
     Notification,
     EmailBranding,
@@ -137,9 +138,11 @@ def test_should_not_send_email_message_when_service_is_inactive_notifcation_is_i
     sample_service.active = False
     send_mock = mocker.patch("app.aws_ses_client.send_email", return_value='reference')
 
-    send_to_providers.send_email_to_provider(sample_notification)
+    with pytest.raises(NotificationTechnicalFailureException) as e:
+        send_to_providers.send_email_to_provider(sample_notification)
     send_mock.assert_not_called()
     assert Notification.query.get(sample_notification.id).status == 'technical-failure'
+    assert str(sample_notification.id) in e.value.message
 
 
 @pytest.mark.parametrize("client_send", ["app.mmg_client.send_sms", "app.firetext_client.send_sms"])
@@ -148,9 +151,11 @@ def test_should_not_send_sms_message_when_service_is_inactive_notifcation_is_in_
     sample_service.active = False
     send_mock = mocker.patch(client_send, return_value='reference')
 
-    send_to_providers.send_sms_to_provider(sample_notification)
+    with pytest.raises(NotificationTechnicalFailureException) as e:
+        send_to_providers.send_sms_to_provider(sample_notification)
     send_mock.assert_not_called()
     assert Notification.query.get(sample_notification.id).status == 'technical-failure'
+    assert str(sample_notification.id) in e.value.message
 
 
 def test_send_sms_should_use_template_version_from_notification_not_latest(