Retry in only certain scenarios

Instead of retrying if there are genuine errors, only retry if there are
errors which are unexpected as otherwise the retries will happen and
fail for the same reason e.g. that the message has changed format and
will require a code update.

- Updated process_ses_results to only retry if there in an unknown
exception
- Update test and assert that there is a retry there is a unknown
exception

app/celery/tasks.py

@@ -555,7 +555,10 @@ def process_incomplete_job(job_id):
 @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
 @statsd(namespace="tasks")
 def process_ses_results(self, response):
-    errors = process_ses_response(response)
+    try:
-    if errors:
+        errors = process_ses_response(response)
-        current_app.logger.error(errors)
+        if errors:
+            current_app.logger.error(errors)
+    except Exception:
+        current_app.logger.exception('Error processing SES results')
         self.retry(queue=QueueNames.RETRY, exc="SES responses processed with error")

tests/app/celery/test_tasks.py

@@ -1398,7 +1398,15 @@ def test_process_ses_results(notify_db, notify_db_session, sample_email_template
     assert process_ses_results(response=response) is None
 
 
+def test_process_ses_results_does_not_retry_if_errors(notify_db, mocker):
+    mocked = mocker.patch('app.celery.tasks.process_ses_results.retry')
+    response = json.loads(ses_notification_callback())
+    process_ses_results(response=response)
+    assert mocked.call_count == 0
+
+
 def test_process_ses_results_retry_called(notify_db, mocker):
+    mocker.patch("app.dao.notifications_dao.update_notification_status_by_reference", side_effect=Exception("EXPECTED"))
     mocked = mocker.patch('app.celery.tasks.process_ses_results.retry')
     response = json.loads(ses_notification_callback())
     process_ses_results(response=response)