From d85a71758ccd8b0dfb742dd7d77fc4374bdb3f61 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Mon, 30 Oct 2017 13:43:23 +0000 Subject: [PATCH] Retry in only certain scenarios Instead of retrying if there are genuine errors, only retry if there are errors which are unexpected as otherwise the retries will happen and fail for the same reason e.g. that the message has changed format and will require a code update. - Updated process_ses_results to only retry if there in an unknown exception - Update test and assert that there is a retry there is a unknown exception --- app/celery/tasks.py | 9 ++++++--- tests/app/celery/test_tasks.py | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index abafa6b78..b5e732df5 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -555,7 +555,10 @@ def process_incomplete_job(job_id): @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300) @statsd(namespace="tasks") def process_ses_results(self, response): - errors = process_ses_response(response) - if errors: - current_app.logger.error(errors) + try: + errors = process_ses_response(response) + if errors: + current_app.logger.error(errors) + except Exception: + current_app.logger.exception('Error processing SES results') self.retry(queue=QueueNames.RETRY, exc="SES responses processed with error") diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 39bce8e69..69f32c421 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -1398,7 +1398,15 @@ def test_process_ses_results(notify_db, notify_db_session, sample_email_template assert process_ses_results(response=response) is None +def test_process_ses_results_does_not_retry_if_errors(notify_db, mocker): + mocked = mocker.patch('app.celery.tasks.process_ses_results.retry') + response = json.loads(ses_notification_callback()) + process_ses_results(response=response) + assert mocked.call_count == 0 + + def test_process_ses_results_retry_called(notify_db, mocker): + mocker.patch("app.dao.notifications_dao.update_notification_status_by_reference", side_effect=Exception("EXPECTED")) mocked = mocker.patch('app.celery.tasks.process_ses_results.retry') response = json.loads(ses_notification_callback()) process_ses_results(response=response)