From c2f2d39d22a1baec4b57f32fd8b90031b7c95917 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Tue, 24 Oct 2017 16:58:39 +0100 Subject: [PATCH] Retry process_ses_results if it fails If the SES task fails retry every 5 mins for an hour. - Updated the process_ses_results task to retry - Added tests to check for retry --- app/celery/tasks.py | 7 +++-- tests/app/celery/test_tasks.py | 52 ++++++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index e4dcdafda..e2bda04da 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -550,7 +550,10 @@ def process_incomplete_job(job_id): job_complete(job, job.service, template, resumed=True) -@notify_celery.task(bind=True, name='process-ses-result') +@notify_celery.task(bind=True, name="process-ses-result", max_retries=12, default_retry_delay=300000) @statsd(namespace="tasks") def process_ses_results(self, response): - process_ses_response(response) + errors = process_ses_response(response) + if errors: + current_app.logger.error(errors) + self.retry(queue=QueueNames.RETRY, exc="SES responses processed with error") diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index e018a4f32..39bce8e69 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -1,3 +1,4 @@ +import codecs import json import uuid from datetime import datetime, timedelta @@ -26,8 +27,8 @@ from app.celery.tasks import ( process_incomplete_jobs, get_template_class, s3, - send_inbound_sms_to_service -) + send_inbound_sms_to_service, + process_ses_results) from app.config import QueueNames from app.dao import jobs_dao, services_dao from app.models import ( @@ -1381,3 +1382,50 @@ def test_process_incomplete_job_letter(mocker, sample_letter_template): assert completed_job.job_status == JOB_STATUS_FINISHED assert mock_letter_saver.call_count == 8 + + +def test_process_ses_results(notify_db, notify_db_session, sample_email_template): + + create_sample_notification( + notify_db, + notify_db_session, + template=sample_email_template, + reference='ref1', + sent_at=datetime.utcnow(), + status='sending') + + response = json.loads(ses_notification_callback()) + assert process_ses_results(response=response) is None + + +def test_process_ses_results_retry_called(notify_db, mocker): + mocked = mocker.patch('app.celery.tasks.process_ses_results.retry') + response = json.loads(ses_notification_callback()) + process_ses_results(response=response) + assert mocked.call_count != 0 + + +def ses_notification_callback(): + return '{\n "Type" : "Notification",\n "MessageId" : "ref1",' \ + '\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",' \ + '\n "Message" : "{\\"notificationType\\":\\"Delivery\\",' \ + '\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",' \ + '\\"source\\":\\"test@test-domain.com\\",' \ + '\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",' \ + '\\"sendingAccountId\\":\\"123456789012\\",' \ + '\\"messageId\\":\\"ref1\\",' \ + '\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},' \ + '\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",' \ + '\\"processingTimeMillis\\":658,' \ + '\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],' \ + '\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",' \ + '\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",' \ + '\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",' \ + '\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUt' \ + 'OowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYL' \ + 'VSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMA' \ + 'PmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",' \ + '\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750' \ + 'dd426d95ee9390147a5624348ee.pem",' \ + '\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&S' \ + 'ubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}'