mirror of
https://github.com/GSA/notifications-api.git
synced 2025-12-18 05:41:41 -05:00
update research mode email callbacks to add process-ses-response task to queue
this involved: * moving that task to callback_tasks to prevent circular imports * updating the dummy research mode callbacks (with actual SNS messages from the ses simulator emails) * refactoring tests
This commit is contained in:
Leo Hemsted
18
app/celery/callback_tasks.py
Normal file
18
app/celery/callback_tasks.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from flask import current_app
|
||||
|
||||
from app import notify_celery
|
||||
from app.config import QueueNames
|
||||
from app.statsd_decorators import statsd
|
||||
from app.notifications.notifications_ses_callback import process_ses_response
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def process_ses_results(self, response):
|
||||
try:
|
||||
errors = process_ses_response(response)
|
||||
if errors:
|
||||
current_app.logger.error(errors)
|
||||
except Exception:
|
||||
current_app.logger.exception('Error processing SES results')
|
||||
self.retry(queue=QueueNames.RETRY, exc="SES responses processed with error")
|
||||
@@ -1,10 +1,11 @@
|
||||
import json
|
||||
|
||||
from flask import current_app
|
||||
from app import notify_celery
|
||||
from requests import request, RequestException, HTTPError
|
||||
|
||||
from app.models import SMS_TYPE
|
||||
from app.config import QueueNames
|
||||
from app.celery.callback_tasks import process_ses_results
|
||||
|
||||
temp_fail = "7700900003"
|
||||
perm_fail = "7700900002"
|
||||
@@ -36,7 +37,7 @@ def send_sms_response(provider, reference, to):
|
||||
make_request(SMS_TYPE, provider, body, headers)
|
||||
|
||||
|
||||
def send_email_response(provider, reference, to):
|
||||
def send_email_response(reference, to):
|
||||
if to == perm_fail_email:
|
||||
body = ses_hard_bounce_callback(reference)
|
||||
elif to == temp_fail_email:
|
||||
@@ -44,7 +45,7 @@ def send_email_response(provider, reference, to):
|
||||
else:
|
||||
body = ses_notification_callback(reference)
|
||||
|
||||
make_request('email', provider, body, headers={"Content-type": "application/json"})
|
||||
process_ses_results.apply_async([body], queue=QueueNames.RESEARCH_MODE)
|
||||
|
||||
|
||||
def make_request(notification_type, provider, data, headers):
|
||||
@@ -115,12 +116,44 @@ def firetext_callback(notification_id, to):
|
||||
|
||||
|
||||
def ses_notification_callback(reference):
|
||||
return '{ "Type" : "Notification", "MessageId" : "%s", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"%s\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' % (reference, reference) # noqa
|
||||
return {
|
||||
'EventSource': 'aws:sns',
|
||||
'EventVersion': '1.0',
|
||||
'EventSubscriptionArn': 'arn:aws:sns:eu-west-1:302763885840:ses_notifications:27447d51-7008-4d9c-83f2-519983b60937',
|
||||
'Sns': {
|
||||
'Type': 'Notification',
|
||||
'MessageId': '8e83c020-3a50-5957-a2ca-92a8ee9baa0a',
|
||||
'TopicArn': 'arn:aws:sns:eu-west-1:302763885840:ses_notifications',
|
||||
'Subject': None,
|
||||
'Message': '''{"notificationType":"Delivery","mail":{"timestamp":"2017-11-17T12:14:01.643Z","source":"\\"sakis\\" <sakis@notify.works>","sourceArn":"arn:aws:ses:eu-west-1:302763885840:identity/notify.works","sourceIp":"52.208.24.161","sendingAccountId":"302763885840","messageId":"0102015fc9e669ab-d1395dba-84c7-4311-9f25-405396a3f7aa-000000","destination":["success@simulator.amazonses.com"],"headersTruncated":false,"headers":[{"name":"From","value":"sakis <sakis@notify.works>"},{"name":"To","value":"success@simulator.amazonses.com"},{"name":"Subject","value":"lambda test"},{"name":"MIME-Version","value":"1.0"},{"name":"Content-Type","value":"multipart/alternative; boundary=\\"----=_Part_617203_1627511946.1510920841645\\""}],"commonHeaders":{"from":["sakis <sakis@notify.works>"],"to":["success@simulator.amazonses.com"],"subject":"lambda test"}},"delivery":{"timestamp":"2017-11-17T12:14:03.646Z","processingTimeMillis":2003,"recipients":["success@simulator.amazonses.com"],"smtpResponse":"250 2.6.0 Message received","remoteMtaIp":"207.171.163.188","reportingMTA":"a7-32.smtp-out.eu-west-1.amazonses.com"}}', 'Timestamp': '2017-11-17T12:14:03.710Z', 'SignatureVersion': '1', 'Signature': 'IxQPpK5kHVSiYhFqWjH35ElZSUOkE29hDcdCjFrA6Cx51Fw5ZNyFGsYJQsCskVEIXteTrn/9VU9zeW5oSf81dbGzv5GnFF4iq8hq+WISQ3etVGx9cOzRABudt82okoIPLU71dsENwj3scibVvsBSP8vD4NJsnOXVfVo1CczumbDT601dFomF45u1bRpg684zUOxvZBpStUfkFaBkDrWp9yt6j5SuDx+AqC0nuQdGPN0+LFbLXN20SZmUwqDiX89xm2JlPBaWimnm0/jBRAqLkSJcix7ssD6ELsCIebljzggibnKzQo3vQV1Frji6+713WlYC7lnziNhVT1VL1tCrhA==', 'SigningCertUrl': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-433026a4050d206028891664da859041.pem', 'UnsubscribeUrl': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:ses_notifications:27447d51-7008-4d9c-83f2-519983b60937''', # noqa
|
||||
'MessageAttributes': {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def ses_hard_bounce_callback(reference):
|
||||
return '{ "Type" : "Notification", "MessageId" : "%s", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Permanent\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' % (reference, reference) # noqa
|
||||
return {
|
||||
'Message': '{"notificationType":"Bounce","bounce":{"bounceType":"Permanent","bounceSubType":"General"}, "mail":{"messageId":"%s","timestamp":"2016-03-14T12:35:25.909Z","source":"test@test-domain.com","sourceArn":"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify","sendingAccountId":"123456789012","destination":["testing@digital.cabinet-office.gov.uk"]},"delivery":{"timestamp":"2016-03-14T12:35:26.567Z","processingTimeMillis":658,"recipients":["testing@digital.cabinet-office.gov.uk"],"smtpResponse":"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp","reportingMTA":"a6-238.smtp-out.eu-west-1.amazonses.com"}}' % reference, # noqa
|
||||
'MessageId': reference,
|
||||
'Signature': 'X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==', # noqa
|
||||
'SignatureVersion': '1',
|
||||
'SigningCertURL': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem', # noqa
|
||||
'Timestamp': '2016-03-14T12:35:26.665Z',
|
||||
'TopicArn': 'arn:aws:sns:eu-west-1:123456789012:testing',
|
||||
'Type': 'Notification',
|
||||
'UnsubscribeURL': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da' # noqa
|
||||
}
|
||||
|
||||
|
||||
def ses_soft_bounce_callback(reference):
|
||||
return '{ "Type" : "Notification", "MessageId" : "%s", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Undetermined\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' % (reference, reference) # noqa
|
||||
return {
|
||||
'Message': '{"notificationType":"Bounce","bounce":{"bounceType":"Temporary","bounceSubType":"General"}, "mail":{"messageId":"%s","timestamp":"2016-03-14T12:35:25.909Z","source":"test@test-domain.com","sourceArn":"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify","sendingAccountId":"123456789012","destination":["testing@digital.cabinet-office.gov.uk"]},"delivery":{"timestamp":"2016-03-14T12:35:26.567Z","processingTimeMillis":658,"recipients":["testing@digital.cabinet-office.gov.uk"],"smtpResponse":"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp","reportingMTA":"a6-238.smtp-out.eu-west-1.amazonses.com"}}' % reference, # noqa
|
||||
'MessageId': reference,
|
||||
'Signature': 'X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==', # noqa
|
||||
'SignatureVersion': '1',
|
||||
'SigningCertURL': 'https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem', # noqa
|
||||
'Timestamp': '2016-03-14T12:35:26.665Z',
|
||||
'TopicArn': 'arn:aws:sns:eu-west-1:123456789012:testing',
|
||||
'Type': 'Notification',
|
||||
'UnsubscribeURL': 'https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da' # noqa
|
||||
}
|
||||
|
||||
@@ -63,7 +63,6 @@ from app.models import (
|
||||
SMS_TYPE,
|
||||
)
|
||||
from app.notifications.process_notifications import persist_notification
|
||||
from app.notifications.notifications_ses_callback import process_ses_response
|
||||
from app.service.utils import service_allowed_to_send_to
|
||||
from app.statsd_decorators import statsd
|
||||
from notifications_utils.s3 import s3upload
|
||||
@@ -562,15 +561,3 @@ def process_incomplete_job(job_id):
|
||||
process_row(row_number, recipient, personalisation, template, job, job.service)
|
||||
|
||||
job_complete(job, job.service, template, resumed=True)
|
||||
|
||||
|
||||
@notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
|
||||
@statsd(namespace="tasks")
|
||||
def process_ses_results(self, response):
|
||||
try:
|
||||
errors = process_ses_response(response)
|
||||
if errors:
|
||||
current_app.logger.error(errors)
|
||||
except Exception:
|
||||
current_app.logger.exception('Error processing SES results')
|
||||
self.retry(queue=QueueNames.RETRY, exc="SES responses processed with error")
|
||||
|
||||
@@ -126,7 +126,7 @@ def send_email_to_provider(notification):
|
||||
notification.billable_units = 0
|
||||
notification.reference = reference
|
||||
update_notification(notification, provider)
|
||||
send_email_response(provider.get_name(), reference, notification.to)
|
||||
send_email_response(reference, notification.to)
|
||||
else:
|
||||
from_address = '"{}" <{}@{}>'.format(service.name, service.email_from,
|
||||
current_app.config['NOTIFY_EMAIL_DOMAIN'])
|
||||
|
||||
@@ -28,7 +28,7 @@ def process_ses_response(ses_request):
|
||||
|
||||
notification_type = ses_message['notificationType']
|
||||
if notification_type == 'Bounce':
|
||||
current_app.logger.info('SES bounce dict: {}'.format(ses_message['bounce']))
|
||||
current_app.logger.info('SES bounce dict: {}'.format(remove_emails_from_bounce(ses_message['bounce'])))
|
||||
if ses_message['bounce']['bounceType'] == 'Permanent':
|
||||
notification_type = ses_message['bounce']['bounceType'] # permanent or not
|
||||
else:
|
||||
@@ -85,3 +85,8 @@ def process_ses_response(ses_request):
|
||||
except ValueError:
|
||||
error = "{} callback failed: invalid json".format(client_name)
|
||||
return error
|
||||
|
||||
|
||||
def remove_emails_from_bounce(bounce_dict):
|
||||
for recip in bounce_dict['bouncedRecipients']:
|
||||
recip.pop('emailAddress')
|
||||
|
||||
58
tests/app/celery/test_callback_tasks.py
Normal file
58
tests/app/celery/test_callback_tasks.py
Normal file
@@ -0,0 +1,58 @@
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
from app.celery.callback_tasks import process_ses_results
|
||||
|
||||
from tests.app.db import create_notification
|
||||
|
||||
|
||||
def test_process_ses_results(sample_email_template):
|
||||
create_notification(
|
||||
sample_email_template,
|
||||
reference='ref1',
|
||||
sent_at=datetime.utcnow(),
|
||||
status='sending')
|
||||
|
||||
response = json.loads(ses_notification_callback())
|
||||
assert process_ses_results(response=response) is None
|
||||
|
||||
|
||||
def test_process_ses_results_does_not_retry_if_errors(notify_db, mocker):
|
||||
mocked = mocker.patch('app.celery.callback_tasks.process_ses_results.retry')
|
||||
response = json.loads(ses_notification_callback())
|
||||
process_ses_results(response=response)
|
||||
assert mocked.call_count == 0
|
||||
|
||||
|
||||
def test_process_ses_results_retry_called(notify_db, mocker):
|
||||
mocker.patch("app.dao.notifications_dao.update_notification_status_by_reference", side_effect=Exception("EXPECTED"))
|
||||
mocked = mocker.patch('app.celery.callback_tasks.process_ses_results.retry')
|
||||
response = json.loads(ses_notification_callback())
|
||||
process_ses_results(response=response)
|
||||
assert mocked.call_count != 0
|
||||
|
||||
|
||||
def ses_notification_callback():
|
||||
return '{\n "Type" : "Notification",\n "MessageId" : "ref1",' \
|
||||
'\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",' \
|
||||
'\n "Message" : "{\\"notificationType\\":\\"Delivery\\",' \
|
||||
'\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",' \
|
||||
'\\"source\\":\\"test@test-domain.com\\",' \
|
||||
'\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",' \
|
||||
'\\"sendingAccountId\\":\\"123456789012\\",' \
|
||||
'\\"messageId\\":\\"ref1\\",' \
|
||||
'\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},' \
|
||||
'\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",' \
|
||||
'\\"processingTimeMillis\\":658,' \
|
||||
'\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],' \
|
||||
'\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",' \
|
||||
'\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",' \
|
||||
'\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",' \
|
||||
'\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUt' \
|
||||
'OowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYL' \
|
||||
'VSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMA' \
|
||||
'PmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",' \
|
||||
'\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750' \
|
||||
'dd426d95ee9390147a5624348ee.pem",' \
|
||||
'\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&S' \
|
||||
'ubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}'
|
||||
@@ -12,8 +12,6 @@ def test_should_have_decorated_tasks_functions():
|
||||
|
||||
|
||||
def test_should_call_send_sms_to_provider_from_deliver_sms_task(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
sample_notification,
|
||||
mocker):
|
||||
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider')
|
||||
@@ -23,7 +21,6 @@ def test_should_call_send_sms_to_provider_from_deliver_sms_task(
|
||||
|
||||
|
||||
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
mocker):
|
||||
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider')
|
||||
@@ -37,8 +34,6 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task
|
||||
|
||||
|
||||
def test_should_call_send_email_to_provider_from_deliver_email_task(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
sample_notification,
|
||||
mocker):
|
||||
mocker.patch('app.delivery.send_to_providers.send_email_to_provider')
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import uuid
|
||||
from unittest.mock import ANY
|
||||
|
||||
import pytest
|
||||
from flask import json
|
||||
|
||||
from app.config import QueueNames
|
||||
from app.celery.research_mode_tasks import (
|
||||
send_sms_response,
|
||||
send_email_response,
|
||||
@@ -42,16 +47,14 @@ def test_make_firetext_callback(notify_api, rmock, phone_number):
|
||||
assert 'mobile={}'.format(phone_number) in rmock.request_history[0].text
|
||||
|
||||
|
||||
def test_make_ses_callback(notify_api, rmock):
|
||||
endpoint = "http://localhost:6011/notifications/email/ses"
|
||||
rmock.request(
|
||||
"POST",
|
||||
endpoint,
|
||||
json={"status": "success"},
|
||||
status_code=200)
|
||||
send_email_response("ses", "1234", "test@test.com")
|
||||
def test_make_ses_callback(notify_api, mocker):
|
||||
mock_task = mocker.patch('app.celery.research_mode_tasks.process_ses_results')
|
||||
some_ref = str(uuid.uuid4())
|
||||
|
||||
assert rmock.called
|
||||
send_email_response(reference=some_ref, to="test@test.com")
|
||||
|
||||
mock_task.apply_async.assert_called_once_with(ANY, queue=QueueNames.RESEARCH_MODE)
|
||||
assert mock_task.apply_async.call_args[0][0][0] == ses_notification_callback(some_ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("phone_number", ["07700900001", "+447700900001", "7700900001", "+44 7700900001",
|
||||
@@ -101,15 +104,3 @@ def test_failure_firetext_callback(phone_number):
|
||||
'time': '2016-03-10 14:17:00',
|
||||
'reference': '1234'
|
||||
}
|
||||
|
||||
|
||||
def test_delivered_ses_callback():
|
||||
assert ses_notification_callback("my-reference") == '{ "Type" : "Notification", "MessageId" : "my-reference", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"my-reference\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' # noqa
|
||||
|
||||
|
||||
def test_ses_hard_bounce_callback():
|
||||
assert ses_hard_bounce_callback("my-reference") == '{ "Type" : "Notification", "MessageId" : "my-reference", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Permanent\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"my-reference\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' # noqa
|
||||
|
||||
|
||||
def ses_soft_bounce_callback():
|
||||
assert ses_soft_bounce_callback("my-reference") == '{ "Type" : "Notification", "MessageId" : "my-reference", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Undetermined\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' # noqa
|
||||
|
||||
@@ -28,8 +28,7 @@ from app.celery.tasks import (
|
||||
process_incomplete_jobs,
|
||||
get_template_class,
|
||||
s3,
|
||||
send_inbound_sms_to_service,
|
||||
process_ses_results)
|
||||
send_inbound_sms_to_service)
|
||||
from app.config import QueueNames
|
||||
from app.dao import jobs_dao, services_dao
|
||||
from app.models import (
|
||||
@@ -1403,58 +1402,3 @@ def test_process_incomplete_job_letter(mocker, sample_letter_template):
|
||||
assert completed_job.job_status == JOB_STATUS_FINISHED
|
||||
|
||||
assert mock_letter_saver.call_count == 8
|
||||
|
||||
|
||||
def test_process_ses_results(notify_db, notify_db_session, sample_email_template):
|
||||
|
||||
create_sample_notification(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
template=sample_email_template,
|
||||
reference='ref1',
|
||||
sent_at=datetime.utcnow(),
|
||||
status='sending')
|
||||
|
||||
response = json.loads(ses_notification_callback())
|
||||
assert process_ses_results(response=response) is None
|
||||
|
||||
|
||||
def test_process_ses_results_does_not_retry_if_errors(notify_db, mocker):
|
||||
mocked = mocker.patch('app.celery.tasks.process_ses_results.retry')
|
||||
response = json.loads(ses_notification_callback())
|
||||
process_ses_results(response=response)
|
||||
assert mocked.call_count == 0
|
||||
|
||||
|
||||
def test_process_ses_results_retry_called(notify_db, mocker):
|
||||
mocker.patch("app.dao.notifications_dao.update_notification_status_by_reference", side_effect=Exception("EXPECTED"))
|
||||
mocked = mocker.patch('app.celery.tasks.process_ses_results.retry')
|
||||
response = json.loads(ses_notification_callback())
|
||||
process_ses_results(response=response)
|
||||
assert mocked.call_count != 0
|
||||
|
||||
|
||||
def ses_notification_callback():
|
||||
return '{\n "Type" : "Notification",\n "MessageId" : "ref1",' \
|
||||
'\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",' \
|
||||
'\n "Message" : "{\\"notificationType\\":\\"Delivery\\",' \
|
||||
'\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",' \
|
||||
'\\"source\\":\\"test@test-domain.com\\",' \
|
||||
'\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",' \
|
||||
'\\"sendingAccountId\\":\\"123456789012\\",' \
|
||||
'\\"messageId\\":\\"ref1\\",' \
|
||||
'\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},' \
|
||||
'\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",' \
|
||||
'\\"processingTimeMillis\\":658,' \
|
||||
'\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],' \
|
||||
'\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",' \
|
||||
'\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",' \
|
||||
'\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",' \
|
||||
'\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUt' \
|
||||
'OowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYL' \
|
||||
'VSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMA' \
|
||||
'PmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",' \
|
||||
'\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750' \
|
||||
'dd426d95ee9390147a5624348ee.pem",' \
|
||||
'\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&S' \
|
||||
'ubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}'
|
||||
|
||||
@@ -373,7 +373,7 @@ def test_send_email_to_provider_should_call_research_mode_task_response_task_if_
|
||||
|
||||
assert not app.aws_ses_client.send_email.called
|
||||
stats_mock.assert_called_once_with(notification)
|
||||
app.delivery.send_to_providers.send_email_response.assert_called_once_with('ses', str(reference), 'john@smith.com')
|
||||
app.delivery.send_to_providers.send_email_response.assert_called_once_with(str(reference), 'john@smith.com')
|
||||
persisted_notification = Notification.query.filter_by(id=notification.id).one()
|
||||
assert persisted_notification.to == 'john@smith.com'
|
||||
assert persisted_notification.template_id == sample_email_template.id
|
||||
|
||||
@@ -6,7 +6,7 @@ from freezegun import freeze_time
|
||||
|
||||
from app import statsd_client
|
||||
from app.dao.notifications_dao import get_notification_by_id
|
||||
from app.notifications.notifications_ses_callback import process_ses_response
|
||||
from app.notifications.notifications_ses_callback import process_ses_response, remove_emails_from_bounce
|
||||
from tests.app.conftest import sample_notification as create_sample_notification
|
||||
|
||||
|
||||
@@ -180,6 +180,27 @@ def test_ses_callback_should_set_status_to_permanent_failure(client,
|
||||
stats_mock.assert_called_once_with(notification)
|
||||
|
||||
|
||||
def test_remove_emails_from_bounce():
|
||||
# an actual bouncedict example
|
||||
message_dict = {
|
||||
'feedbackId': '0102015fc9acfc14-12341234-1234-1234-1234-123412341234-000000',
|
||||
'bounceType': 'Permanent',
|
||||
'reportingMTA': 'dsn; a1-23.smtp-out.eu-west-4.amazonses.com',
|
||||
'remoteMtaIp': '123.123.123.123',
|
||||
'bounceSubType': 'General',
|
||||
'bouncedRecipients': [{
|
||||
'diagnosticCode': "smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try\n550-5.1.1 double-checking the recipient's email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1 https://support.google.com/mail/?p=NoSuchUser x33si3064453edx.66 - gsmtp", # noqa
|
||||
'action': 'failed',
|
||||
'status': '5.1.1',
|
||||
'emailAddress': 'not-real@gmail.com'}],
|
||||
'timestamp': '2017-11-17T11:11:18.098Z'
|
||||
}
|
||||
|
||||
remove_emails_from_bounce(message_dict['bounce'])
|
||||
|
||||
assert 'not-real@gmail.com' not in json.dumps(message_dict)
|
||||
|
||||
|
||||
def ses_notification_callback(ref='ref'):
|
||||
return str.encode(
|
||||
'{\n "Type" : "Notification",\n "MessageId" : "%(ref)s",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"%(ref)s\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' % {'ref': ref} # noqa
|
||||
|
||||
Reference in New Issue
Block a user