update research mode email callbacks to add process-ses-response task to queue

this involved:
* moving that task to callback_tasks to prevent circular imports
* updating the dummy research mode callbacks (with actual SNS messages from the
  ses simulator emails)
* refactoring tests
This commit is contained in:
Leo Hemsted
2017-11-17 13:41:45 +00:00
parent 0e477b7715
commit d2154451e5
11 changed files with 158 additions and 106 deletions

View File

@@ -0,0 +1,58 @@
import json
from datetime import datetime
from app.celery.callback_tasks import process_ses_results
from tests.app.db import create_notification
def test_process_ses_results(sample_email_template):
create_notification(
sample_email_template,
reference='ref1',
sent_at=datetime.utcnow(),
status='sending')
response = json.loads(ses_notification_callback())
assert process_ses_results(response=response) is None
def test_process_ses_results_does_not_retry_if_errors(notify_db, mocker):
mocked = mocker.patch('app.celery.callback_tasks.process_ses_results.retry')
response = json.loads(ses_notification_callback())
process_ses_results(response=response)
assert mocked.call_count == 0
def test_process_ses_results_retry_called(notify_db, mocker):
mocker.patch("app.dao.notifications_dao.update_notification_status_by_reference", side_effect=Exception("EXPECTED"))
mocked = mocker.patch('app.celery.callback_tasks.process_ses_results.retry')
response = json.loads(ses_notification_callback())
process_ses_results(response=response)
assert mocked.call_count != 0
def ses_notification_callback():
return '{\n "Type" : "Notification",\n "MessageId" : "ref1",' \
'\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",' \
'\n "Message" : "{\\"notificationType\\":\\"Delivery\\",' \
'\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",' \
'\\"source\\":\\"test@test-domain.com\\",' \
'\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",' \
'\\"sendingAccountId\\":\\"123456789012\\",' \
'\\"messageId\\":\\"ref1\\",' \
'\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},' \
'\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",' \
'\\"processingTimeMillis\\":658,' \
'\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],' \
'\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",' \
'\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",' \
'\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",' \
'\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUt' \
'OowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYL' \
'VSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMA' \
'PmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",' \
'\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750' \
'dd426d95ee9390147a5624348ee.pem",' \
'\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&S' \
'ubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}'

View File

@@ -12,8 +12,6 @@ def test_should_have_decorated_tasks_functions():
def test_should_call_send_sms_to_provider_from_deliver_sms_task(
notify_db,
notify_db_session,
sample_notification,
mocker):
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider')
@@ -23,7 +21,6 @@ def test_should_call_send_sms_to_provider_from_deliver_sms_task(
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task(
notify_db,
notify_db_session,
mocker):
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider')
@@ -37,8 +34,6 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task
def test_should_call_send_email_to_provider_from_deliver_email_task(
notify_db,
notify_db_session,
sample_notification,
mocker):
mocker.patch('app.delivery.send_to_providers.send_email_to_provider')

View File

@@ -1,5 +1,10 @@
import uuid
from unittest.mock import ANY
import pytest
from flask import json
from app.config import QueueNames
from app.celery.research_mode_tasks import (
send_sms_response,
send_email_response,
@@ -42,16 +47,14 @@ def test_make_firetext_callback(notify_api, rmock, phone_number):
assert 'mobile={}'.format(phone_number) in rmock.request_history[0].text
def test_make_ses_callback(notify_api, rmock):
endpoint = "http://localhost:6011/notifications/email/ses"
rmock.request(
"POST",
endpoint,
json={"status": "success"},
status_code=200)
send_email_response("ses", "1234", "test@test.com")
def test_make_ses_callback(notify_api, mocker):
mock_task = mocker.patch('app.celery.research_mode_tasks.process_ses_results')
some_ref = str(uuid.uuid4())
assert rmock.called
send_email_response(reference=some_ref, to="test@test.com")
mock_task.apply_async.assert_called_once_with(ANY, queue=QueueNames.RESEARCH_MODE)
assert mock_task.apply_async.call_args[0][0][0] == ses_notification_callback(some_ref)
@pytest.mark.parametrize("phone_number", ["07700900001", "+447700900001", "7700900001", "+44 7700900001",
@@ -101,15 +104,3 @@ def test_failure_firetext_callback(phone_number):
'time': '2016-03-10 14:17:00',
'reference': '1234'
}
def test_delivered_ses_callback():
assert ses_notification_callback("my-reference") == '{ "Type" : "Notification", "MessageId" : "my-reference", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"my-reference\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' # noqa
def test_ses_hard_bounce_callback():
assert ses_hard_bounce_callback("my-reference") == '{ "Type" : "Notification", "MessageId" : "my-reference", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Permanent\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"my-reference\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' # noqa
def ses_soft_bounce_callback():
assert ses_soft_bounce_callback("my-reference") == '{ "Type" : "Notification", "MessageId" : "my-reference", "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing", "Message" : "{\\"notificationType\\":\\"Bounce\\",\\"bounce\\":{\\"bounceType\\":\\"Undetermined\\",\\"bounceSubType\\":\\"General\\"}, \\"mail\\":{\\"messageId\\":\\"%s\\",\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}", "Timestamp" : "2016-03-14T12:35:26.665Z", "SignatureVersion" : "1", "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==", "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem", "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"}' # noqa

View File

@@ -28,8 +28,7 @@ from app.celery.tasks import (
process_incomplete_jobs,
get_template_class,
s3,
send_inbound_sms_to_service,
process_ses_results)
send_inbound_sms_to_service)
from app.config import QueueNames
from app.dao import jobs_dao, services_dao
from app.models import (
@@ -1403,58 +1402,3 @@ def test_process_incomplete_job_letter(mocker, sample_letter_template):
assert completed_job.job_status == JOB_STATUS_FINISHED
assert mock_letter_saver.call_count == 8
def test_process_ses_results(notify_db, notify_db_session, sample_email_template):
create_sample_notification(
notify_db,
notify_db_session,
template=sample_email_template,
reference='ref1',
sent_at=datetime.utcnow(),
status='sending')
response = json.loads(ses_notification_callback())
assert process_ses_results(response=response) is None
def test_process_ses_results_does_not_retry_if_errors(notify_db, mocker):
mocked = mocker.patch('app.celery.tasks.process_ses_results.retry')
response = json.loads(ses_notification_callback())
process_ses_results(response=response)
assert mocked.call_count == 0
def test_process_ses_results_retry_called(notify_db, mocker):
mocker.patch("app.dao.notifications_dao.update_notification_status_by_reference", side_effect=Exception("EXPECTED"))
mocked = mocker.patch('app.celery.tasks.process_ses_results.retry')
response = json.loads(ses_notification_callback())
process_ses_results(response=response)
assert mocked.call_count != 0
def ses_notification_callback():
return '{\n "Type" : "Notification",\n "MessageId" : "ref1",' \
'\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",' \
'\n "Message" : "{\\"notificationType\\":\\"Delivery\\",' \
'\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",' \
'\\"source\\":\\"test@test-domain.com\\",' \
'\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",' \
'\\"sendingAccountId\\":\\"123456789012\\",' \
'\\"messageId\\":\\"ref1\\",' \
'\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},' \
'\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",' \
'\\"processingTimeMillis\\":658,' \
'\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],' \
'\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",' \
'\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",' \
'\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",' \
'\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUt' \
'OowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYL' \
'VSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMA' \
'PmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",' \
'\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750' \
'dd426d95ee9390147a5624348ee.pem",' \
'\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&S' \
'ubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}'

View File

@@ -373,7 +373,7 @@ def test_send_email_to_provider_should_call_research_mode_task_response_task_if_
assert not app.aws_ses_client.send_email.called
stats_mock.assert_called_once_with(notification)
app.delivery.send_to_providers.send_email_response.assert_called_once_with('ses', str(reference), 'john@smith.com')
app.delivery.send_to_providers.send_email_response.assert_called_once_with(str(reference), 'john@smith.com')
persisted_notification = Notification.query.filter_by(id=notification.id).one()
assert persisted_notification.to == 'john@smith.com'
assert persisted_notification.template_id == sample_email_template.id

View File

@@ -6,7 +6,7 @@ from freezegun import freeze_time
from app import statsd_client
from app.dao.notifications_dao import get_notification_by_id
from app.notifications.notifications_ses_callback import process_ses_response
from app.notifications.notifications_ses_callback import process_ses_response, remove_emails_from_bounce
from tests.app.conftest import sample_notification as create_sample_notification
@@ -180,6 +180,27 @@ def test_ses_callback_should_set_status_to_permanent_failure(client,
stats_mock.assert_called_once_with(notification)
def test_remove_emails_from_bounce():
# an actual bouncedict example
message_dict = {
'feedbackId': '0102015fc9acfc14-12341234-1234-1234-1234-123412341234-000000',
'bounceType': 'Permanent',
'reportingMTA': 'dsn; a1-23.smtp-out.eu-west-4.amazonses.com',
'remoteMtaIp': '123.123.123.123',
'bounceSubType': 'General',
'bouncedRecipients': [{
'diagnosticCode': "smtp; 550-5.1.1 The email account that you tried to reach does not exist. Please try\n550-5.1.1 double-checking the recipient's email address for typos or\n550-5.1.1 unnecessary spaces. Learn more at\n550 5.1.1 https://support.google.com/mail/?p=NoSuchUser x33si3064453edx.66 - gsmtp", # noqa
'action': 'failed',
'status': '5.1.1',
'emailAddress': 'not-real@gmail.com'}],
'timestamp': '2017-11-17T11:11:18.098Z'
}
remove_emails_from_bounce(message_dict['bounce'])
assert 'not-real@gmail.com' not in json.dumps(message_dict)
def ses_notification_callback(ref='ref'):
return str.encode(
'{\n "Type" : "Notification",\n "MessageId" : "%(ref)s",\n "TopicArn" : "arn:aws:sns:eu-west-1:123456789012:testing",\n "Message" : "{\\"notificationType\\":\\"Delivery\\",\\"mail\\":{\\"timestamp\\":\\"2016-03-14T12:35:25.909Z\\",\\"source\\":\\"test@test-domain.com\\",\\"sourceArn\\":\\"arn:aws:ses:eu-west-1:123456789012:identity/testing-notify\\",\\"sendingAccountId\\":\\"123456789012\\",\\"messageId\\":\\"%(ref)s\\",\\"destination\\":[\\"testing@digital.cabinet-office.gov.uk\\"]},\\"delivery\\":{\\"timestamp\\":\\"2016-03-14T12:35:26.567Z\\",\\"processingTimeMillis\\":658,\\"recipients\\":[\\"testing@digital.cabinet-office.gov.uk\\"],\\"smtpResponse\\":\\"250 2.0.0 OK 1457958926 uo5si26480932wjc.221 - gsmtp\\",\\"reportingMTA\\":\\"a6-238.smtp-out.eu-west-1.amazonses.com\\"}}",\n "Timestamp" : "2016-03-14T12:35:26.665Z",\n "SignatureVersion" : "1",\n "Signature" : "X8d7eTAOZ6wlnrdVVPYanrAlsX0SMPfOzhoTEBnQqYkrNWTqQY91C0f3bxtPdUhUtOowyPAOkTQ4KnZuzphfhVb2p1MyVYMxNKcBFB05/qaCX99+92fjw4x9LeUOwyGwMv5F0Vkfi5qZCcEw69uVrhYLVSTFTrzi/yCtru+yFULMQ6UhbY09GwiP6hjxZMVr8aROQy5lLHglqQzOuSZ4KeD85JjifHdKzlx8jjQ+uj+FLzHXPMAPmPU1JK9kpoHZ1oPshAFgPDpphJe+HwcJ8ezmk+3AEUr3wWli3xF+49y8Z2anASSVp6YI2YP95UT8Rlh3qT3T+V9V8rbSVislxA==",\n "SigningCertURL" : "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem",\n "UnsubscribeURL" : "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:302763885840:preview-emails:d6aad3ef-83d6-4cf3-a470-54e2e75916da"\n}' % {'ref': ref} # noqa