Update notification status by message reference

- SES sends a reference to allow us to identify the notification
- use this to update status

If source of email is one of our internal emails (invites or validations) - don't try and update a notification.

app/dao/notifications_dao.py

@@ -80,20 +80,9 @@ def update_notification_status_by_id(notification_id, status):
     return count
 
 
-def update_notification_status_by_to(to, status):
-    count = db.session.query(Notification).filter_by(
-        to=to
-    ).update({
-        Notification.status: status,
-        Notification.updated_at: datetime.utcnow()
-    })
-    db.session.commit()
-    return count
-
-
 def update_notification_status_by_reference(reference, status):
     count = db.session.query(Notification).filter_by(
-        refernce=reference
+        reference=reference
     ).update({
         Notification.status: status,
         Notification.updated_at: datetime.utcnow()

app/notifications/rest.py

@@ -67,9 +67,17 @@ def process_ses_response():
             ), 400
 
         try:
-            recipients = ses_request['Message']['mail']['destination']
+            source = ses_request['Message']['mail']['source']
+            if is_not_a_notification(ses_request['Message']['mail']['source']):
+                current_app.logger.info(
+                    "SES callback for notify success:. source {} status {}".format(source, status['notify_status'])
+                )
+                return jsonify(
+                    result="success", message="SES callback succeeded"
+                ), 200
 
-            if notifications_dao.update_notification_status_by_to(recipients[0], status['notify_status']) == 0:
+            reference = ses_request['Message']['mail']['messageId']
+            if notifications_dao.update_notification_status_by_reference(reference, status['notify_status']) == 0:
                 current_app.logger.info(
                     "SES callback failed: notification not found. Status {}".format(status['notify_status'])
                 )
@@ -83,10 +91,10 @@ def process_ses_response():
 
         except KeyError:
             current_app.logger.error(
-                "SES callback failed: destination missing"
+                "SES callback failed: messageId missing"
             )
             return jsonify(
-                result="error", message="SES callback failed: destination missing"
+                result="error", message="SES callback failed: messageId missing"
             ), 400
 
     except ValueError as ex:
@@ -98,6 +106,18 @@ def process_ses_response():
         ), 400
 
 
+def is_not_a_notification(source):
+    invite_email = "{}@{}".format(
+        current_app.config['INVITATION_EMAIL_FROM'],
+        current_app.config['NOTIFY_EMAIL_DOMAIN']
+    )
+    if current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'] == source:
+        return True
+    if invite_email == source:
+        return True
+    return False
+
+
 @notifications.route('/notifications/sms/firetext', methods=['POST'])
 def process_firetext_response():
     if 'status' not in request.form:

tests/app/conftest.py

@@ -282,6 +282,7 @@ def sample_notification(notify_db,
                         job=None,
                         to_field=None,
                         status='sent',
+                        reference=None,
                         created_at=datetime.utcnow()):
     if service is None:
         service = sample_service(notify_db, notify_db_session)
@@ -305,6 +306,7 @@ def sample_notification(notify_db,
         'service': service,
         'template': template,
         'status': status,
+        'reference': reference,
         'created_at': created_at
     }
     notification = Notification(**data)

tests/app/dao/test_notification_dao.py

@@ -18,8 +18,8 @@ from app.dao.notifications_dao import (
     delete_failed_notifications_created_more_than_a_week_ago,
     dao_get_notification_statistics_for_service_and_day,
     update_notification_status_by_id,
-    update_notification_status_by_to,
-    update_notification_reference_by_id
+    update_notification_reference_by_id,
+    update_notification_status_by_reference
 )
 from tests.app.conftest import sample_job
 from tests.app.conftest import sample_notification
@@ -32,6 +32,13 @@ def test_should_by_able_to_update_reference_by_id(sample_notification):
     assert Notification.query.get(sample_notification.id).reference == 'reference'
 
 
+def test_should_by_able_to_update_status_by_reference(sample_notification):
+    assert Notification.query.get(sample_notification.id).status == "sent"
+    update_notification_reference_by_id(sample_notification.id, 'reference')
+    update_notification_status_by_reference('reference', 'delivered')
+    assert Notification.query.get(sample_notification.id).status == 'delivered'
+
+
 def test_should_by_able_to_update_status_by_id(sample_notification):
     assert Notification.query.get(sample_notification.id).status == 'sent'
     count = update_notification_status_by_id(sample_notification.id, 'delivered')
@@ -43,15 +50,8 @@ def test_should_return_zero_count_if_no_notification_with_id():
     assert update_notification_status_by_id(str(uuid.uuid4()), 'delivered') == 0
 
 
-def test_should_return_zero_count_if_no_notification_with_to():
-    assert update_notification_status_by_to('something', 'delivered') == 0
-
-
-def test_should_by_able_to_update_status_by_to(sample_notification):
-    assert Notification.query.get(sample_notification.id).status == 'sent'
-    count = update_notification_status_by_to(sample_notification.to, 'delivered')
-    assert count == 1
-    assert Notification.query.get(sample_notification.id).status == 'delivered'
+def test_should_return_zero_count_if_no_notification_with_reference():
+    assert update_notification_status_by_reference('something', 'delivered') == 0
 
 
 def test_should_be_able_to_get_statistics_for_a_service(sample_template):

tests/app/notifications/test_rest.py

@@ -1089,11 +1089,11 @@ def test_ses_callback_should_fail_if_invalid_notification_type(notify_api):
             assert json_resp['message'] == 'SES callback failed: status Unknown not found'
 
 
-def test_ses_callback_should_fail_if_missing_destination(notify_api):
+def test_ses_callback_should_fail_if_missing_message_id(notify_api):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
             ses_response = json.loads(load_example_ses('ses_response'))
-            del(ses_response['Message']['mail']['destination'])
+            del(ses_response['Message']['mail']['messageId'])
 
             response = client.post(
                 path='/notifications/email/ses',
@@ -1103,14 +1103,14 @@ def test_ses_callback_should_fail_if_missing_destination(notify_api):
             json_resp = json.loads(response.get_data(as_text=True))
             assert response.status_code == 400
             assert json_resp['result'] == 'error'
-            assert json_resp['message'] == 'SES callback failed: destination missing'
+            assert json_resp['message'] == 'SES callback failed: messageId missing'
 
 
 def test_ses_callback_should_fail_if_notification_cannot_be_found(notify_db, notify_db_session, notify_api):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
             ses_response = json.loads(load_example_ses('ses_response'))
-            ses_response['Message']['mail']['destination'] = ['wont find this']
+            ses_response['Message']['mail']['messageId'] = 'wont find this'
 
             response = client.post(
                 path='/notifications/email/ses',
@@ -1118,19 +1118,66 @@ def test_ses_callback_should_fail_if_notification_cannot_be_found(notify_db, not
                 headers=[('Content-Type', 'text/plain; charset=UTF-8')]
             )
             json_resp = json.loads(response.get_data(as_text=True))
+            print(json_resp)
             assert response.status_code == 404
             assert json_resp['result'] == 'error'
             assert json_resp['message'] == 'SES callback failed: notification not found. Status delivered'
 
 
-def test_ses_callback_should_update_notification_status(notify_api, sample_notification):
+def test_ses_callback_should_update_notification_status(notify_api, notify_db, notify_db_session):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
 
-            assert get_notification_by_id(sample_notification.id).status == 'sent'
+            notification = sample_notification(notify_db, notify_db_session, reference='ref')
+
+            assert get_notification_by_id(notification.id).status == 'sent'
 
             ses_response = json.loads(load_example_ses('ses_response'))
-            ses_response['Message']['mail']['destination'] = [sample_notification.to]
+            ses_response['Message']['mail']['messageId'] = 'ref'
+
+            response = client.post(
+                path='/notifications/email/ses',
+                data=json.dumps(ses_response),
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert json_resp['result'] == 'success'
+            assert json_resp['message'] == 'SES callback succeeded'
+            assert get_notification_by_id(notification.id).status == 'delivered'
+
+
+def test_should_handle_invite_email_callbacks(notify_api, notify_db, notify_db_session):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+
+            notify_api.config['INVITATION_EMAIL_FROM'] = 'test-invite'
+            notify_api.config['NOTIFY_EMAIL_DOMAIN'] = 'test-domain.com'
+
+            ses_response = json.loads(load_example_ses('ses_response'))
+            ses_response['Message']['mail']['messageId'] = 'ref'
+            ses_response['Message']['mail']['source'] = 'test-invite@test-domain.com'
+
+            response = client.post(
+                path='/notifications/email/ses',
+                data=json.dumps(ses_response),
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert json_resp['result'] == 'success'
+            assert json_resp['message'] == 'SES callback succeeded'
+
+
+def test_should_handle_validation_code_callbacks(notify_api, notify_db, notify_db_session):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+
+            notify_api.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'] = 'valid-code@test.com'
+
+            ses_response = json.loads(load_example_ses('ses_response'))
+            ses_response['Message']['mail']['messageId'] = 'ref'
+            ses_response['Message']['mail']['source'] = 'valid-code@test.com'
 
             response = client.post(
                 path='/notifications/email/ses',
@@ -1141,4 +1188,3 @@ def test_ses_callback_should_update_notification_status(notify_api, sample_notif
             assert response.status_code == 200
             assert json_resp['result'] == 'success'
             assert json_resp['message'] == 'SES callback succeeded'
-            assert get_notification_by_id(sample_notification.id).status == 'delivered'