Merge pull request #146 from alphagov/client-callbacks

Client callbacks

app/__init__.py

@@ -71,7 +71,12 @@ def create_app():
 def init_app(app):
     @app.before_request
     def required_authentication():
-        if request.path != url_for('status.show_status'):
+        no_auth_req = [
+            url_for('status.show_status'),
+            url_for('notifications.process_ses_response'),
+            url_for('notifications.process_firetext_response')
+        ]
+        if request.path not in no_auth_req:
             from app.authentication import auth
             error = auth.requires_auth()
             if error:

app/celery/tasks.py

@@ -9,7 +9,8 @@ from app.dao.notifications_dao import (
     dao_update_notification,
     delete_failed_notifications_created_more_than_a_week_ago,
     delete_successful_notifications_created_more_than_a_day_ago,
-    dao_get_notification_statistics_for_service_and_day
+    dao_get_notification_statistics_for_service_and_day,
+    update_notification_reference_by_id
 )
 from app.dao.jobs_dao import dao_update_job, dao_get_job_by_id
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
@@ -203,10 +204,14 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
                 )
 
                 client.send_sms(
-                    notification['to'],
-                    template.replaced
+                    to=notification['to'],
+                    content=template.replaced,
+                    reference=str(notification_id)
                 )
             except FiretextClientException as e:
+                current_app.logger.error(
+                    "SMS notification {} failed".format(notification_id)
+                )
                 current_app.logger.exception(e)
                 notification_db_object.status = 'failed'
                 dao_update_notification(notification_db_object)
@@ -271,14 +276,15 @@ def send_email(service_id, notification_id, subject, from_address, encrypted_not
                     values=notification.get('personalisation', {})
                 )
 
-                client.send_email(
+                reference = client.send_email(
                     from_address,
                     notification['to'],
                     subject,
                     template.replaced
                 )
+                update_notification_reference_by_id(notification_id, reference)
             except AwsSesClientException as e:
-                current_app.logger.debug(e)
+                current_app.logger.exception(e)
                 notification_db_object.status = 'failed'
                 dao_update_notification(notification_db_object)
 
@@ -293,7 +299,9 @@ def send_email(service_id, notification_id, subject, from_address, encrypted_not
 def send_sms_code(encrypted_verification):
     verification_message = encryption.decrypt(encrypted_verification)
     try:
-        firetext_client.send_sms(verification_message['to'], verification_message['secret_code'])
+        firetext_client.send_sms(
+            verification_message['to'], verification_message['secret_code'], 'send-sms-code'
+        )
     except FiretextClientException as e:
         current_app.logger.exception(e)
 

app/clients/email/aws_ses.py

@@ -3,6 +3,21 @@ from flask import current_app
 from monotonic import monotonic
 from app.clients.email import (EmailClientException, EmailClient)
 
+ses_response_status = {
+    'Bounce': {
+        "success": False,
+        "notify_status": 'bounce'
+    },
+    'Delivery': {
+        "success": True,
+        "notify_status": 'delivered'
+    },
+    'Complaint': {
+        "success": False,
+        "notify_status": 'complaint'
+    }
+}
+
 
 class AwsSesClientException(EmailClientException):
     pass

app/clients/sms/firetext.py

@@ -9,9 +9,32 @@ from requests import request, RequestException, HTTPError
 
 logger = logging.getLogger(__name__)
 
+firetext_response_status = {
+    '0': {
+        "firetext_message": 'delivered',
+        "success": True,
+        "notify_status": 'delivered'
+    },
+    '1': {
+        "firetext_message": 'declined',
+        "success": False,
+        "notify_status": 'failed'
+    },
+    '2': {
+        "firetext_message": 'Undelivered (Pending with Network)',
+        "success": False,
+        "notify_status": 'sent'
+    }
+}
+
 
 class FiretextClientException(SmsClientException):
-    pass
+    def __init__(self, response):
+        self.code = response['code']
+        self.description = response['description']
+
+    def __str__(self):
+        return "Code {} description {}".format(self.code, self.description)
 
 
 class FiretextClient(SmsClient):
@@ -28,22 +51,26 @@ class FiretextClient(SmsClient):
     def get_name(self):
         return self.name
 
-    def send_sms(self, to, content):
+    def send_sms(self, to, content, reference):
 
         data = {
             "apiKey": self.api_key,
             "from": self.from_number,
             "to": to.replace('+', ''),
-            "message": content
+            "message": content,
+            "reference": reference
         }
 
+        start_time = monotonic()
         try:
-            start_time = monotonic()
             response = request(
                 "POST",
-                "https://www.firetext.co.uk/api/sendsms",
+                "https://www.firetext.co.uk/api/sendsms/json",
                 data=data
             )
+            firetext_response = response.json()
+            if firetext_response['code'] != 0:
+                raise FiretextClientException(firetext_response)
             response.raise_for_status()
         except RequestException as e:
             api_error = HTTPError.create(e)

app/dao/notifications_dao.py

@@ -21,7 +21,12 @@ def dao_get_notification_statistics_for_service_and_day(service_id, day):
 def dao_create_notification(notification, notification_type):
     try:
         if notification.job_id:
-            update_job_sent_count(notification)
+            db.session.query(Job).filter_by(
+                id=notification.job_id
+            ).update({
+                Job.notifications_sent: Job.notifications_sent + 1,
+                Job.updated_at: datetime.utcnow()
+            })
 
         if update_notification_stats(notification, notification_type) == 0:
             stats = NotificationStatistics(
@@ -54,20 +59,42 @@ def update_notification_stats(notification, notification_type):
     ).update(update)
 
 
-def update_job_sent_count(notification):
-    db.session.query(Job).filter_by(
-        id=notification.job_id
-    ).update({
-        Job.notifications_sent: Job.notifications_sent + 1,
-        Job.updated_at: datetime.utcnow()
-    })
-
-
 def dao_update_notification(notification):
+    notification.updated_at = datetime.utcnow()
     db.session.add(notification)
     db.session.commit()
 
 
+def update_notification_status_by_id(notification_id, status):
+    count = db.session.query(Notification).filter_by(
+        id=notification_id
+    ).update({
+        Notification.status: status
+    })
+    db.session.commit()
+    return count
+
+
+def update_notification_status_by_reference(reference, status):
+    count = db.session.query(Notification).filter_by(
+        reference=reference
+    ).update({
+        Notification.status: status
+    })
+    db.session.commit()
+    return count
+
+
+def update_notification_reference_by_id(id, reference):
+    count = db.session.query(Notification).filter_by(
+        id=id
+    ).update({
+        Notification.reference: reference
+    })
+    db.session.commit()
+    return count
+
+
 def get_notification_for_job(service_id, job_id, notification_id):
     return Notification.query.filter_by(service_id=service_id, job_id=job_id, id=notification_id).one()
 
@@ -86,6 +113,10 @@ def get_notification(service_id, notification_id):
     return Notification.query.filter_by(service_id=service_id, id=notification_id).one()
 
 
+def get_notification_by_id(notification_id):
+    return Notification.query.filter_by(id=notification_id).first()
+
+
 def get_notifications_for_service(service_id, page=1):
     query = Notification.query.filter_by(service_id=service_id).order_by(desc(Notification.created_at)).paginate(
         page=page,

app/models.py

@@ -231,7 +231,7 @@ class VerifyCode(db.Model):
         return check_hash(cde, self._code)
 
 
-NOTIFICATION_STATUS_TYPES = ['sent', 'failed']
+NOTIFICATION_STATUS_TYPES = ['sent', 'delivered', 'failed', 'complaint', 'bounce']
 
 
 class Notification(db.Model):
@@ -265,6 +265,7 @@ class Notification(db.Model):
         onupdate=datetime.datetime.now)
     status = db.Column(
         db.Enum(*NOTIFICATION_STATUS_TYPES, name='notification_status_types'), nullable=False, default='sent')
+    reference = db.Column(db.String, nullable=True, index=True)
 
 
 INVITED_USER_STATUS_TYPES = ['pending', 'accepted', 'cancelled']

app/notifications/rest.py

@@ -1,15 +1,18 @@
 from datetime import datetime
+import uuid
 
 from flask import (
     Blueprint,
     jsonify,
     request,
     current_app,
-    url_for
+    url_for,
+    json
 )
 
 from utils.template import Template
-
+from app.clients.sms.firetext import firetext_response_status
+from app.clients.email.aws_ses import ses_response_status
 from app import api_user, encryption, create_uuid, DATETIME_FORMAT, DATE_FORMAT
 from app.authentication.auth import require_admin
 from app.dao import (
@@ -23,7 +26,6 @@ from app.schemas import (
     notification_status_schema
 )
 from app.celery.tasks import send_sms, send_email
-from sqlalchemy.orm.exc import NoResultFound
 
 notifications = Blueprint('notifications', __name__)
 
@@ -32,6 +34,150 @@ from app.errors import register_errors
 register_errors(notifications)
 
 
+@notifications.route('/notifications/email/ses', methods=['POST'])
+def process_ses_response():
+    try:
+        ses_request = json.loads(request.data)
+
+        if 'Message' not in ses_request:
+            current_app.logger.error(
+                "SES callback failed: message missing"
+            )
+            return jsonify(
+                result="error", message="SES callback failed: message missing"
+            ), 400
+
+        if 'notificationType' not in ses_request['Message']:
+            current_app.logger.error(
+                "SES callback failed: notificationType missing"
+            )
+            return jsonify(
+                result="error", message="SES callback failed: notificationType missing"
+            ), 400
+
+        status = ses_response_status.get(ses_request['Message']['notificationType'], None)
+        if not status:
+            current_app.logger.info(
+                "SES callback failed: status {} not found.".format(status)
+            )
+            return jsonify(
+                result="error",
+                message="SES callback failed: status {} not found".format(ses_request['Message']['notificationType'])
+            ), 400
+
+        try:
+            source = ses_request['Message']['mail']['source']
+            if is_not_a_notification(ses_request['Message']['mail']['source']):
+                current_app.logger.info(
+                    "SES callback for notify success:. source {} status {}".format(source, status['notify_status'])
+                )
+                return jsonify(
+                    result="success", message="SES callback succeeded"
+                ), 200
+
+            reference = ses_request['Message']['mail']['messageId']
+            if notifications_dao.update_notification_status_by_reference(reference, status['notify_status']) == 0:
+                current_app.logger.info(
+                    "SES callback failed: notification not found. Status {}".format(status['notify_status'])
+                )
+                return jsonify(
+                    result="error",
+                    message="SES callback failed: notification not found. Status {}".format(status['notify_status'])
+                ), 404
+            return jsonify(
+                result="success", message="SES callback succeeded"
+            ), 200
+
+        except KeyError:
+            current_app.logger.error(
+                "SES callback failed: messageId missing"
+            )
+            return jsonify(
+                result="error", message="SES callback failed: messageId missing"
+            ), 400
+
+    except ValueError as ex:
+        current_app.logger.exception(
+            "SES callback failed: invalid json {}".format(ex)
+        )
+        return jsonify(
+            result="error", message="SES callback failed: invalid json"
+        ), 400
+
+
+def is_not_a_notification(source):
+    invite_email = "{}@{}".format(
+        current_app.config['INVITATION_EMAIL_FROM'],
+        current_app.config['NOTIFY_EMAIL_DOMAIN']
+    )
+    if current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'] == source:
+        return True
+    if invite_email == source:
+        return True
+    return False
+
+
+@notifications.route('/notifications/sms/firetext', methods=['POST'])
+def process_firetext_response():
+    if 'status' not in request.form:
+        current_app.logger.info(
+            "Firetext callback failed: status missing"
+        )
+        return jsonify(result="error", message="Firetext callback failed: status missing"), 400
+
+    if len(request.form.get('reference', '')) <= 0:
+        current_app.logger.info(
+            "Firetext callback with no reference"
+        )
+        return jsonify(result="error", message="Firetext callback failed: reference missing"), 400
+
+    reference = request.form['reference']
+    status = request.form['status']
+
+    if reference == 'send-sms-code':
+        return jsonify(result="success", message="Firetext callback succeeded: send-sms-code"), 200
+
+    try:
+        uuid.UUID(reference, version=4)
+    except ValueError:
+        current_app.logger.info(
+            "Firetext callback with invalid reference {}".format(reference)
+        )
+        return jsonify(
+            result="error", message="Firetext callback with invalid reference {}".format(reference)
+        ), 400
+
+    notification_status = firetext_response_status.get(status, None)
+    if not notification_status:
+        current_app.logger.info(
+            "Firetext callback failed: status {} not found.".format(status)
+        )
+        return jsonify(result="error", message="Firetext callback failed: status {} not found.".format(status)), 400
+
+    if notifications_dao.update_notification_status_by_id(reference, notification_status['notify_status']) == 0:
+        current_app.logger.info(
+            "Firetext callback failed: notification {} not found. Status {}".format(reference, status)
+        )
+        return jsonify(
+            result="error",
+            message="Firetext callback failed: notification {} not found. Status {}".format(
+                reference,
+                notification_status['firetext_message']
+            )
+        ), 404
+
+    if not notification_status['success']:
+        current_app.logger.info(
+            "Firetext delivery failed: notification {} has error found. Status {}".format(
+                reference,
+                firetext_response_status[status]['firetext_message']
+            )
+        )
+    return jsonify(
+        result="success", message="Firetext callback succeeded. reference {} updated".format(reference)
+    ), 200
+
+
 @notifications.route('/notifications/<uuid:notification_id>', methods=['GET'])
 def get_notifications(notification_id):
     notification = notifications_dao.get_notification(api_user['client'], notification_id)

migrations/versions/0039_more_notification_states.py

@@ -0,0 +1,31 @@
+"""empty message
+
+Revision ID: 0039_more_notification_states
+Revises: 0038_reduce_limits
+Create Date: 2016-03-08 11:16:25.659463
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0039_more_notification_states'
+down_revision = '0038_reduce_limits'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+def upgrade():
+    op.drop_column('notifications', 'status')
+    op.execute('DROP TYPE notification_status_types')
+    notification_status_types = sa.Enum('sent', 'delivered', 'failed', 'complaint', 'bounce', name='notification_status_types')
+    notification_status_types.create(op.get_bind())
+    op.add_column('notifications', sa.Column('status', notification_status_types, nullable=True))
+    op.get_bind()
+    op.execute("update notifications set status='delivered'")
+    op.alter_column('notifications', 'status', nullable=False)
+
+
+def downgrade():
+    op.drop_column('notifications', 'status')
+    op.execute('DROP TYPE notification_status_types')

migrations/versions/0040_add_reference.py

@@ -0,0 +1,24 @@
+"""empty message
+
+Revision ID: 0040_add_reference
+Revises: 0039_more_notification_states
+Create Date: 2016-03-11 09:15:57.900192
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0040_add_reference'
+down_revision = '0039_more_notification_states'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('notifications', sa.Column('reference', sa.String(), nullable=True))
+    op.create_index(op.f('ix_notifications_reference'), 'notifications', ['reference'], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f('ix_notifications_reference'), table_name='notifications')
+    op.drop_column('notifications', 'reference')

test_ses_responses/ses_response.json

@@ -0,0 +1,32 @@
+{
+  "MessageId": "35efe472-ba36-5808-89bb-ab2925158b13",
+  "UnsubscribeURL": "https://sns.eu-west-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:eu-west-1:123456789012:preview-emails:12345678-1234-1234-1234-123456789012",
+  "SignatureVersion": "1",
+  "TopicArn": "arn:aws:sns:eu-west-1:123456789012:testing",
+  "Timestamp": "2016-03-10T16:12:19.876Z",
+  "Type": "Notification",
+  "Signature": "sig",
+  "Message": {
+    "notificationType": "Delivery",
+    "mail": {
+      "timestamp": "2016-03-10T16:12:19.016Z",
+      "source": "invites@testing-notify.com",
+      "sourceArn": "arn:aws:ses:eu-west-1:123456789012:identity/testing-notify",
+      "sendingAccountId": "123456789012",
+      "messageId": "01020153614cd6c8-2ec4bd32-7ddc-4344-811e-65d05519251f-000000",
+      "destination": [
+        "testing@digital.cabinet-office.gov.uk"
+      ]
+    },
+    "delivery": {
+      "timestamp": "2016-03-10T16:12:19.751Z",
+      "processingTimeMillis": 735,
+      "recipients": [
+        "testing@digital.cabinet-office.gov.uk"
+      ],
+      "smtpResponse": "250 2.0.0 OK 1457626339 u62si5491824wme.91 - gsmtp",
+      "reportingMTA": "a6-15.smtp-out.eu-west-1.amazonses.com"
+    }
+  },
+  "SigningCertURL": "https://sns.eu-west-1.amazonaws.com/SimpleNotificationService-bb750dd426d95ee9390147a5624348ee.pem"
+}

tests/app/__init__.py

@@ -5,3 +5,9 @@ def load_example_csv(file):
     file_path = os.path.join("test_csv_files", "{}.csv".format(file))
     with open(file_path) as f:
         return f.read()
+
+
+def load_example_ses(file):
+    file_path = os.path.join("test_ses_responses", "{}.json".format(file))
+    with open(file_path) as f:
+        return f.read()

tests/app/celery/test_tasks.py

@@ -35,6 +35,10 @@ from tests.app.conftest import (
 )
 
 
+def firetext_error():
+    return {'code': 0, 'description': 'error'}
+
+
 def test_should_call_delete_successful_notifications_in_task(notify_api, mocker):
     mocker.patch('app.celery.tasks.delete_successful_notifications_created_more_than_a_day_ago')
     delete_successful_notifications()
@@ -260,7 +264,11 @@ def test_should_send_template_to_correct_sms_provider_and_persist(sample_templat
         now.strftime(DATETIME_FORMAT)
     )
 
-    firetext_client.send_sms.assert_called_once_with("+441234123123", "Sample service: Hello Jo")
+    firetext_client.send_sms.assert_called_once_with(
+        to="+441234123123",
+        content="Sample service: Hello Jo",
+        reference=str(notification_id)
+    )
     persisted_notification = notifications_dao.get_notification(
         sample_template_with_placeholders.service_id, notification_id
     )
@@ -292,7 +300,11 @@ def test_should_send_sms_without_personalisation(sample_template, mocker):
         now.strftime(DATETIME_FORMAT)
     )
 
-    firetext_client.send_sms.assert_called_once_with("+441234123123", "Sample service: This is a template")
+    firetext_client.send_sms.assert_called_once_with(
+        to="+441234123123",
+        content="Sample service: This is a template",
+        reference=str(notification_id)
+    )
 
 
 def test_should_send_sms_if_restricted_service_and_valid_number(notify_db, notify_db_session, mocker):
@@ -317,7 +329,11 @@ def test_should_send_sms_if_restricted_service_and_valid_number(notify_db, notif
         now.strftime(DATETIME_FORMAT)
     )
 
-    firetext_client.send_sms.assert_called_once_with("+447700900890", "Sample service: This is a template")
+    firetext_client.send_sms.assert_called_once_with(
+        to="+447700900890",
+        content="Sample service: This is a template",
+        reference=str(notification_id)
+    )
 
 
 def test_should_not_send_sms_if_restricted_service_and_invalid_number(notify_db, notify_db_session, mocker):
@@ -394,7 +410,11 @@ def test_should_send_template_to_correct_sms_provider_and_persist_with_job_id(sa
         "encrypted-in-reality",
         now.strftime(DATETIME_FORMAT)
     )
-    firetext_client.send_sms.assert_called_once_with("+441234123123", "Sample service: This is a template")
+    firetext_client.send_sms.assert_called_once_with(
+        to="+441234123123",
+        content="Sample service: This is a template",
+        reference=str(notification_id)
+    )
     persisted_notification = notifications_dao.get_notification(sample_job.template.service_id, notification_id)
     assert persisted_notification.id == notification_id
     assert persisted_notification.to == '+441234123123'
@@ -444,6 +464,31 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh
     assert persisted_notification.sent_by == 'ses'
 
 
+def test_should_use_email_template_and_persist_ses_reference(sample_email_template_with_placeholders, mocker):
+    notification = {
+        "template": sample_email_template_with_placeholders.id,
+        "to": "my_email@my_email.com",
+        "personalisation": {"name": "Jo"}
+    }
+    mocker.patch('app.encryption.decrypt', return_value=notification)
+    mocker.patch('app.aws_ses_client.send_email', return_value='reference')
+
+    notification_id = uuid.uuid4()
+    now = datetime.utcnow()
+    send_email(
+        sample_email_template_with_placeholders.service_id,
+        notification_id,
+        'subject',
+        'email_from',
+        "encrypted-in-reality",
+        now.strftime(DATETIME_FORMAT)
+    )
+    persisted_notification = notifications_dao.get_notification(
+        sample_email_template_with_placeholders.service_id, notification_id
+    )
+    assert persisted_notification.reference == 'reference'
+
+
 def test_should_use_email_template_and_persist_without_personalisation(
         sample_email_template, mocker
 ):
@@ -451,7 +496,7 @@ def test_should_use_email_template_and_persist_without_personalisation(
         "template": sample_email_template.id,
         "to": "my_email@my_email.com",
     })
-    mocker.patch('app.aws_ses_client.send_email')
+    mocker.patch('app.aws_ses_client.send_email', return_value="ref")
     mocker.patch('app.aws_ses_client.get_name', return_value='ses')
 
     notification_id = uuid.uuid4()
@@ -478,7 +523,7 @@ def test_should_persist_notification_as_failed_if_sms_client_fails(sample_templa
         "to": "+441234123123"
     }
     mocker.patch('app.encryption.decrypt', return_value=notification)
-    mocker.patch('app.firetext_client.send_sms', side_effect=FiretextClientException())
+    mocker.patch('app.firetext_client.send_sms', side_effect=FiretextClientException(firetext_error()))
     mocker.patch('app.firetext_client.get_name', return_value="firetext")
     now = datetime.utcnow()
 
@@ -490,7 +535,11 @@ def test_should_persist_notification_as_failed_if_sms_client_fails(sample_templa
         "encrypted-in-reality",
         now.strftime(DATETIME_FORMAT)
     )
-    firetext_client.send_sms.assert_called_once_with("+441234123123", "Sample service: This is a template")
+    firetext_client.send_sms.assert_called_once_with(
+        to="+441234123123",
+        content="Sample service: This is a template",
+        reference=str(notification_id)
+    )
     persisted_notification = notifications_dao.get_notification(sample_template.service_id, notification_id)
     assert persisted_notification.id == notification_id
     assert persisted_notification.to == '+441234123123'
@@ -596,7 +645,7 @@ def test_should_send_sms_code(mocker):
 
     mocker.patch('app.firetext_client.send_sms')
     send_sms_code(encrypted_notification)
-    firetext_client.send_sms.assert_called_once_with(notification['to'], notification['secret_code'])
+    firetext_client.send_sms.assert_called_once_with(notification['to'], notification['secret_code'], 'send-sms-code')
 
 
 def test_should_throw_firetext_client_exception(mocker):
@@ -604,9 +653,9 @@ def test_should_throw_firetext_client_exception(mocker):
                     'secret_code': '12345'}
 
     encrypted_notification = encryption.encrypt(notification)
-    mocker.patch('app.firetext_client.send_sms', side_effect=FiretextClientException)
+    mocker.patch('app.firetext_client.send_sms', side_effect=FiretextClientException(firetext_error()))
     send_sms_code(encrypted_notification)
-    firetext_client.send_sms.assert_called_once_with(notification['to'], notification['secret_code'])
+    firetext_client.send_sms.assert_called_once_with(notification['to'], notification['secret_code'], 'send-sms-code')
 
 
 def test_should_send_email_code(mocker):

tests/app/conftest.py

@@ -282,6 +282,7 @@ def sample_notification(notify_db,
                         job=None,
                         to_field=None,
                         status='sent',
+                        reference=None,
                         created_at=datetime.utcnow()):
     if service is None:
         service = sample_service(notify_db, notify_db_session)
@@ -305,6 +306,7 @@ def sample_notification(notify_db,
         'service': service,
         'template': template,
         'status': status,
+        'reference': reference,
         'created_at': created_at
     }
     notification = Notification(**data)

tests/app/dao/test_notification_dao.py

@@ -16,12 +16,44 @@ from app.dao.notifications_dao import (
     dao_get_notification_statistics_for_service,
     delete_successful_notifications_created_more_than_a_day_ago,
     delete_failed_notifications_created_more_than_a_week_ago,
-    dao_get_notification_statistics_for_service_and_day
+    dao_get_notification_statistics_for_service_and_day,
+    update_notification_status_by_id,
+    update_notification_reference_by_id,
+    update_notification_status_by_reference
 )
 from tests.app.conftest import sample_job
 from tests.app.conftest import sample_notification
 
 
+def test_should_by_able_to_update_reference_by_id(sample_notification):
+    assert not Notification.query.get(sample_notification.id).reference
+    count = update_notification_reference_by_id(sample_notification.id, 'reference')
+    assert count == 1
+    assert Notification.query.get(sample_notification.id).reference == 'reference'
+
+
+def test_should_by_able_to_update_status_by_reference(sample_notification):
+    assert Notification.query.get(sample_notification.id).status == "sent"
+    update_notification_reference_by_id(sample_notification.id, 'reference')
+    update_notification_status_by_reference('reference', 'delivered')
+    assert Notification.query.get(sample_notification.id).status == 'delivered'
+
+
+def test_should_by_able_to_update_status_by_id(sample_notification):
+    assert Notification.query.get(sample_notification.id).status == 'sent'
+    count = update_notification_status_by_id(sample_notification.id, 'delivered')
+    assert count == 1
+    assert Notification.query.get(sample_notification.id).status == 'delivered'
+
+
+def test_should_return_zero_count_if_no_notification_with_id():
+    assert update_notification_status_by_id(str(uuid.uuid4()), 'delivered') == 0
+
+
+def test_should_return_zero_count_if_no_notification_with_reference():
+    assert update_notification_status_by_reference('something', 'delivered') == 0
+
+
 def test_should_be_able_to_get_statistics_for_a_service(sample_template):
     data = {
         'to': '+44709123456',
@@ -568,7 +600,7 @@ def test_should_not_delete_sent_notifications_before_one_day(notify_db, notify_d
 
 def test_should_not_delete_failed_notifications_before_seven_days(notify_db, notify_db_session):
     expired = datetime.utcnow() - timedelta(hours=24 * 7)
-    valid = datetime.utcnow() - timedelta(hours=(24 * 6) + 23,  minutes=59, seconds=59)
+    valid = datetime.utcnow() - timedelta(hours=(24 * 6) + 23, minutes=59, seconds=59)
     sample_notification(notify_db, notify_db_session, created_at=expired, status="failed", to_field="expired")
     sample_notification(notify_db, notify_db_session, created_at=valid, status="failed", to_field="valid")
     assert len(Notification.query.all()) == 2

tests/app/notifications/test_rest.py

@@ -7,7 +7,9 @@ from flask import json
 from app.models import Service
 from app.dao.templates_dao import dao_get_all_templates_for_service
 from app.dao.services_dao import dao_update_service
+from app.dao.notifications_dao import get_notification_by_id
 from freezegun import freeze_time
+from tests.app import load_example_ses
 
 
 def test_get_notification_by_id(notify_api, sample_notification):
@@ -854,3 +856,331 @@ def test_should_allow_api_call_if_under_day_limit_regardless_of_type(notify_db,
                 headers=[('Content-Type', 'application/json'), auth_header])
 
             assert response.status_code == 201
+
+
+def test_firetext_callback_should_not_need_auth(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=0&reference=send-sms-code&time=2016-03-10 14:17:00',
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            assert response.status_code == 200
+
+
+def test_firetext_callback_should_return_400_if_empty_reference(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=0&reference=&time=2016-03-10 14:17:00',
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 400
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'Firetext callback failed: reference missing'
+
+
+def test_firetext_callback_should_return_400_if_no_reference(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=0&time=2016-03-10 14:17:00',
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 400
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'Firetext callback failed: reference missing'
+
+
+def test_firetext_callback_should_return_200_if_send_sms_reference(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=0&time=2016-03-10 14:17:00&reference=send-sms-code',
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert json_resp['result'] == 'success'
+            assert json_resp['message'] == 'Firetext callback succeeded: send-sms-code'
+
+
+def test_firetext_callback_should_return_400_if_no_status(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&time=2016-03-10 14:17:00',
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 400
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'Firetext callback failed: status missing'
+
+
+def test_firetext_callback_should_return_400_if_unknown_status(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=99&time=2016-03-10 14:17:00&reference={}'.format(uuid.uuid4()),
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 400
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'Firetext callback failed: status 99 not found.'
+
+
+def test_firetext_callback_should_return_400_if_invalid_guid_notification_id(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=0&time=2016-03-10 14:17:00&reference=1234',
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 400
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'Firetext callback with invalid reference 1234'
+
+
+def test_firetext_callback_should_return_404_if_cannot_find_notification_id(notify_db, notify_db_session, notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            missing_notification_id = uuid.uuid4()
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=0&time=2016-03-10 14:17:00&reference={}'.format(
+                    missing_notification_id
+                ),
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 404
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'Firetext callback failed: notification {} not found. Status {}'.format(
+                missing_notification_id,
+                'delivered'
+            )
+
+
+def test_firetext_callback_should_update_notification_status(notify_api, sample_notification):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            original = get_notification_by_id(sample_notification.id)
+            assert original.status == 'sent'
+
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=0&time=2016-03-10 14:17:00&reference={}'.format(
+                    sample_notification.id
+                ),
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert json_resp['result'] == 'success'
+            assert json_resp['message'] == 'Firetext callback succeeded. reference {} updated'.format(
+                sample_notification.id
+            )
+            updated = get_notification_by_id(sample_notification.id)
+            assert updated.status == 'delivered'
+
+
+def test_firetext_callback_should_update_notification_status_failed(notify_api, sample_notification):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            original = get_notification_by_id(sample_notification.id)
+            assert original.status == 'sent'
+
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=1&time=2016-03-10 14:17:00&reference={}'.format(
+                    sample_notification.id
+                ),
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert json_resp['result'] == 'success'
+            assert json_resp['message'] == 'Firetext callback succeeded. reference {} updated'.format(
+                sample_notification.id
+            )
+            updated = get_notification_by_id(sample_notification.id)
+            assert updated.status == 'failed'
+
+
+def test_firetext_callback_should_update_notification_status_sent(notify_api, notify_db, notify_db_session):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            notification = sample_notification(notify_db, notify_db_session, status='delivered')
+            original = get_notification_by_id(notification.id)
+            assert original.status == 'delivered'
+
+            response = client.post(
+                path='/notifications/sms/firetext',
+                data='mobile=441234123123&status=2&time=2016-03-10 14:17:00&reference={}'.format(
+                    notification.id
+                ),
+                headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert json_resp['result'] == 'success'
+            assert json_resp['message'] == 'Firetext callback succeeded. reference {} updated'.format(
+                notification.id
+            )
+            updated = get_notification_by_id(notification.id)
+            assert updated.status == 'sent'
+
+
+def test_ses_callback_should_not_need_auth(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            response = client.post(
+                path='/notifications/email/ses',
+                data=load_example_ses('ses_response'),
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            assert response.status_code == 404
+
+
+def test_ses_callback_should_fail_if_invalid_json(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            response = client.post(
+                path='/notifications/email/ses',
+                data="nonsense",
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 400
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'SES callback failed: invalid json'
+
+
+def test_ses_callback_should_fail_if_invalid_notification_type(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            ses_response = json.loads(load_example_ses('ses_response'))
+            ses_response['Message']['notificationType'] = 'Unknown'
+
+            response = client.post(
+                path='/notifications/email/ses',
+                data=json.dumps(ses_response),
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 400
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'SES callback failed: status Unknown not found'
+
+
+def test_ses_callback_should_fail_if_missing_message_id(notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            ses_response = json.loads(load_example_ses('ses_response'))
+            del(ses_response['Message']['mail']['messageId'])
+
+            response = client.post(
+                path='/notifications/email/ses',
+                data=json.dumps(ses_response),
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 400
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'SES callback failed: messageId missing'
+
+
+def test_ses_callback_should_fail_if_notification_cannot_be_found(notify_db, notify_db_session, notify_api):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            ses_response = json.loads(load_example_ses('ses_response'))
+            ses_response['Message']['mail']['messageId'] = 'wont find this'
+
+            response = client.post(
+                path='/notifications/email/ses',
+                data=json.dumps(ses_response),
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            json_resp = json.loads(response.get_data(as_text=True))
+            print(json_resp)
+            assert response.status_code == 404
+            assert json_resp['result'] == 'error'
+            assert json_resp['message'] == 'SES callback failed: notification not found. Status delivered'
+
+
+def test_ses_callback_should_update_notification_status(notify_api, notify_db, notify_db_session):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+
+            notification = sample_notification(notify_db, notify_db_session, reference='ref')
+
+            assert get_notification_by_id(notification.id).status == 'sent'
+
+            ses_response = json.loads(load_example_ses('ses_response'))
+            ses_response['Message']['mail']['messageId'] = 'ref'
+
+            response = client.post(
+                path='/notifications/email/ses',
+                data=json.dumps(ses_response),
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert json_resp['result'] == 'success'
+            assert json_resp['message'] == 'SES callback succeeded'
+            assert get_notification_by_id(notification.id).status == 'delivered'
+
+
+def test_should_handle_invite_email_callbacks(notify_api, notify_db, notify_db_session):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+
+            notify_api.config['INVITATION_EMAIL_FROM'] = 'test-invite'
+            notify_api.config['NOTIFY_EMAIL_DOMAIN'] = 'test-domain.com'
+
+            ses_response = json.loads(load_example_ses('ses_response'))
+            ses_response['Message']['mail']['messageId'] = 'ref'
+            ses_response['Message']['mail']['source'] = 'test-invite@test-domain.com'
+
+            response = client.post(
+                path='/notifications/email/ses',
+                data=json.dumps(ses_response),
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert json_resp['result'] == 'success'
+            assert json_resp['message'] == 'SES callback succeeded'
+
+
+def test_should_handle_validation_code_callbacks(notify_api, notify_db, notify_db_session):
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+
+            notify_api.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'] = 'valid-code@test.com'
+
+            ses_response = json.loads(load_example_ses('ses_response'))
+            ses_response['Message']['mail']['messageId'] = 'ref'
+            ses_response['Message']['mail']['source'] = 'valid-code@test.com'
+
+            response = client.post(
+                path='/notifications/email/ses',
+                data=json.dumps(ses_response),
+                headers=[('Content-Type', 'text/plain; charset=UTF-8')]
+            )
+            json_resp = json.loads(response.get_data(as_text=True))
+            assert response.status_code == 200
+            assert json_resp['result'] == 'success'
+            assert json_resp['message'] == 'SES callback succeeded'