Merge pull request #995 from alphagov/inbound-sms

add inbound sms table

app/dao/inbound_sms_dao.py

@@ -0,0 +1,7 @@
+from app import db
+from app.dao.dao_utils import transactional
+
+
+@transactional
+def dao_create_inbound_sms(inbound_sms):
+    db.session.add(inbound_sms)

app/dao/services_dao.py

@@ -67,6 +67,12 @@ def dao_fetch_service_by_id(service_id, only_active=False):
     return query.one()
 
 
+def dao_fetch_services_by_sms_sender(sms_sender):
+    return Service.query.filter(
+        Service.sms_sender == sms_sender
+    ).all()
+
+
 def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False):
     query = Service.query.filter_by(
         id=service_id

app/models.py

@@ -1152,3 +1152,26 @@ class JobStatistics(db.Model):
         )
         the_string += "created at {}".format(self.created_at)
         return the_string
+
+
+class InboundSms(db.Model):
+    __tablename__ = 'inbound_sms'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
+    service = db.relationship('Service', backref='inbound_sms')
+
+    notify_number = db.Column(db.String, nullable=False)  # the service's number, that the msg was sent to
+    user_number = db.Column(db.String, nullable=False)  # the end user's number, that the msg was sent from
+    provider_date = db.Column(db.DateTime)
+    provider_reference = db.Column(db.String)
+    _content = db.Column('content', db.String, nullable=False)
+
+    @property
+    def content(self):
+        return encryption.decrypt(self._content)
+
+    @content.setter
+    def content(self, content):
+        self._content = encryption.encrypt(content)

app/notifications/receive_notifications.py

@@ -1,7 +1,13 @@
-from flask import Blueprint
+from urllib.parse import unquote
-from flask import current_app, jsonify
+from flask import jsonify
-from flask import request
 
+from flask import Blueprint, current_app, request
+from notifications_utils.recipients import normalise_phone_number
+
+from app import statsd_client
+from app.dao.services_dao import dao_fetch_services_by_sms_sender
+from app.dao.inbound_sms_dao import dao_create_inbound_sms
+from app.models import InboundSms
 from app.errors import register_errors
 
 receive_notifications_blueprint = Blueprint('receive_notifications', __name__)
@@ -10,11 +16,55 @@ register_errors(receive_notifications_blueprint)
 
 @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST'])
 def receive_mmg_sms():
+    """
+    {
+        'MSISDN': '447123456789'
+        'Number': '40604',
+        'Message': 'some+uri+encoded+message%3A',
+        'ID': 'SOME-MMG-SPECIFIC-ID',
+        'DateRecieved': '2017-05-21+11%3A56%3A11'
+    }
+    """
     post_data = request.get_json()
-    post_data.pop('MSISDN', None)
+    potential_services = dao_fetch_services_by_sms_sender(post_data['Number'])
-    current_app.logger.info("Recieve notification form data: {}".format(post_data))
 
-    return "RECEIVED"
+    if len(potential_services) != 1:
+        current_app.logger.error('Inbound number "{}" not associated with exactly one service'.format(
+            post_data['Number']
+        ))
+        statsd_client.incr('inbound.mmg.failed')
+        # since this is an issue with our service <-> number mapping, we should still tell MMG that we received
+        # succesfully
+        return 'RECEIVED', 200
+
+    statsd_client.incr('inbound.mmg.succesful')
+
+    service = potential_services[0]
+
+    inbound = create_inbound_sms_object(service, post_data)
+
+    current_app.logger.info('{} received inbound SMS with reference {}'.format(service.id, inbound.provider_reference))
+
+    return 'RECEIVED', 200
+
+
+def format_message(message):
+    return unquote(message.replace('+', ' '))
+
+
+def create_inbound_sms_object(service, json):
+    message = format_message(json['Message'])
+    user_number = normalise_phone_number(json['MSISDN'])
+    inbound = InboundSms(
+        service=service,
+        notify_number=service.sms_sender,
+        user_number=user_number,
+        provider_date=json['DateReceived'],
+        provider_reference=json['ID'],
+        content=message,
+    )
+    dao_create_inbound_sms(inbound)
+    return inbound
 
 
 @receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST'])

migrations/versions/0090_inbound_sms.py

@@ -0,0 +1,37 @@
+"""empty message
+
+Revision ID: 0090_inbound_sms
+Revises: 0089_govuk_sms_sender
+Create Date: 2017-05-22 11:28:53.471004
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0090_inbound_sms'
+down_revision = '0089_govuk_sms_sender'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    op.create_table(
+        'inbound_sms',
+        sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('content', sa.String, nullable=False),
+        sa.Column('notify_number', sa.String, nullable=False),
+        sa.Column('user_number', sa.String, nullable=False),
+        sa.Column('created_at', sa.DateTime, nullable=False),
+        sa.Column('provider_date', sa.DateTime, nullable=True),
+        sa.Column('provider_reference', sa.String, nullable=True),
+
+        sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_inbound_sms_service_id'), 'inbound_sms', ['service_id'], unique=False)
+    op.create_index(op.f('ix_inbound_sms_user_number'), 'inbound_sms', ['user_number'], unique=False)
+
+
+def downgrade():
+    op.drop_table('inbound_sms')

tests/app/dao/test_services_dao.py

@@ -25,7 +25,8 @@ from app.dao.services_dao import (
     fetch_stats_by_date_range_for_all_services,
     dao_suspend_service,
     dao_resume_service,
-    dao_fetch_active_users_for_service
+    dao_fetch_active_users_for_service,
+    dao_fetch_services_by_sms_sender
 )
 from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission
 from app.dao.users_dao import save_model_user
@@ -948,3 +949,13 @@ def test_dao_fetch_active_users_for_service_returns_active_only(notify_db, notif
     users = dao_fetch_active_users_for_service(service.id)
 
     assert len(users) == 1
+
+
+def test_dao_fetch_services_by_sms_sender(notify_db_session):
+    foo1 = create_service(service_name='a', sms_sender='foo')
+    foo2 = create_service(service_name='b', sms_sender='foo')
+    bar = create_service(service_name='c', sms_sender='bar')
+
+    services = dao_fetch_services_by_sms_sender('foo')
+
+    assert {foo1.id, foo2.id} == {x.id for x in services}

tests/app/db.py

@@ -39,14 +39,20 @@ def create_user(mobile_number="+447700900986", email="notify@digital.cabinet-off
 
 
 def create_service(
-        user=None, service_name="Sample service", service_id=None, restricted=False,
+    user=None,
-        service_permissions=[EMAIL_TYPE, SMS_TYPE]):
+    service_name="Sample service",
+    service_id=None,
+    restricted=False,
+    service_permissions=[EMAIL_TYPE, SMS_TYPE],
+    sms_sender='testing'
+):
     service = Service(
         name=service_name,
         message_limit=1000,
         restricted=restricted,
         email_from=service_name.lower().replace(' ', '.'),
-        created_by=user or create_user()
+        created_by=user or create_user(),
+        sms_sender=sms_sender
     )
     dao_create_service(service, service.created_by, service_id, service_permissions=service_permissions)
     return service

tests/app/notifications/test_receive_notification.py

@@ -1,14 +1,25 @@
+from datetime import datetime
+
+import pytest
 from flask import json
 
+from app.notifications.receive_notifications import (
+    format_message,
+    create_inbound_sms_object
+)
 
-def test_receive_notification_returns_received_to_mmg(client):
+from app.models import InboundSms
+from tests.app.db import create_service
+
+
+def test_receive_notification_returns_received_to_mmg(client, sample_service):
     data = {"ID": "1234",
             "MSISDN": "447700900855",
             "Message": "Some message to notify",
             "Trigger": "Trigger?",
-            "Number": "40604",
+            "Number": "testing",
             "Channel": "SMS",
-            "DateReceived": "2012-06-27-12:33:00"
+            "DateReceived": "2012-06-27 12:33:00"
             }
     response = client.post(path='/notifications/sms/receive/mmg',
                            data=json.dumps(data),
@@ -18,6 +29,60 @@ def test_receive_notification_returns_received_to_mmg(client):
     assert response.get_data(as_text=True) == 'RECEIVED'
 
 
+@pytest.mark.parametrize('message, expected_output', [
+    ('abc', 'abc'),
+    ('', ''),
+    ('lots+of+words', 'lots of words'),
+    ('%F0%9F%93%A9+%F0%9F%93%A9+%F0%9F%93%A9', '📩 📩 📩'),
+    ('x+%2B+y', 'x + y')
+])
+def test_format_message(message, expected_output):
+    assert format_message(message) == expected_output
+
+
+def test_create_inbound_sms_object(sample_service):
+    sample_service.sms_sender = 'foo'
+    data = {
+        'Message': 'hello+there+%F0%9F%93%A9',
+        'Number': 'foo',
+        'MSISDN': '07700 900 001',
+        'DateReceived': '2017-01-02 03:04:05',
+        'ID': 'bar',
+    }
+
+    inbound_sms = create_inbound_sms_object(sample_service, data)
+
+    assert inbound_sms.service_id == sample_service.id
+    assert inbound_sms.notify_number == 'foo'
+    assert inbound_sms.user_number == '7700900001'
+    assert inbound_sms.provider_date == datetime(2017, 1, 2, 3, 4, 5)
+    assert inbound_sms.provider_reference == 'bar'
+    assert inbound_sms._content != 'hello there 📩'
+    assert inbound_sms.content == 'hello there 📩'
+
+
+@pytest.mark.parametrize('notify_number', ['foo', 'baz'], ids=['two_matching_services', 'no_matching_services'])
+def test_receive_notification_error_if_not_single_matching_service(client, notify_db_session, notify_number):
+    create_service(service_name='a', sms_sender='foo')
+    create_service(service_name='b', sms_sender='foo')
+
+    data = {
+        'Message': 'hello',
+        'Number': notify_number,
+        'MSISDN': '7700900001',
+        'DateReceived': '2017-01-02 03:04:05',
+        'ID': 'bar',
+    }
+    response = client.post(path='/notifications/sms/receive/mmg',
+                           data=json.dumps(data),
+                           headers=[('Content-Type', 'application/json')])
+
+    # we still return 'RECEIVED' to MMG
+    assert response.status_code == 200
+    assert response.get_data(as_text=True) == 'RECEIVED'
+    assert InboundSms.query.count() == 0
+
+
 def test_receive_notification_returns_received_to_firetext(client):
     data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"