Merge branch 'populate_provider_column' into firetext-inbound-sms-not-null-provider

app/__init__.py

@@ -93,6 +93,7 @@ def register_blueprint(application):
     from app.organisation.rest import organisation_blueprint
     from app.dvla_organisation.rest import dvla_organisation_blueprint
     from app.delivery.rest import delivery_blueprint
+    from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint
     from app.notifications.receive_notifications import receive_notifications_blueprint
     from app.notifications.notifications_ses_callback import ses_callback_blueprint
     from app.notifications.notifications_sms_callback import sms_callback_blueprint
@@ -133,6 +134,9 @@ def register_blueprint(application):
     delivery_blueprint.before_request(requires_admin_auth)
     application.register_blueprint(delivery_blueprint)
 
+    inbound_sms_blueprint.before_request(requires_admin_auth)
+    application.register_blueprint(inbound_sms_blueprint)
+
     accept_invite.before_request(requires_admin_auth)
     application.register_blueprint(accept_invite, url_prefix='/invite')
 

app/dao/inbound_sms_dao.py

@@ -1,7 +1,30 @@
 from app import db
 from app.dao.dao_utils import transactional
+from app.models import InboundSms
 
 
 @transactional
 def dao_create_inbound_sms(inbound_sms):
     db.session.add(inbound_sms)
+
+
+def dao_get_inbound_sms_for_service(service_id, limit=None, user_number=None):
+    q = InboundSms.query.filter(
+        InboundSms.service_id == service_id
+    ).order_by(
+        InboundSms.created_at.desc()
+    )
+
+    if user_number:
+        q = q.filter(InboundSms.user_number == user_number)
+
+    if limit:
+        q = q.limit(limit)
+
+    return q.all()
+
+
+def dao_count_inbound_sms_for_service(service_id):
+    return InboundSms.query.filter(
+        InboundSms.service_id == service_id
+    ).count()

app/inbound_sms/rest.py

@@ -0,0 +1,42 @@
+from flask import (
+    Blueprint,
+    jsonify,
+    request
+)
+from notifications_utils.recipients import validate_and_format_phone_number
+
+from app.dao.inbound_sms_dao import dao_get_inbound_sms_for_service, dao_count_inbound_sms_for_service
+from app.errors import register_errors
+
+inbound_sms = Blueprint(
+    'inbound_sms',
+    __name__,
+    url_prefix='/service/<service_id>/inbound-sms'
+)
+
+register_errors(inbound_sms)
+
+
+@inbound_sms.route('')
+def get_inbound_sms_for_service(service_id):
+    limit = request.args.get('limit')
+    user_number = request.args.get('user_number')
+
+    if user_number:
+        # we use this to normalise to an international phone number
+        user_number = validate_and_format_phone_number(user_number, international=True)
+
+    results = dao_get_inbound_sms_for_service(service_id, limit, user_number)
+
+    return jsonify(data=[row.serialize() for row in results])
+
+
+@inbound_sms.route('/summary')
+def get_inbound_sms_summary_for_service(service_id):
+    count = dao_count_inbound_sms_for_service(service_id)
+    most_recent = dao_get_inbound_sms_for_service(service_id, limit=1)
+
+    return jsonify(
+        count=count,
+        most_recent=most_recent[0].created_at.isoformat() if most_recent else None
+    )

app/models.py

@@ -671,7 +671,6 @@ NOTIFICATION_STATUS_TYPES_BILLABLE = [
     NOTIFICATION_PERMANENT_FAILURE,
 ]
 
-
 NOTIFICATION_STATUS_TYPES = [
     NOTIFICATION_CREATED,
     NOTIFICATION_SENDING,
@@ -1176,3 +1175,32 @@ class InboundSms(db.Model):
     @content.setter
     def content(self, content):
         self._content = encryption.encrypt(content)
+
+    def serialize(self):
+        return {
+            'id': str(self.id),
+            'created_at': self.created_at.isoformat(),
+            'service_id': str(self.service_id),
+            'notify_number': self.notify_number,
+            'user_number': self.user_number,
+            'content': self.content,
+            'provider_date': self.provider_date and self.provider_date.isoformat(),
+            'provider_reference': self.provider_reference
+        }
+
+
+class LetterRate(db.Model):
+    __tablename__ = 'letter_rates'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    valid_from = valid_from = db.Column(db.DateTime, nullable=False)
+
+
+class LetterRateDetail(db.Model):
+    __tablename__ = 'letter_rate_details'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    letter_rate_id = db.Column(UUID(as_uuid=True), db.ForeignKey('letter_rates.id'), index=True, nullable=False)
+    letter_rate = db.relationship('LetterRate', backref='letter_rates')
+    page_total = db.Column(db.Integer, nullable=False)
+    rate = db.Column(db.Numeric(), nullable=False)

app/notifications/receive_notifications.py

@@ -2,7 +2,7 @@ from urllib.parse import unquote
 
 import iso8601
 from flask import jsonify, Blueprint, current_app, request
-from notifications_utils.recipients import normalise_phone_number
+from notifications_utils.recipients import validate_and_format_phone_number
 
 from app import statsd_client, firetext_client, mmg_client
 from app.dao.services_dao import dao_fetch_services_by_sms_sender
@@ -38,7 +38,7 @@ def receive_mmg_sms():
         # succesfully
         return 'RECEIVED', 200
 
-    statsd_client.incr('inbound.mmg.succesful')
+    statsd_client.incr('inbound.mmg.successful')
 
     service = potential_services[0]
 
@@ -65,7 +65,7 @@ def format_mmg_datetime(date):
 
 def create_inbound_mmg_sms_object(service, json):
     message = format_mmg_message(json['Message'])
-    user_number = normalise_phone_number(json['MSISDN'])
+    user_number = validate_and_format_phone_number(json['MSISDN'], international=True)
 
     provider_date = json.get('DateRecieved')
     if provider_date:
@@ -100,7 +100,7 @@ def receive_firetext_sms():
 
     service = potential_services[0]
 
-    user_number = normalise_phone_number(post_data['source'])
+    user_number = validate_and_format_phone_number(post_data['source'], international=True)
     message = post_data['message']
     timestamp = post_data['time']
 

migrations/versions/0091_letter_billing.py

@@ -0,0 +1,50 @@
+"""empty message
+
+Revision ID: 0091_letter_billing
+Revises: 0090_inbound_sms
+Create Date: 2017-05-31 11:43:55.744631
+
+"""
+import uuid
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision = '0091_letter_billing'
+down_revision = '0090_inbound_sms'
+
+
+def upgrade():
+    op.create_table('letter_rates',
+                    sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+                    sa.Column('valid_from', sa.DateTime(), nullable=False),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+    op.create_table('letter_rate_details',
+                    sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+                    sa.Column('letter_rate_id', postgresql.UUID(as_uuid=True), nullable=False),
+                    sa.Column('page_total', sa.Integer(), nullable=False),
+                    sa.Column('rate', sa.Numeric(), nullable=False),
+                    sa.ForeignKeyConstraint(['letter_rate_id'], ['letter_rates.id'], ),
+                    sa.PrimaryKeyConstraint('id')
+                    )
+    op.create_index(op.f('ix_letter_rate_details_letter_rate_id'), 'letter_rate_details', ['letter_rate_id'],
+                    unique=False)
+
+    op.get_bind()
+    letter_id = uuid.uuid4()
+    op.execute("insert into letter_rates(id, valid_from) values('{}', '2017-03-31 23:00:00')".format(letter_id))
+    insert_details = "insert into letter_rate_details(id, letter_rate_id, page_total, rate) values('{}', '{}', {}, {})"
+    op.execute(
+        insert_details.format(uuid.uuid4(), letter_id, 1, 29.3))
+    op.execute(
+        insert_details.format(uuid.uuid4(), letter_id, 2, 32))
+    op.execute(
+        insert_details.format(uuid.uuid4(), letter_id, 3, 35))
+
+
+def downgrade():
+    op.get_bind()
+    op.drop_index('ix_letter_rate_details_letter_rate_id')
+    op.drop_table('letter_rate_details')
+    op.drop_table('letter_rates')

migrations/versions/0092_add_inbound_provider.py

@@ -1,14 +1,14 @@
 """empty message
 
-Revision ID: 0091_add_inbound_provider
+Revision ID: 0092_add_inbound_provider
-Revises: 0090_inbound_sms
+Revises: 0091_letter_billing
 Create Date: 2017-06-02 16:07:35.445423
 
 """
 
 # revision identifiers, used by Alembic.
-revision = '0091_add_inbound_provider'
+revision = '0092_add_inbound_provider'
-down_revision = '0090_inbound_sms'
+down_revision = '0091_letter_billing'
 
 from alembic import op
 import sqlalchemy as sa

migrations/versions/0093_populate_inbound_provider.py

@@ -1,14 +1,14 @@
 """empty message
 
-Revision ID: 0092_populate_inbound_provider
+Revision ID: 0093_populate_inbound_provider
-Revises: 0091_add_inbound_provider
+Revises: 0092_add_inbound_provider
 Create Date: 2017-05-22 10:23:43.939050
 
 """
 
 # revision identifiers, used by Alembic.
-revision = '0092_populate_inbound_provider'
+revision = '0093_populate_inbound_provider'
-down_revision = '0091_add_inbound_provider'
+down_revision = '0092_add_inbound_provider'
 
 from alembic import op
 import sqlalchemy as sa

tests/app/conftest.py

@@ -1,11 +1,12 @@
-import requests_mock
+import json
-import pytest
 import uuid
 from datetime import (datetime, date, timedelta)
 
+import requests_mock
+import pytest
 from sqlalchemy import asc
 from sqlalchemy.orm.session import make_transient
-from flask import current_app
+from flask import current_app, url_for
 
 from app import db
 from app.models import (
@@ -35,6 +36,7 @@ from app.dao.invited_user_dao import save_invited_user
 from app.dao.provider_rates_dao import create_provider_rates
 from app.clients.sms.firetext import FiretextClient
 
+from tests import create_authorization_header
 from tests.app.db import create_user, create_template, create_notification
 
 
@@ -976,3 +978,41 @@ def restore_provider_details(notify_db, notify_db_session):
     notify_db.session.add_all(existing_provider_details)
     notify_db.session.add_all(existing_provider_details_history)
     notify_db.session.commit()
+
+
+@pytest.fixture
+def admin_request(client):
+    class AdminRequest:
+
+        @staticmethod
+        def get(endpoint, endpoint_kwargs=None, expected_status=200):
+            resp = client.get(
+                url_for(endpoint, **(endpoint_kwargs or {})),
+                headers=[create_authorization_header()]
+            )
+            json_resp = json.loads(resp.get_data(as_text=True))
+            assert resp.status_code == expected_status
+            return json_resp
+
+        @staticmethod
+        def post(endpoint, endpoint_kwargs=None, data=None, expected_status=200):
+            resp = client.post(
+                url_for(endpoint, **(endpoint_kwargs or {})),
+                data=json.dumps(data),
+                headers=[('Content-Type', 'application/json'), create_authorization_header()]
+            )
+            json_resp = json.loads(resp.get_data(as_text=True))
+            assert resp.status_code == expected_status
+            return json_resp
+
+        @staticmethod
+        def delete(endpoint, endpoint_kwargs=None, expected_status=204):
+            resp = client.delete(
+                url_for(endpoint, **(endpoint_kwargs or {})),
+                headers=[create_authorization_header()]
+            )
+            json_resp = json.loads(resp.get_data(as_text=True))
+            assert resp.status_code == expected_status
+            return json_resp
+
+    return AdminRequest

tests/app/dao/test_inbound_sms_dao.py

@@ -0,0 +1,59 @@
+from datetime import datetime
+
+from freezegun import freeze_time
+
+from app.dao.inbound_sms_dao import dao_get_inbound_sms_for_service, dao_count_inbound_sms_for_service
+
+from tests.app.db import create_inbound_sms, create_service
+
+
+def test_get_all_inbound_sms(sample_service):
+    inbound = create_inbound_sms(sample_service)
+
+    res = dao_get_inbound_sms_for_service(sample_service.id)
+    assert len(res) == 1
+    assert res[0] == inbound
+
+
+def test_get_all_inbound_sms_when_none_exist(sample_service):
+    res = dao_get_inbound_sms_for_service(sample_service.id)
+    assert len(res) == 0
+
+
+def test_get_all_inbound_sms_limits_and_orders(sample_service):
+    with freeze_time('2017-01-01'):
+        one = create_inbound_sms(sample_service)
+    with freeze_time('2017-01-03'):
+        three = create_inbound_sms(sample_service)
+    with freeze_time('2017-01-02'):
+        two = create_inbound_sms(sample_service)
+
+    res = dao_get_inbound_sms_for_service(sample_service.id, limit=2)
+    assert len(res) == 2
+    assert res[0] == three
+    assert res[0].created_at == datetime(2017, 1, 3)
+    assert res[1] == two
+    assert res[1].created_at == datetime(2017, 1, 2)
+
+
+def test_get_all_inbound_sms_filters_on_service(notify_db_session):
+    service_one = create_service(service_name='one')
+    service_two = create_service(service_name='two')
+
+    sms_one = create_inbound_sms(service_one)
+    sms_two = create_inbound_sms(service_two)
+
+    res = dao_get_inbound_sms_for_service(service_one.id)
+    assert len(res) == 1
+    assert res[0] == sms_one
+
+
+def test_count_inbound_sms_for_service(notify_db_session):
+    service_one = create_service(service_name='one')
+    service_two = create_service(service_name='two')
+
+    create_inbound_sms(service_one)
+    create_inbound_sms(service_one)
+    create_inbound_sms(service_two)
+
+    assert dao_count_inbound_sms_for_service(service_one.id) == 2

tests/app/db.py

@@ -4,6 +4,7 @@ import uuid
 
 from app.dao.jobs_dao import dao_create_job
 from app.models import (
+    InboundSms,
     Service,
     User,
     Template,
@@ -20,6 +21,7 @@ from app.dao.notifications_dao import dao_create_notification, dao_created_sched
 from app.dao.templates_dao import dao_create_template
 from app.dao.services_dao import dao_create_service
 from app.dao.service_permissions_dao import dao_add_service_permission
+from app.dao.inbound_sms_dao import dao_create_inbound_sms
 
 
 def create_user(mobile_number="+447700900986", email="notify@digital.cabinet-office.gov.uk", state='active'):
@@ -183,3 +185,24 @@ def create_service_permission(service_id, permission=EMAIL_TYPE):
     service_permissions = ServicePermission.query.all()
 
     return service_permissions
+
+
+def create_inbound_sms(
+    service,
+    notify_number=None,
+    user_number='447700900111',
+    provider_date=None,
+    provider_reference=None,
+    content='Hello'
+):
+    inbound = InboundSms(
+        service=service,
+        created_at=datetime.utcnow(),
+        notify_number=notify_number or service.sms_sender,
+        user_number=user_number,
+        provider_date=provider_date or datetime.utcnow(),
+        provider_reference=provider_reference or 'foo',
+        content=content,
+    )
+    dao_create_inbound_sms(inbound)
+    return inbound

tests/app/inbound_sms/test_rest.py

@@ -0,0 +1,114 @@
+from datetime import datetime
+
+import pytest
+from freezegun import freeze_time
+
+from tests.app.db import create_inbound_sms, create_service
+
+
+def test_get_inbound_sms(admin_request, sample_service):
+    one = create_inbound_sms(sample_service)
+    two = create_inbound_sms(sample_service)
+
+    json_resp = admin_request.get(
+        'inbound_sms.get_inbound_sms_for_service',
+        endpoint_kwargs={'service_id': sample_service.id}
+    )
+
+    sms = json_resp['data']
+
+    assert len(sms) == 2
+    assert {inbound['id'] for inbound in sms} == {str(one.id), str(two.id)}
+    assert sms[0]['content'] == 'Hello'
+    assert set(sms[0].keys()) == {
+        'id',
+        'created_at',
+        'service_id',
+        'notify_number',
+        'user_number',
+        'content',
+        'provider_date',
+        'provider_reference'
+    }
+
+
+def test_get_inbound_sms_limits(admin_request, sample_service):
+    with freeze_time('2017-01-01'):
+        one = create_inbound_sms(sample_service)
+    with freeze_time('2017-01-02'):
+        two = create_inbound_sms(sample_service)
+
+    sms = admin_request.get(
+        'inbound_sms.get_inbound_sms_for_service',
+        endpoint_kwargs={'service_id': sample_service.id, 'limit': 1}
+    )
+
+    assert len(sms['data']) == 1
+    assert sms['data'][0]['id'] == str(two.id)
+
+
+@pytest.mark.parametrize('user_number', [
+    '(07700) 900-001',
+    '+4407700900001',
+    '447700900001',
+])
+def test_get_inbound_sms_filters_user_number(admin_request, sample_service, user_number):
+    # user_number in the db is international and normalised
+    one = create_inbound_sms(sample_service, user_number='447700900001')
+    two = create_inbound_sms(sample_service, user_number='447700900002')
+
+    sms = admin_request.get(
+        'inbound_sms.get_inbound_sms_for_service',
+        endpoint_kwargs={'service_id': sample_service.id, 'user_number': user_number}
+    )
+
+    assert len(sms['data']) == 1
+    assert sms['data'][0]['id'] == str(one.id)
+    assert sms['data'][0]['user_number'] == str(one.user_number)
+
+
+def test_get_inbound_sms_filters_international_user_number(admin_request, sample_service):
+    # user_number in the db is international and normalised
+    one = create_inbound_sms(sample_service, user_number='12025550104')
+    two = create_inbound_sms(sample_service)
+
+    sms = admin_request.get(
+        'inbound_sms.get_inbound_sms_for_service',
+        endpoint_kwargs={'service_id': sample_service.id, 'user_number': '+1 (202) 555-0104'}
+    )
+
+    assert len(sms['data']) == 1
+    assert sms['data'][0]['id'] == str(one.id)
+    assert sms['data'][0]['user_number'] == str(one.user_number)
+
+
+def test_get_inbound_sms_summary(admin_request, sample_service):
+    other_service = create_service(service_name='other_service')
+    with freeze_time('2017-01-01'):
+        create_inbound_sms(sample_service)
+    with freeze_time('2017-01-02'):
+        create_inbound_sms(sample_service)
+    with freeze_time('2017-01-03'):
+        create_inbound_sms(other_service)
+
+    summary = admin_request.get(
+        'inbound_sms.get_inbound_sms_summary_for_service',
+        endpoint_kwargs={'service_id': sample_service.id}
+    )
+
+    assert summary == {
+        'count': 2,
+        'most_recent': datetime(2017, 1, 2).isoformat()
+    }
+
+
+def test_get_inbound_sms_summary_with_no_inbound(admin_request, sample_service):
+    summary = admin_request.get(
+        'inbound_sms.get_inbound_sms_summary_for_service',
+        endpoint_kwargs={'service_id': sample_service.id}
+    )
+
+    assert summary == {
+        'count': 0,
+        'most_recent': None
+    }

tests/app/notifications/test_receive_notification.py

@@ -65,7 +65,7 @@ def test_create_inbound_mmg_sms_object(sample_service):
 
     assert inbound_sms.service_id == sample_service.id
     assert inbound_sms.notify_number == 'foo'
-    assert inbound_sms.user_number == '7700900001'
+    assert inbound_sms.user_number == '447700900001'
     assert inbound_sms.provider_date == datetime(2017, 1, 2, 3, 4, 5)
     assert inbound_sms.provider_reference == 'bar'
     assert inbound_sms._content != 'hello there 📩'
@@ -116,7 +116,7 @@ def test_receive_notification_returns_received_to_firetext(notify_db_session, cl
 
 
 def test_receive_notification_from_firetext_persists_message(notify_db_session, client, mocker):
-    mock = mocker.patch('app.notifications.receive_notifications.statsd_client.incr')
+    mocker.patch('app.notifications.receive_notifications.statsd_client.incr')
 
     service = create_service(service_name='b', sms_sender='07111111111')
 
@@ -134,7 +134,7 @@ def test_receive_notification_from_firetext_persists_message(notify_db_session,
 
     assert result['status'] == 'ok'
     assert persisted.notify_number == '07111111111'
-    assert persisted.user_number == '7999999999'
+    assert persisted.user_number == '447999999999'
     assert persisted.service == service
     assert persisted.content == 'this is a message'
     assert persisted.provider == 'firetext'