Merge branch 'master' into letters-billing-table

Conflicts:
	app/models.py

Makefile

@@ -306,5 +306,5 @@ cf-push:
 
 .PHONY: check-if-migrations-to-run
 check-if-migrations-to-run:
-	@echo $(shell python scripts/check_if_new_migration.py)
+	@echo $(shell python3 scripts/check_if_new_migration.py)
 

app/config.py

@@ -24,23 +24,6 @@ class QueueNames(object):
     NOTIFY = 'notify-internal-tasks'
     PROCESS_FTP = 'process-ftp-tasks'
 
-    @staticmethod
-    def old_queues():
-        return [
-            'db-sms',
-            'db-email',
-            'db-letter',
-            'priority',
-            'periodic',
-            'send-sms',
-            'send-email',
-            'research-mode',
-            'statistics',
-            'notify',
-            'retry',
-            'process-job'
-        ]
-
     @staticmethod
     def all_queues():
         return [
@@ -124,6 +107,7 @@ class Config(object):
     SMS_CHAR_COUNT_LIMIT = 495
     BRANDING_PATH = '/images/email-template/crests/'
     TEST_MESSAGE_FILENAME = 'Test message'
+    ONE_OFF_MESSAGE_FILENAME = 'Report'
     MAX_VERIFY_CODE_COUNT = 10
 
     NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
@@ -262,8 +246,6 @@ class Development(Config):
     NOTIFICATION_QUEUE_PREFIX = 'development'
     DEBUG = True
 
-    queues = QueueNames.all_queues() + QueueNames.old_queues()
-
     for queue in QueueNames.all_queues():
         Config.CELERY_QUEUES.append(
             Queue(queue, Exchange('default'), routing_key=queue)
@@ -283,9 +265,7 @@ class Test(Config):
     STATSD_HOST = "localhost"
     STATSD_PORT = 1000
 
-    queues = QueueNames.all_queues() + QueueNames.old_queues()
+    for queue in QueueNames.all_queues():
-
-    for queue in queues:
         Config.CELERY_QUEUES.append(
             Queue(queue, Exchange('default'), routing_key=queue)
         )

app/dao/inbound_sms_dao.py

@@ -0,0 +1,7 @@
+from app import db
+from app.dao.dao_utils import transactional
+
+
+@transactional
+def dao_create_inbound_sms(inbound_sms):
+    db.session.add(inbound_sms)

app/dao/jobs_dao.py

@@ -53,7 +53,8 @@ def dao_get_job_by_service_id_and_job_id(service_id, job_id):
 def dao_get_jobs_by_service_id(service_id, limit_days=None, page=1, page_size=50, statuses=None):
     query_filter = [
         Job.service_id == service_id,
-        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME']
+        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
+        Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'],
     ]
     if limit_days is not None:
         query_filter.append(cast(Job.created_at, sql_date) >= days_ago(limit_days))

app/dao/notification_usage_dao.py

@@ -153,3 +153,70 @@ def rate_multiplier():
         (NotificationHistory.rate_multiplier == None, literal_column("'1'")),  # noqa
         (NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier),  # noqa
     ]), Integer())
+
+
+@statsd(namespace="dao")
+def get_total_billable_units_for_sent_sms_notifications_in_date_range(start_date, end_date, service_id):
+
+    billable_units = 0
+    total_cost = 0.0
+
+    rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date)
+    for rate_boundary in rate_boundaries:
+        result = db.session.query(
+            func.sum(
+                NotificationHistory.billable_units * func.coalesce(NotificationHistory.rate_multiplier, 1)
+            ).label('billable_units')
+        ).filter(
+            NotificationHistory.service_id == service_id,
+            NotificationHistory.notification_type == 'sms',
+            NotificationHistory.created_at >= rate_boundary['start_date'],
+            NotificationHistory.created_at < rate_boundary['end_date'],
+            NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE)
+        )
+        billable_units_by_rate_boundry = result.scalar()
+        if billable_units_by_rate_boundry:
+            billable_units += int(billable_units_by_rate_boundry)
+            total_cost += int(billable_units_by_rate_boundry) * rate_boundary['rate']
+
+    return billable_units, total_cost
+
+
+def discover_rate_bounds_for_billing_query(start_date, end_date):
+    bounds = []
+    rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
+
+    def current_valid_from(index):
+        return rates[index].valid_from
+
+    def next_valid_from(index):
+        return rates[index + 1].valid_from
+
+    def current_rate(index):
+        return rates[index].rate
+
+    def append_rate(rate_start_date, rate_end_date, rate):
+        bounds.append({
+            'start_date': rate_start_date,
+            'end_date': rate_end_date,
+            'rate': rate
+        })
+
+    if len(rates) == 1:
+        append_rate(start_date, end_date, current_rate(0))
+        return bounds
+
+    for i in range(len(rates)):
+        # first boundary
+        if i == 0:
+            append_rate(start_date, next_valid_from(i), current_rate(i))
+
+        # last boundary
+        elif i == (len(rates) - 1):
+            append_rate(current_valid_from(i), end_date, current_rate(i))
+
+        # other boundaries
+        else:
+            append_rate(current_valid_from(i), next_valid_from(i), current_rate(i))
+
+    return bounds

app/dao/services_dao.py

@@ -3,6 +3,7 @@ from datetime import date, datetime, timedelta
 
 from sqlalchemy import asc, func
 from sqlalchemy.orm import joinedload
+from flask import current_app
 
 from app import db
 from app.dao.dao_utils import (
@@ -66,6 +67,12 @@ def dao_fetch_service_by_id(service_id, only_active=False):
     return query.one()
 
 
+def dao_fetch_services_by_sms_sender(sms_sender):
+    return Service.query.filter(
+        Service.sms_sender == sms_sender
+    ).all()
+
+
 def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False):
     query = Service.query.filter_by(
         id=service_id
@@ -131,6 +138,12 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
 @transactional
 @version_class(Service)
 def dao_create_service(service, user, service_id=None, service_permissions=[SMS_TYPE, EMAIL_TYPE]):
+    # the default property does not appear to work when there is a difference between the sqlalchemy schema and the
+    # db schema (ie: during a migration), so we have to set sms_sender manually here. After the GOVUK sms_sender
+    # migration is completed, this code should be able to be removed.
+    if not service.sms_sender:
+        service.sms_sender = current_app.config['FROM_NUMBER']
+
     from app.dao.permissions_dao import permission_dao
     service.users.append(user)
     permission_dao.add_default_service_permissions_for_user(user, service)

app/models.py

@@ -146,8 +146,10 @@ class DVLAOrganisation(db.Model):
 
 INTERNATIONAL_SMS_TYPE = 'international_sms'
 INBOUND_SMS_TYPE = 'inbound_sms'
+SCHEDULE_NOTIFICATIONS = 'schedule_notifications'
 
-SERVICE_PERMISSION_TYPES = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE]
+SERVICE_PERMISSION_TYPES = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE,
+                            SCHEDULE_NOTIFICATIONS]
 
 
 class ServicePermissionTypes(db.Model):
@@ -188,7 +190,7 @@ class Service(db.Model, Versioned):
     created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
     reply_to_email_address = db.Column(db.Text, index=False, unique=False, nullable=True)
     letter_contact_block = db.Column(db.Text, index=False, unique=False, nullable=True)
-    sms_sender = db.Column(db.String(11), nullable=True, default=lambda: current_app.config['FROM_NUMBER'])
+    sms_sender = db.Column(db.String(11), nullable=False, default=lambda: current_app.config['FROM_NUMBER'])
     organisation_id = db.Column(UUID(as_uuid=True), db.ForeignKey('organisation.id'), index=True, nullable=True)
     organisation = db.relationship('Organisation')
     dvla_organisation_id = db.Column(
@@ -681,6 +683,8 @@ NOTIFICATION_STATUS_TYPES = [
     NOTIFICATION_PERMANENT_FAILURE,
 ]
 
+NOTIFICATION_STATUS_TYPES_NON_BILLABLE = list(set(NOTIFICATION_STATUS_TYPES) - set(NOTIFICATION_STATUS_TYPES_BILLABLE))
+
 NOTIFICATION_STATUS_TYPES_ENUM = db.Enum(*NOTIFICATION_STATUS_TYPES, name='notify_status_type')
 
 
@@ -977,7 +981,7 @@ INVITED_USER_STATUS_TYPES = ['pending', 'accepted', 'cancelled']
 class ScheduledNotification(db.Model):
     __tablename__ = 'scheduled_notifications'
 
-    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4())
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     notification_id = db.Column(UUID(as_uuid=True), db.ForeignKey('notifications.id'), index=True, nullable=False)
     notification = db.relationship('Notification', uselist=False)
     scheduled_for = db.Column(db.DateTime, index=False, nullable=False)
@@ -1097,6 +1101,12 @@ class Rate(db.Model):
     rate = db.Column(db.Float(asdecimal=False), nullable=False)
     notification_type = db.Column(notification_types, index=True, nullable=False)
 
+    def __str__(self):
+        the_string = "{}".format(self.rate)
+        the_string += " {}".format(self.notification_type)
+        the_string += " {}".format(self.valid_from)
+        return the_string
+
 
 class JobStatistics(db.Model):
     __tablename__ = 'job_statistics'
@@ -1143,6 +1153,29 @@ class JobStatistics(db.Model):
         return the_string
 
 
+class InboundSms(db.Model):
+    __tablename__ = 'inbound_sms'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
+    service = db.relationship('Service', backref='inbound_sms')
+
+    notify_number = db.Column(db.String, nullable=False)  # the service's number, that the msg was sent to
+    user_number = db.Column(db.String, nullable=False)  # the end user's number, that the msg was sent from
+    provider_date = db.Column(db.DateTime)
+    provider_reference = db.Column(db.String)
+    _content = db.Column('content', db.String, nullable=False)
+
+    @property
+    def content(self):
+        return encryption.decrypt(self._content)
+
+    @content.setter
+    def content(self, content):
+        self._content = encryption.encrypt(content)
+
+
 class LetterRate(db.Model):
     __tablename__ = 'letter_rates'
 

app/notifications/receive_notifications.py

@@ -1,8 +1,15 @@
-from flask import Blueprint
+from urllib.parse import unquote
-from flask import current_app
-from flask import request
 
+import iso8601
+from flask import jsonify, Blueprint, current_app, request
+from notifications_utils.recipients import normalise_phone_number
+
+from app import statsd_client
+from app.dao.services_dao import dao_fetch_services_by_sms_sender
+from app.dao.inbound_sms_dao import dao_create_inbound_sms
+from app.models import InboundSms
 from app.errors import register_errors
+from app.utils import convert_bst_to_utc
 
 receive_notifications_blueprint = Blueprint('receive_notifications', __name__)
 register_errors(receive_notifications_blueprint)
@@ -10,8 +17,77 @@ register_errors(receive_notifications_blueprint)
 
 @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST'])
 def receive_mmg_sms():
+    """
+    {
+        'MSISDN': '447123456789'
+        'Number': '40604',
+        'Message': 'some+uri+encoded+message%3A',
+        'ID': 'SOME-MMG-SPECIFIC-ID',
+        'DateRecieved': '2017-05-21+11%3A56%3A11'
+    }
+    """
     post_data = request.get_json()
-    post_data.pop('MSISDN', None)
+    potential_services = dao_fetch_services_by_sms_sender(post_data['Number'])
-    current_app.logger.info("Recieve notification form data: {}".format(post_data))
 
-    return "RECEIVED"
+    if len(potential_services) != 1:
+        current_app.logger.error('Inbound number "{}" not associated with exactly one service'.format(
+            post_data['Number']
+        ))
+        statsd_client.incr('inbound.mmg.failed')
+        # since this is an issue with our service <-> number mapping, we should still tell MMG that we received
+        # succesfully
+        return 'RECEIVED', 200
+
+    statsd_client.incr('inbound.mmg.succesful')
+
+    service = potential_services[0]
+
+    inbound = create_inbound_mmg_sms_object(service, post_data)
+
+    current_app.logger.info('{} received inbound SMS with reference {}'.format(service.id, inbound.provider_reference))
+
+    return 'RECEIVED', 200
+
+
+def format_mmg_message(message):
+    return unquote(message.replace('+', ' '))
+
+
+def format_mmg_datetime(date):
+    """
+    We expect datetimes in format 2017-05-21+11%3A56%3A11 - ie, spaces replaced with pluses, and URI encoded
+    (the same as UTC)
+    """
+    orig_date = format_mmg_message(date)
+    parsed_datetime = iso8601.parse_date(orig_date).replace(tzinfo=None)
+    return convert_bst_to_utc(parsed_datetime)
+
+
+def create_inbound_mmg_sms_object(service, json):
+    message = format_mmg_message(json['Message'])
+    user_number = normalise_phone_number(json['MSISDN'])
+
+    provider_date = json.get('DateRecieved')
+    if provider_date:
+        provider_date = format_mmg_datetime(provider_date)
+
+    inbound = InboundSms(
+        service=service,
+        notify_number=service.sms_sender,
+        user_number=user_number,
+        provider_date=provider_date,
+        provider_reference=json.get('ID'),
+        content=message,
+    )
+    dao_create_inbound_sms(inbound)
+    return inbound
+
+
+@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST'])
+def receive_firetext_sms():
+    post_data = request.form
+    current_app.logger.info("Received Firetext notification form data: {}".format(post_data))
+
+    return jsonify({
+        "status": "ok"
+    }), 200

app/notifications/validators.py

@@ -6,7 +6,7 @@ from notifications_utils.recipients import (
 )
 
 from app.dao import services_dao
-from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE
+from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE, SCHEDULE_NOTIFICATIONS
 from app.service.utils import service_allowed_to_send_to
 from app.v2.errors import TooManyRequestsError, BadRequestError, RateLimitError
 from app import redis_store
@@ -92,6 +92,7 @@ def check_sms_content_char_count(content_count):
         raise BadRequestError(message=message)
 
 
-def service_can_schedule_notification(service):
+def service_can_schedule_notification(service, scheduled_for):
-    # TODO: implement once the service permission works.
+    if scheduled_for:
-    raise BadRequestError(message="Your service must be invited to schedule notifications via the API.")
+        if SCHEDULE_NOTIFICATIONS not in [p.permission for p in service.permissions]:
+            raise BadRequestError(message="Cannot schedule notifications (this feature is invite-only)")

app/service/rest.py

@@ -10,6 +10,7 @@ from flask import (
 )
 from sqlalchemy.orm.exc import NoResultFound
 
+from app import redis_store
 from app.dao import notification_usage_dao, notifications_dao
 from app.dao.dao_utils import dao_rollback
 from app.dao.api_key_dao import (
@@ -17,6 +18,8 @@ from app.dao.api_key_dao import (
     get_model_api_keys,
     get_unsigned_secret,
     expire_api_key)
+from app.dao.date_util import get_financial_year
+from app.dao.notification_usage_dao import get_total_billable_units_for_sent_sms_notifications_in_date_range
 from app.dao.services_dao import (
     dao_fetch_service_by_id,
     dao_fetch_all_services,
@@ -60,6 +63,7 @@ from app.schemas import (
     detailed_service_schema
 )
 from app.utils import pagination_links
+from notifications_utils.clients.redis import sms_billable_units_cache_key
 
 service_blueprint = Blueprint('service', __name__)
 
@@ -447,6 +451,39 @@ def get_monthly_template_stats(service_id):
         raise InvalidRequest('Year must be a number', status_code=400)
 
 
+@service_blueprint.route('/<uuid:service_id>/yearly-sms-billable-units')
+def get_yearly_sms_billable_units(service_id):
+    cache_key = sms_billable_units_cache_key(service_id)
+    cached_billable_sms_units = redis_store.get_all_from_hash(cache_key)
+    if cached_billable_sms_units:
+        return jsonify({
+            'billable_sms_units': int(cached_billable_sms_units[b'billable_units']),
+            'total_cost': float(cached_billable_sms_units[b'total_cost'])
+        })
+    else:
+        try:
+            start_date, end_date = get_financial_year(int(request.args.get('year')))
+        except (ValueError, TypeError) as e:
+            current_app.logger.exception(e)
+            return jsonify(result='error', message='No valid year provided'), 400
+
+        billable_units, total_cost = get_total_billable_units_for_sent_sms_notifications_in_date_range(
+            start_date,
+            end_date,
+            service_id)
+
+        cached_values = {
+            'billable_units': billable_units,
+            'total_cost': total_cost
+        }
+
+        redis_store.set_hash_and_expire(cache_key, cached_values, expire_in_seconds=60)
+        return jsonify({
+            'billable_sms_units': billable_units,
+            'total_cost': total_cost
+        })
+
+
 @service_blueprint.route('/<uuid:service_id>/yearly-usage')
 def get_yearly_billing_usage(service_id):
     try:

app/v2/notifications/notification_schemas.py

@@ -113,7 +113,7 @@ post_sms_request = {
         "phone_number": {"type": "string", "format": "phone_number"},
         "template_id": uuid,
         "personalisation": personalisation,
-        "scheduled_for": {"type": "string", "format": "datetime"}
+        "scheduled_for": {"type": ["string", "null"], "format": "datetime"}
     },
     "required": ["phone_number", "template_id"]
 }
@@ -141,7 +141,7 @@ post_sms_response = {
         "content": sms_content,
         "uri": {"type": "string", "format": "uri"},
         "template": template,
-        "scheduled_for": {"type": "string"}
+        "scheduled_for": {"type": ["string", "null"]}
     },
     "required": ["id", "content", "uri", "template"]
 }
@@ -157,7 +157,7 @@ post_email_request = {
         "email_address": {"type": "string", "format": "email_address"},
         "template_id": uuid,
         "personalisation": personalisation,
-        "scheduled_for": {"type": "string", "format": "datetime"}
+        "scheduled_for": {"type": ["string", "null"], "format": "datetime"}
     },
     "required": ["email_address", "template_id"]
 }
@@ -186,7 +186,7 @@ post_email_response = {
         "content": email_content,
         "uri": {"type": "string", "format": "uri"},
         "template": template,
-        "scheduled_for": {"type": "string"}
+        "scheduled_for": {"type": ["string", "null"]}
     },
     "required": ["id", "content", "uri", "template"]
 }

app/v2/notifications/post_notifications.py

@@ -35,9 +35,8 @@ def post_notification(notification_type):
         form = validate(request.get_json(), post_sms_request)
 
     scheduled_for = form.get("scheduled_for", None)
-    if scheduled_for:
+    service_can_schedule_notification(authenticated_service, scheduled_for)
-        if not service_can_schedule_notification(authenticated_service):
+
-            return
     check_rate_limiting(authenticated_service, api_user)
 
     form_send_to = form['phone_number'] if notification_type == SMS_TYPE else form['email_address']

manifest-api-preview.yml

@@ -8,4 +8,4 @@ routes:
   - route: api.notify.works
 
 instances: 1
-memory: 256M
+memory: 1G

manifest-delivery-base.yml

@@ -23,34 +23,32 @@ applications:
       NOTIFY_APP_NAME: delivery-celery-beat
 
   - name: notify-delivery-worker-database
-    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q db-sms,db-email,db-letter,database-tasks
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q database-tasks
-    memory: 1G
     env:
       NOTIFY_APP_NAME: delivery-worker-database
 
   - name: notify-delivery-worker-research
-    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode,research-mode-tasks
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q research-mode-tasks
     env:
       NOTIFY_APP_NAME: delivery-worker-research
 
   - name: notify-delivery-worker-sender
-    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-sms,send-email,send-tasks
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q send-tasks
-    memory: 1G
     env:
       NOTIFY_APP_NAME: delivery-worker-sender
 
   - name: notify-delivery-worker-periodic
-    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic,statistics,periodic-tasks,statistics-tasks
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=2 -Q periodic-tasks,statistics-tasks
     instances: 1
     env:
       NOTIFY_APP_NAME: delivery-worker-periodic
 
   - name: notify-delivery-worker-priority
-    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority,priority-tasks
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=5 -Q priority-tasks
     env:
       NOTIFY_APP_NAME: delivery-worker-priority
 
   - name: notify-delivery-worker
-    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q process-job,notify,retry,job-tasks,retry-tasks,notify-internal-tasks
+    command: scripts/run_app_paas.sh celery -A aws_run_celery.notify_celery worker --loglevel=INFO --concurrency=11 -Q job-tasks,retry-tasks,notify-internal-tasks
     env:
       NOTIFY_APP_NAME: delivery-worker

manifest-delivery-preview.yml

@@ -1,4 +1,4 @@
 ---
 
 inherit: manifest-delivery-base.yml
-memory: 256M
+memory: 1G

manifest-delivery-production.yml

@@ -3,4 +3,4 @@
 inherit: manifest-delivery-base.yml
 
 instances: 2
-memory: 768M
+memory: 1G

manifest-delivery-staging.yml

@@ -3,4 +3,4 @@
 inherit: manifest-delivery-base.yml
 
 instances: 2
-memory: 768M
+memory: 1G

migrations/versions/0088_add_schedule_serv_perm.py

@@ -0,0 +1,24 @@
+"""empty message
+
+Revision ID: 0088_add_schedule_serv_perm
+Revises: 0087_scheduled_notifications
+Create Date: 2017-05-26 14:53:18.581320
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0088_add_schedule_serv_perm'
+down_revision = '0087_scheduled_notifications'
+
+from alembic import op
+
+
+def upgrade():
+    op.get_bind()
+    op.execute("insert into service_permission_types values('schedule_notifications')")
+
+
+def downgrade():
+    op.get_bind()
+    op.execute("delete from service_permissions where permission = 'schedule_notifications'")
+    op.execute("delete from service_permission_types where name = 'schedule_notifications'")

migrations/versions/0089_govuk_sms_sender.py

@@ -0,0 +1,25 @@
+"""empty message
+
+Revision ID: 0089_govuk_sms_sender
+Revises: 0088_add_schedule_serv_perm
+Create Date: 2017-05-22 13:46:09.584801
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0089_govuk_sms_sender'
+down_revision = '0088_add_schedule_serv_perm'
+
+from alembic import op
+
+
+def upgrade():
+    op.execute("UPDATE services SET sms_sender = 'GOVUK' where sms_sender is null")
+    op.execute("UPDATE services_history SET sms_sender = 'GOVUK' where sms_sender is null")
+    op.alter_column('services', 'sms_sender', nullable=False)
+    op.alter_column('services_history', 'sms_sender', nullable=False)
+
+
+def downgrade():
+    op.alter_column('services_history', 'sms_sender', nullable=True)
+    op.alter_column('services', 'sms_sender', nullable=True)

migrations/versions/0090_inbound_sms.py

@@ -0,0 +1,37 @@
+"""empty message
+
+Revision ID: 0090_inbound_sms
+Revises: 0089_govuk_sms_sender
+Create Date: 2017-05-22 11:28:53.471004
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0090_inbound_sms'
+down_revision = '0089_govuk_sms_sender'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    op.create_table(
+        'inbound_sms',
+        sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('content', sa.String, nullable=False),
+        sa.Column('notify_number', sa.String, nullable=False),
+        sa.Column('user_number', sa.String, nullable=False),
+        sa.Column('created_at', sa.DateTime, nullable=False),
+        sa.Column('provider_date', sa.DateTime, nullable=True),
+        sa.Column('provider_reference', sa.String, nullable=True),
+
+        sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_inbound_sms_service_id'), 'inbound_sms', ['service_id'], unique=False)
+    op.create_index(op.f('ix_inbound_sms_user_number'), 'inbound_sms', ['user_number'], unique=False)
+
+
+def downgrade():
+    op.drop_table('inbound_sms')

migrations/versions/0091_letter_billing.py

@@ -1,7 +1,7 @@
 """empty message
 
-Revision ID: 0088_letter_billing
+Revision ID: 0091_letter_billing
-Revises: 0087_scheduled_notifications
+Revises: 0090_inbound_sms
 Create Date: 2017-05-31 11:43:55.744631
 
 """
@@ -10,8 +10,8 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
-revision = '0088_letter_billing'
+revision = '0091_letter_billing'
-down_revision = '0087_scheduled_notifications'
+down_revision = '0090_inbound_sms'
 
 
 def upgrade():

requirements.txt

@@ -29,6 +29,6 @@ notifications-python-client>=3.1,<3.2
 awscli>=1.11,<1.12
 awscli-cwlogs>=1.4,<1.5
 
-git+https://github.com/alphagov/notifications-utils.git@17.1.0#egg=notifications-utils==17.1.0
+git+https://github.com/alphagov/notifications-utils.git@17.1.3#egg=notifications-utils==17.1.3
 
 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3

tests/app/dao/test_jobs_dao.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta
 from functools import partial
+import pytest
 import uuid
 
 from freezegun import freeze_time
@@ -334,13 +335,24 @@ def test_get_jobs_for_service_is_paginated(notify_db, notify_db_session, sample_
     assert res.items[1].created_at == datetime(2015, 1, 1, 7)
 
 
-def test_get_jobs_for_service_doesnt_return_test_messages(notify_db, notify_db_session, sample_template, sample_job):
+@pytest.mark.parametrize('file_name', [
+    'Test message',
+    'Report',
+])
+def test_get_jobs_for_service_doesnt_return_test_messages(
+    notify_db,
+    notify_db_session,
+    sample_template,
+    sample_job,
+    file_name,
+):
     test_job = create_job(
         notify_db,
         notify_db_session,
         sample_template.service,
         sample_template,
-        original_file_name='Test message')
+        original_file_name=file_name,
+    )
 
     jobs = dao_get_jobs_by_service_id(sample_job.service_id).items
 

tests/app/dao/test_notification_usage_dao.py

@@ -1,11 +1,25 @@
 import uuid
-from datetime import datetime
+from datetime import datetime, timedelta
+
+import pytest
 
 from app.dao.date_util import get_financial_year
-from app.dao.notification_usage_dao import (get_rates_for_year, get_yearly_billing_data,
+from app.dao.notification_usage_dao import (
-                                            get_monthly_billing_data)
+    get_rates_for_year,
-from app.models import Rate
+    get_yearly_billing_data,
+    get_monthly_billing_data,
+    get_total_billable_units_for_sent_sms_notifications_in_date_range,
+    discover_rate_bounds_for_billing_query
+)
+from app.models import (
+    Rate,
+    NOTIFICATION_DELIVERED,
+    NOTIFICATION_STATUS_TYPES_BILLABLE,
+    NOTIFICATION_STATUS_TYPES_NON_BILLABLE,
+    Notification)
+from tests.app.conftest import sample_notification, sample_email_template, sample_letter_template, sample_service
 from tests.app.db import create_notification
+from freezegun import freeze_time
 
 
 def test_get_rates_for_year(notify_db, notify_db_session):
@@ -248,3 +262,345 @@ def test_get_monthly_billing_data_with_no_notifications_for_year(notify_db, noti
 def set_up_rate(notify_db, start_date, value):
     rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type='sms')
     notify_db.session.add(rate)
+
+
+@freeze_time("2016-01-10 12:00:00.000000")
+def test_returns_total_billable_units_for_sms_notifications(notify_db, notify_db_session, sample_service):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 0.016)
+
+    sample_notification(
+        notify_db, notify_db_session, service=sample_service, billable_units=1, status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db, notify_db_session, service=sample_service, billable_units=2, status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db, notify_db_session, service=sample_service, billable_units=3, status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db, notify_db_session, service=sample_service, billable_units=4, status=NOTIFICATION_DELIVERED)
+
+    start = datetime.utcnow() - timedelta(minutes=10)
+    end = datetime.utcnow() + timedelta(minutes=10)
+
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 10
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 0.16
+
+
+@freeze_time("2016-01-10 12:00:00.000000")
+def test_returns_total_billable_units_multiplied_by_multipler_for_sms_notifications(
+        notify_db, notify_db_session, sample_service
+):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 2.5)
+
+    sample_notification(
+        notify_db, notify_db_session, service=sample_service, rate_multiplier=1.0, status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db, notify_db_session, service=sample_service, rate_multiplier=2.0, status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db, notify_db_session, service=sample_service, rate_multiplier=5.0, status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db, notify_db_session, service=sample_service, rate_multiplier=10.0, status=NOTIFICATION_DELIVERED)
+
+    start = datetime.utcnow() - timedelta(minutes=10)
+    end = datetime.utcnow() + timedelta(minutes=10)
+
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 18
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 45
+
+
+def test_returns_total_billable_units_multiplied_by_multipler_for_sms_notifications_for_several_rates(
+        notify_db, notify_db_session, sample_service
+):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 2)
+    set_up_rate(notify_db, datetime(2016, 10, 1), 4)
+    set_up_rate(notify_db, datetime(2017, 1, 1), 6)
+
+    eligble_rate_1 = datetime(2016, 2, 1)
+    eligble_rate_2 = datetime(2016, 11, 1)
+    eligble_rate_3 = datetime(2017, 2, 1)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=sample_service,
+        rate_multiplier=1.0,
+        status=NOTIFICATION_DELIVERED,
+        created_at=eligble_rate_1)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=sample_service,
+        rate_multiplier=2.0,
+        status=NOTIFICATION_DELIVERED,
+        created_at=eligble_rate_2)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=sample_service,
+        rate_multiplier=5.0,
+        status=NOTIFICATION_DELIVERED,
+        created_at=eligble_rate_3)
+
+    start = datetime(2016, 1, 1)
+    end = datetime(2018, 1, 1)
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 8
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 40
+
+
+def test_returns_total_billable_units_for_sms_notifications_for_several_rates_where_dates_match_rate_boundary(
+        notify_db, notify_db_session, sample_service
+):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 2)
+    set_up_rate(notify_db, datetime(2016, 10, 1), 4)
+    set_up_rate(notify_db, datetime(2017, 1, 1), 6)
+
+    eligble_rate_1_start = datetime(2016, 1, 1, 0, 0, 0, 0)
+    eligble_rate_1_end = datetime(2016, 9, 30, 23, 59, 59, 999)
+    eligble_rate_2_start = datetime(2016, 10, 1, 0, 0, 0, 0)
+    eligble_rate_2_end = datetime(2016, 12, 31, 23, 59, 59, 999)
+    eligble_rate_3_start = datetime(2017, 1, 1, 0, 0, 0, 0)
+    eligble_rate_3_whenever = datetime(2017, 12, 12, 0, 0, 0, 0)
+
+    def make_notification(created_at):
+        sample_notification(
+            notify_db,
+            notify_db_session,
+            service=sample_service,
+            rate_multiplier=1.0,
+            status=NOTIFICATION_DELIVERED,
+            created_at=created_at)
+
+    make_notification(eligble_rate_1_start)
+    make_notification(eligble_rate_1_end)
+    make_notification(eligble_rate_2_start)
+    make_notification(eligble_rate_2_end)
+    make_notification(eligble_rate_3_start)
+    make_notification(eligble_rate_3_whenever)
+
+    start = datetime(2016, 1, 1)
+    end = datetime(2018, 1, 1)
+
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 6
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 24.0
+
+
+@freeze_time("2016-01-10 12:00:00.000000")
+def test_returns_total_billable_units_for_sms_notifications_ignoring_letters_and_emails(
+        notify_db, notify_db_session, sample_service
+):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 2.5)
+
+    email_template = sample_email_template(notify_db, notify_db_session, service=sample_service)
+    letter_template = sample_letter_template(sample_service)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=sample_service,
+        billable_units=2,
+        status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        template=email_template,
+        service=sample_service,
+        billable_units=2,
+        status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        template=letter_template,
+        service=sample_service,
+        billable_units=2,
+        status=NOTIFICATION_DELIVERED
+    )
+
+    start = datetime.utcnow() - timedelta(minutes=10)
+    end = datetime.utcnow() + timedelta(minutes=10)
+
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 2
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 5
+
+
+@freeze_time("2016-01-10 12:00:00.000000")
+def test_returns_total_billable_units_for_sms_notifications_for_only_requested_service(
+        notify_db, notify_db_session
+):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 2.5)
+
+    service_1 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4()))
+    service_2 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4()))
+    service_3 = sample_service(notify_db, notify_db_session, service_name=str(uuid.uuid4()))
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=service_1,
+        billable_units=2,
+        status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=service_2,
+        billable_units=2,
+        status=NOTIFICATION_DELIVERED)
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=service_3,
+        billable_units=2,
+        status=NOTIFICATION_DELIVERED
+    )
+
+    start = datetime.utcnow() - timedelta(minutes=10)
+    end = datetime.utcnow() + timedelta(minutes=10)
+
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, service_1.id)[0] == 2
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, service_1.id)[1] == 5
+
+
+@freeze_time("2016-01-10 12:00:00.000000")
+def test_returns_total_billable_units_for_sms_notifications_handling_null_values(
+    notify_db, notify_db_session, sample_service
+):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 2.5)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=sample_service,
+        billable_units=2,
+        rate_multiplier=None,
+        status=NOTIFICATION_DELIVERED)
+
+    start = datetime.utcnow() - timedelta(minutes=10)
+    end = datetime.utcnow() + timedelta(minutes=10)
+
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 2
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 5
+
+
+@pytest.mark.parametrize('billable_units, states', ([
+    (len(NOTIFICATION_STATUS_TYPES_BILLABLE), NOTIFICATION_STATUS_TYPES_BILLABLE),
+    (0, NOTIFICATION_STATUS_TYPES_NON_BILLABLE)
+]))
+@freeze_time("2016-01-10 12:00:00.000000")
+def test_ignores_non_billable_states_when_returning_billable_units_for_sms_notifications(
+    notify_db, notify_db_session, sample_service, billable_units, states
+):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 2.5)
+
+    for state in states:
+        sample_notification(
+            notify_db,
+            notify_db_session,
+            service=sample_service,
+            billable_units=1,
+            rate_multiplier=None,
+            status=state)
+
+    start = datetime.utcnow() - timedelta(minutes=10)
+    end = datetime.utcnow() + timedelta(minutes=10)
+
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(
+        start, end, sample_service.id
+    )[0] == billable_units
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(
+        start, end, sample_service.id
+    )[1] == billable_units * 2.5
+
+
+@freeze_time("2016-01-10 12:00:00.000000")
+def test_restricts_to_time_period_when_returning_billable_units_for_sms_notifications(
+    notify_db, notify_db_session, sample_service
+):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 2.5)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=sample_service,
+        billable_units=1,
+        rate_multiplier=1.0,
+        created_at=datetime.utcnow() - timedelta(minutes=100),
+        status=NOTIFICATION_DELIVERED)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        service=sample_service,
+        billable_units=1,
+        rate_multiplier=1.0,
+        created_at=datetime.utcnow() - timedelta(minutes=5),
+        status=NOTIFICATION_DELIVERED)
+
+    start = datetime.utcnow() - timedelta(minutes=10)
+    end = datetime.utcnow() + timedelta(minutes=10)
+
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 1
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 2.5
+
+
+def test_returns_zero_if_no_matching_rows_when_returning_billable_units_for_sms_notifications(
+        notify_db, notify_db_session, sample_service
+):
+    set_up_rate(notify_db, datetime(2016, 1, 1), 2.5)
+
+    start = datetime.utcnow() - timedelta(minutes=10)
+    end = datetime.utcnow() + timedelta(minutes=10)
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[0] == 0
+    assert get_total_billable_units_for_sent_sms_notifications_in_date_range(start, end, sample_service.id)[1] == 0.0
+
+
+def test_should_calculate_rate_boundaries_for_billing_query_for_single_relevant_rate(notify_db, notify_db_session):
+    start_date, end_date = get_financial_year(2016)
+    set_up_rate(notify_db, datetime(2016, 1, 1), 0.016)
+    rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date)
+    assert len(rate_boundaries) == 1
+    assert rate_boundaries[0]['start_date'] == start_date
+    assert rate_boundaries[0]['end_date'] == end_date
+    assert rate_boundaries[0]['rate'] == 0.016
+
+
+def test_should_calculate_rate_boundaries_for_billing_query_for_two_relevant_rates(notify_db, notify_db_session):
+    start_date, end_date = get_financial_year(2016)
+
+    rate_1_valid_from = datetime(2016, 1, 1)
+    rate_2_valid_from = datetime(2017, 1, 1)
+
+    set_up_rate(notify_db, rate_1_valid_from, 0.02)
+    set_up_rate(notify_db, rate_2_valid_from, 0.04)
+    rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date)
+    assert len(rate_boundaries) == 2
+    assert rate_boundaries[0]['start_date'] == start_date
+    assert rate_boundaries[0]['end_date'] == rate_2_valid_from
+    assert rate_boundaries[0]['rate'] == 0.02
+
+    assert rate_boundaries[1]['start_date'] == rate_2_valid_from
+    assert rate_boundaries[1]['end_date'] == end_date
+    assert rate_boundaries[1]['rate'] == 0.04
+
+
+def test_should_calculate_rate_boundaries_for_billing_query_for_three_relevant_rates(notify_db, notify_db_session):
+    start_date, end_date = get_financial_year(2016)
+    rate_1_valid_from = datetime(2016, 1, 1)
+    rate_2_valid_from = datetime(2017, 1, 1)
+    rate_3_valid_from = datetime(2017, 2, 1)
+
+    set_up_rate(notify_db, rate_1_valid_from, 0.02)
+    set_up_rate(notify_db, rate_2_valid_from, 0.04)
+    set_up_rate(notify_db, rate_3_valid_from, 0.06)
+    rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date)
+    assert len(rate_boundaries) == 3
+
+    assert rate_boundaries[0]['start_date'] == start_date
+    assert rate_boundaries[0]['end_date'] == rate_2_valid_from
+    assert rate_boundaries[0]['rate'] == 0.02
+
+    assert rate_boundaries[1]['start_date'] == rate_2_valid_from
+    assert rate_boundaries[1]['end_date'] == rate_3_valid_from
+    assert rate_boundaries[1]['rate'] == 0.04
+
+    assert rate_boundaries[2]['start_date'] == rate_3_valid_from
+    assert rate_boundaries[2]['end_date'] == end_date
+    assert rate_boundaries[2]['rate'] == 0.06

tests/app/dao/test_services_dao.py

@@ -25,7 +25,8 @@ from app.dao.services_dao import (
     fetch_stats_by_date_range_for_all_services,
     dao_suspend_service,
     dao_resume_service,
-    dao_fetch_active_users_for_service
+    dao_fetch_active_users_for_service,
+    dao_fetch_services_by_sms_sender
 )
 from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission
 from app.dao.users_dao import save_model_user
@@ -948,3 +949,13 @@ def test_dao_fetch_active_users_for_service_returns_active_only(notify_db, notif
     users = dao_fetch_active_users_for_service(service.id)
 
     assert len(users) == 1
+
+
+def test_dao_fetch_services_by_sms_sender(notify_db_session):
+    foo1 = create_service(service_name='a', sms_sender='foo')
+    foo2 = create_service(service_name='b', sms_sender='foo')
+    bar = create_service(service_name='c', sms_sender='bar')
+
+    services = dao_fetch_services_by_sms_sender('foo')
+
+    assert {foo1.id, foo2.id} == {x.id for x in services}

tests/app/db.py

@@ -39,14 +39,20 @@ def create_user(mobile_number="+447700900986", email="notify@digital.cabinet-off
 
 
 def create_service(
-        user=None, service_name="Sample service", service_id=None, restricted=False,
+    user=None,
-        service_permissions=[EMAIL_TYPE, SMS_TYPE]):
+    service_name="Sample service",
+    service_id=None,
+    restricted=False,
+    service_permissions=[EMAIL_TYPE, SMS_TYPE],
+    sms_sender='testing'
+):
     service = Service(
         name=service_name,
         message_limit=1000,
         restricted=restricted,
         email_from=service_name.lower().replace(' ', '.'),
-        created_by=user or create_user()
+        created_by=user or create_user(),
+        sms_sender=sms_sender
     )
     dao_create_service(service, service.created_by, service_id, service_permissions=service_permissions)
     return service

tests/app/delivery/test_send_to_providers.py

@@ -627,7 +627,6 @@ def test_should_set_international_phone_number_to_sent_status(
     # if 40604 is actually in DB then treat that as if entered manually
     ('40604', '40604', 'bar'),
     # 'testing' is the FROM_NUMBER during unit tests
-    (None, 'testing', 'Sample service: bar'),
     ('testing', 'testing', 'Sample service: bar'),
 ])
 def test_should_handle_sms_sender_and_prefix_message(

tests/app/notifications/test_receive_notification.py

@@ -1,14 +1,26 @@
+from datetime import datetime
+
+import pytest
 from flask import json
 
+from app.notifications.receive_notifications import (
+    format_mmg_message,
+    format_mmg_datetime,
+    create_inbound_mmg_sms_object
+)
 
-def test_receive_notification_returns_received_to_mmg(client):
+from app.models import InboundSms
+from tests.app.db import create_service
+
+
+def test_receive_notification_returns_received_to_mmg(client, sample_service):
     data = {"ID": "1234",
             "MSISDN": "447700900855",
             "Message": "Some message to notify",
             "Trigger": "Trigger?",
-            "Number": "40604",
+            "Number": "testing",
             "Channel": "SMS",
-            "DateReceived": "2012-06-27-12:33:00"
+            "DateRecieved": "2012-06-27 12:33:00"
             }
     response = client.post(path='/notifications/sms/receive/mmg',
                            data=json.dumps(data),
@@ -16,3 +28,79 @@ def test_receive_notification_returns_received_to_mmg(client):
 
     assert response.status_code == 200
     assert response.get_data(as_text=True) == 'RECEIVED'
+
+
+@pytest.mark.parametrize('message, expected_output', [
+    ('abc', 'abc'),
+    ('', ''),
+    ('lots+of+words', 'lots of words'),
+    ('%F0%9F%93%A9+%F0%9F%93%A9+%F0%9F%93%A9', '📩 📩 📩'),
+    ('x+%2B+y', 'x + y')
+])
+def test_format_mmg_message(message, expected_output):
+    assert format_mmg_message(message) == expected_output
+
+
+@pytest.mark.parametrize('provider_date, expected_output', [
+    ('2017-01-21+11%3A56%3A11', datetime(2017, 1, 21, 11, 56, 11)),
+    ('2017-05-21+11%3A56%3A11', datetime(2017, 5, 21, 10, 56, 11))
+])
+def test_format_mmg_datetime(provider_date, expected_output):
+    assert format_mmg_datetime(provider_date) == expected_output
+
+
+def test_create_inbound_mmg_sms_object(sample_service):
+    sample_service.sms_sender = 'foo'
+    data = {
+        'Message': 'hello+there+%F0%9F%93%A9',
+        'Number': 'foo',
+        'MSISDN': '07700 900 001',
+        'DateRecieved': '2017-01-02+03%3A04%3A05',
+        'ID': 'bar',
+    }
+
+    inbound_sms = create_inbound_mmg_sms_object(sample_service, data)
+
+    assert inbound_sms.service_id == sample_service.id
+    assert inbound_sms.notify_number == 'foo'
+    assert inbound_sms.user_number == '7700900001'
+    assert inbound_sms.provider_date == datetime(2017, 1, 2, 3, 4, 5)
+    assert inbound_sms.provider_reference == 'bar'
+    assert inbound_sms._content != 'hello there 📩'
+    assert inbound_sms.content == 'hello there 📩'
+
+
+@pytest.mark.parametrize('notify_number', ['foo', 'baz'], ids=['two_matching_services', 'no_matching_services'])
+def test_receive_notification_error_if_not_single_matching_service(client, notify_db_session, notify_number):
+    create_service(service_name='a', sms_sender='foo')
+    create_service(service_name='b', sms_sender='foo')
+
+    data = {
+        'Message': 'hello',
+        'Number': notify_number,
+        'MSISDN': '7700900001',
+        'DateRecieved': '2017-01-02 03:04:05',
+        'ID': 'bar',
+    }
+    response = client.post(path='/notifications/sms/receive/mmg',
+                           data=json.dumps(data),
+                           headers=[('Content-Type', 'application/json')])
+
+    # we still return 'RECEIVED' to MMG
+    assert response.status_code == 200
+    assert response.get_data(as_text=True) == 'RECEIVED'
+    assert InboundSms.query.count() == 0
+
+
+def test_receive_notification_returns_received_to_firetext(client):
+    data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"
+
+    response = client.post(
+        path='/notifications/sms/receive/firetext',
+        data=data,
+        headers=[('Content-Type', 'application/x-www-form-urlencoded')])
+
+    assert response.status_code == 200
+    result = json.loads(response.get_data(as_text=True))
+
+    assert result['status'] == 'ok'

tests/app/service/test_rest.py

@@ -64,9 +64,9 @@ def test_get_service_list_with_only_active_flag(client, service_factory):
 
 
 def test_get_service_list_with_user_id_and_only_active_flag(
-    client,
+        client,
-    sample_user,
+        sample_user,
-    service_factory
+        service_factory
 ):
     other_user = create_user(email='foo@bar.gov.uk')
 
@@ -850,7 +850,6 @@ def test_add_existing_user_to_another_service_with_all_permissions(notify_api,
                                                                    sample_user):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-
             # check which users part of service
             user_already_in_service = sample_service.users[0]
             auth_header = create_authorization_header()
@@ -925,7 +924,6 @@ def test_add_existing_user_to_another_service_with_send_permissions(notify_api,
                                                                     sample_user):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-
             # they must exist in db first
             user_to_add = User(
                 name='Invited User',
@@ -969,7 +967,6 @@ def test_add_existing_user_to_another_service_with_manage_permissions(notify_api
                                                                       sample_user):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-
             # they must exist in db first
             user_to_add = User(
                 name='Invited User',
@@ -1013,7 +1010,6 @@ def test_add_existing_user_to_another_service_with_manage_api_keys(notify_api,
                                                                    sample_user):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-
             # they must exist in db first
             user_to_add = User(
                 name='Invited User',
@@ -1054,7 +1050,6 @@ def test_add_existing_user_to_non_existing_service_returns404(notify_api,
                                                               sample_user):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-
             user_to_add = User(
                 name='Invited User',
                 email_address='invited@digital.cabinet-office.gov.uk',
@@ -1085,7 +1080,6 @@ def test_add_existing_user_to_non_existing_service_returns404(notify_api,
 def test_add_existing_user_of_service_to_service_returns400(notify_api, notify_db, notify_db_session, sample_service):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-
             existing_user_id = sample_service.users[0].id
 
             data = {'permissions': ['send_messages', 'manage_service', 'manage_api_keys']}
@@ -1108,7 +1102,6 @@ def test_add_existing_user_of_service_to_service_returns400(notify_api, notify_d
 def test_add_unknown_user_to_service_returns404(notify_api, notify_db, notify_db_session, sample_service):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-
             incorrect_id = 9876
 
             data = {'permissions': ['send_messages', 'manage_service', 'manage_api_keys']}
@@ -1129,7 +1122,7 @@ def test_add_unknown_user_to_service_returns404(notify_api, notify_db, notify_db
 
 
 def test_remove_user_from_service(
-    notify_db, notify_db_session, client, sample_user_service_permission
+        notify_db, notify_db_session, client, sample_user_service_permission
 ):
     second_user = create_user(email="new@digital.cabinet-office.gov.uk")
     # Simulates successfully adding a user to the service
@@ -1149,7 +1142,7 @@ def test_remove_user_from_service(
 
 
 def test_remove_non_existant_user_from_service(
-    client, sample_user_service_permission
+        client, sample_user_service_permission
 ):
     second_user = create_user(email="new@digital.cabinet-office.gov.uk")
     endpoint = url_for(
@@ -1185,13 +1178,11 @@ def test_cannot_remove_only_user_from_service(notify_api,
 # This test is just here verify get_service_and_api_key_history that is a temp solution
 # until proper ui is sorted out on admin app
 def test_get_service_and_api_key_history(notify_api, notify_db, notify_db_session, sample_service):
-
     from tests.app.conftest import sample_api_key as create_sample_api_key
     api_key = create_sample_api_key(notify_db, notify_db_session, service=sample_service)
 
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
-
             auth_header = create_authorization_header()
             response = client.get(
                 path='/service/{}/history'.format(sample_service.id),
@@ -1265,12 +1256,12 @@ def test_get_all_notifications_for_service_in_order(notify_api, notify_db, notif
     ]
 )
 def test_get_all_notifications_for_service_including_ones_made_by_jobs(
-    client,
+        client,
-    notify_db,
+        notify_db,
-    notify_db_session,
+        notify_db_session,
-    sample_service,
+        sample_service,
-    include_from_test_key,
+        include_from_test_key,
-    expected_count_of_notifications
+        expected_count_of_notifications
 ):
     with_job = sample_notification_with_job(notify_db, notify_db_session, service=sample_service)
     without_job = create_sample_notification(notify_db, notify_db_session, service=sample_service)
@@ -1295,10 +1286,10 @@ def test_get_all_notifications_for_service_including_ones_made_by_jobs(
 
 
 def test_get_only_api_created_notifications_for_service(
-    client,
+        client,
-    notify_db,
+        notify_db,
-    notify_db_session,
+        notify_db_session,
-    sample_service
+        sample_service
 ):
     with_job = sample_notification_with_job(notify_db, notify_db_session, service=sample_service)
     without_job = create_sample_notification(notify_db, notify_db_session, service=sample_service)
@@ -1315,61 +1306,57 @@ def test_get_only_api_created_notifications_for_service(
     assert response.status_code == 200
 
 
-def test_set_sms_sender_for_service(notify_api, sample_service):
+def test_set_sms_sender_for_service(client, sample_service):
-    with notify_api.test_request_context():
+    data = {
-        with notify_api.test_client() as client:
+        'sms_sender': 'elevenchars',
-            auth_header = create_authorization_header()
+    }
-            resp = client.get(
-                '/service/{}'.format(sample_service.id),
-                headers=[auth_header]
-            )
-            json_resp = json.loads(resp.get_data(as_text=True))
-            assert resp.status_code == 200
-            assert json_resp['data']['name'] == sample_service.name
 
-            data = {
+    auth_header = create_authorization_header()
-                'sms_sender': 'elevenchars',
-            }
 
-            auth_header = create_authorization_header()
+    resp = client.post(
-
+        '/service/{}'.format(sample_service.id),
-            resp = client.post(
+        data=json.dumps(data),
-                '/service/{}'.format(sample_service.id),
+        headers=[('Content-Type', 'application/json'), auth_header]
-                data=json.dumps(data),
+    )
-                headers=[('Content-Type', 'application/json'), auth_header]
+    result = json.loads(resp.get_data(as_text=True))
-            )
+    assert resp.status_code == 200
-            result = json.loads(resp.get_data(as_text=True))
+    assert result['data']['sms_sender'] == 'elevenchars'
-            assert resp.status_code == 200
-            assert result['data']['sms_sender'] == 'elevenchars'
 
 
-def test_set_sms_sender_for_service_rejects_invalid_characters(notify_api, sample_service):
+def test_set_sms_sender_for_service_rejects_invalid_characters(client, sample_service):
-    with notify_api.test_request_context():
+    data = {
-        with notify_api.test_client() as client:
+        'sms_sender': 'invalid####',
-            auth_header = create_authorization_header()
+    }
-            resp = client.get(
-                '/service/{}'.format(sample_service.id),
-                headers=[auth_header]
-            )
-            json_resp = json.loads(resp.get_data(as_text=True))
-            assert resp.status_code == 200
-            assert json_resp['data']['name'] == sample_service.name
 
-            data = {
+    auth_header = create_authorization_header()
-                'sms_sender': 'invalid####',
-            }
 
-            auth_header = create_authorization_header()
+    resp = client.post(
+        '/service/{}'.format(sample_service.id),
+        data=json.dumps(data),
+        headers=[('Content-Type', 'application/json'), auth_header]
+    )
+    result = json.loads(resp.get_data(as_text=True))
+    assert resp.status_code == 400
+    assert result['result'] == 'error'
+    assert result['message'] == {'sms_sender': ['Only alphanumeric characters allowed']}
 
-            resp = client.post(
+
-                '/service/{}'.format(sample_service.id),
+def test_set_sms_sender_for_service_rejects_null(client, sample_service):
-                data=json.dumps(data),
+    data = {
-                headers=[('Content-Type', 'application/json'), auth_header]
+        'sms_sender': None,
-            )
+    }
-            result = json.loads(resp.get_data(as_text=True))
+
-            assert resp.status_code == 400
+    auth_header = create_authorization_header()
-            assert result['result'] == 'error'
+
-            assert result['message'] == {'sms_sender': ['Only alphanumeric characters allowed']}
+    resp = client.post(
+        '/service/{}'.format(sample_service.id),
+        data=json.dumps(data),
+        headers=[('Content-Type', 'application/json'), auth_header]
+    )
+    result = json.loads(resp.get_data(as_text=True))
+    assert resp.status_code == 400
+    assert result['result'] == 'error'
+    assert result['message'] == {'sms_sender': ['Field may not be null.']}
 
 
 @pytest.mark.parametrize('today_only,stats', [
@@ -1639,11 +1626,11 @@ def test_get_notification_billable_unit_count_missing_year(client, sample_servic
     ('?year=abcd', 400, {'message': 'Year must be a number', 'result': 'error'}),
 ])
 def test_get_service_provider_aggregate_statistics(
-    client,
+        client,
-    sample_service,
+        sample_service,
-    query_string,
+        query_string,
-    expected_status,
+        expected_status,
-    expected_json,
+        expected_json,
 ):
     response = client.get(
         '/service/{}/fragment/aggregate_statistics{}'.format(sample_service.id, query_string),
@@ -1684,11 +1671,11 @@ def test_get_template_stats_by_month_returns_correct_data(notify_db, notify_db_s
     ('?year=abcd', 400, {'message': 'Year must be a number', 'result': 'error'}),
 ])
 def test_get_template_stats_by_month_returns_error_for_incorrect_year(
-    client,
+        client,
-    sample_service,
+        sample_service,
-    query_string,
+        query_string,
-    expected_status,
+        expected_status,
-    expected_json
+        expected_json
 ):
     response = client.get(
         '/service/{}/notifications/templates/monthly{}'.format(sample_service.id, query_string),
@@ -1929,20 +1916,90 @@ def test_update_service_does_not_call_send_notification_when_restricted_not_chan
     assert not send_notification_mock.called
 
 
-def test_update_service_works_when_sms_sender_is_null(sample_service, client, mocker):
+def test_get_yearly_billing_usage_count_returns_400_if_missing_year(client, sample_service):
-    sample_service.sms_sender = None
+    response = client.get(
-    data = {'name': 'new name'}
+        '/service/{}/yearly-sms-billable-units'.format(sample_service.id),
+        headers=[create_authorization_header()]
+    )
+    assert response.status_code == 400
+    assert json.loads(response.get_data(as_text=True)) == {
+        'message': 'No valid year provided', 'result': 'error'
+    }
 
-    resp = client.post(
+
-        'service/{}'.format(sample_service.id),
+def test_get_yearly_billing_usage_count_returns_400_if_invalid_year(client, sample_service, mocker):
-        data=json.dumps(data),
+    redis_get_mock = mocker.patch('app.service.rest.redis_store.get_all_from_hash', return_value=None)
-        headers=[create_authorization_header()],
+    redis_set_mock = mocker.patch('app.service.rest.redis_store.set_hash_and_expire')
-        content_type='application/json'
+
+    response = client.get(
+        '/service/{}/yearly-sms-billable-units?year=HAHAHAHAH'.format(sample_service.id),
+        headers=[create_authorization_header()]
+    )
+    assert response.status_code == 400
+    assert json.loads(response.get_data(as_text=True)) == {
+        'message': 'No valid year provided', 'result': 'error'
+    }
+    redis_get_mock.assert_called_once_with("{}-sms_billable_units".format(str(sample_service.id)))
+    redis_set_mock.assert_not_called()
+
+
+def test_get_yearly_billing_usage_count_returns_200_if_year_provided(client, sample_service, mocker):
+    redis_get_mock = mocker.patch('app.service.rest.redis_store.get_all_from_hash', return_value=None)
+    redis_set_mock = mocker.patch('app.service.rest.redis_store.set_hash_and_expire')
+
+    start = datetime.utcnow()
+    end = datetime.utcnow() + timedelta(minutes=10)
+    mock_query = mocker.patch(
+        'app.service.rest.get_total_billable_units_for_sent_sms_notifications_in_date_range', return_value=(100, 200.0)
+    )
+    mock_year = mocker.patch('app.service.rest.get_financial_year', return_value=(start, end))
+    response = client.get(
+        '/service/{}/yearly-sms-billable-units?year=2016'.format(sample_service.id),
+        headers=[create_authorization_header()]
+    )
+    assert response.status_code == 200
+    assert json.loads(response.get_data(as_text=True)) == {
+        'billable_sms_units': 100,
+        'total_cost': 200.0
+    }
+    mock_query.assert_called_once_with(start, end, sample_service.id)
+    mock_year.assert_called_once_with(2016)
+    redis_get_mock.assert_called_once_with("{}-sms_billable_units".format(str(sample_service.id)))
+    redis_set_mock.assert_called_once_with(
+        "{}-sms_billable_units".format(str(sample_service.id)),
+        {'billable_units': 100, 'total_cost': 200.0},
+        expire_in_seconds=60
     )
 
-    assert resp.status_code == 200
+
-    # make sure it wasn't changed to not-null under the hood
+def test_get_yearly_billing_usage_count_returns_from_cache_if_present(client, sample_service, mocker):
-    assert sample_service.sms_sender is None
+    redis_get_mock = mocker.patch(
+        'app.service.rest.redis_store.get_all_from_hash',
+        return_value={b'total_cost': 100.0, b'billable_units': 50}
+    )
+    redis_set_mock = mocker.patch('app.service.rest.redis_store.set_hash_and_expire')
+    mock_query = mocker.patch(
+        'app.service.rest.get_total_billable_units_for_sent_sms_notifications_in_date_range', return_value=(50, 100.0)
+    )
+
+    start = datetime.utcnow()
+    end = datetime.utcnow() + timedelta(minutes=10)
+    mock_year = mocker.patch('app.service.rest.get_financial_year', return_value=(start, end))
+
+    response = client.get(
+        '/service/{}/yearly-sms-billable-units?year=2016'.format(sample_service.id),
+        headers=[create_authorization_header()]
+    )
+    print(response.get_data(as_text=True))
+    assert response.status_code == 200
+    assert json.loads(response.get_data(as_text=True)) == {
+        'billable_sms_units': 50,
+        'total_cost': 100.0
+    }
+    redis_get_mock.assert_called_once_with("{}-sms_billable_units".format(str(sample_service.id)))
+    mock_year.assert_not_called()
+    mock_query.assert_not_called()
+    redis_set_mock.assert_not_called()
 
 
 def test_search_for_notification_by_to_field_filters_by_status(client, notify_db, notify_db_session):

tests/app/v2/notifications/test_post_notifications.py

@@ -3,13 +3,14 @@ import uuid
 import pytest
 from freezegun import freeze_time
 
-from app.models import Notification, ScheduledNotification
+from app.models import Notification, ScheduledNotification, SCHEDULE_NOTIFICATIONS, EMAIL_TYPE, SMS_TYPE
 from flask import json, current_app
 
 from app.models import Notification
 from app.v2.errors import RateLimitError
 from tests import create_authorization_header
 from tests.app.conftest import sample_template as create_sample_template, sample_service
+from tests.app.db import create_service, create_template
 
 
 @pytest.mark.parametrize("reference", [None, "reference_from_client"])
@@ -350,26 +351,28 @@ def test_post_sms_should_persist_supplied_sms_number(client, sample_template_wit
     assert mocked.called
 
 
-@pytest.mark.skip("Once the service can be invited to schedule notifications we can add this test.")
 @pytest.mark.parametrize("notification_type, key_send_to, send_to",
                          [("sms", "phone_number", "07700 900 855"),
                           ("email", "email_address", "sample@email.com")])
 @freeze_time("2017-05-14 14:00:00")
-def test_post_notification_with_scheduled_for(client, sample_template, sample_email_template,
+def test_post_notification_with_scheduled_for(client, notify_db, notify_db_session,
                                               notification_type, key_send_to, send_to):
+    service = create_service(service_name=str(uuid.uuid4()),
+                             service_permissions=[EMAIL_TYPE, SMS_TYPE, SCHEDULE_NOTIFICATIONS])
+    template = create_template(service=service, template_type=notification_type)
     data = {
         key_send_to: send_to,
-        'template_id': str(sample_email_template.id) if notification_type == 'email' else str(sample_template.id),
+        'template_id': str(template.id) if notification_type == 'email' else str(template.id),
         'scheduled_for': '2017-05-14 14:15'
     }
-    auth_header = create_authorization_header(service_id=sample_template.service_id)
+    auth_header = create_authorization_header(service_id=service.id)
 
     response = client.post('/v2/notifications/{}'.format(notification_type),
                            data=json.dumps(data),
                            headers=[('Content-Type', 'application/json'), auth_header])
     assert response.status_code == 201
     resp_json = json.loads(response.get_data(as_text=True))
-    scheduled_notification = ScheduledNotification.query.all()
+    scheduled_notification = ScheduledNotification.query.filter_by(notification_id=resp_json["id"]).all()
     assert len(scheduled_notification) == 1
     assert resp_json["id"] == str(scheduled_notification[0].notification_id)
     assert resp_json["scheduled_for"] == '2017-05-14 14:15'
@@ -379,8 +382,8 @@ def test_post_notification_with_scheduled_for(client, sample_template, sample_em
                          [("sms", "phone_number", "07700 900 855"),
                           ("email", "email_address", "sample@email.com")])
 @freeze_time("2017-05-14 14:00:00")
-def test_post_notification_with_scheduled_for_raises_bad_request(client, sample_template, sample_email_template,
+def test_post_notification_raises_bad_request_if_service_not_invited_to_schedule(
-                                                                 notification_type, key_send_to, send_to):
+        client, sample_template, sample_email_template, notification_type, key_send_to, send_to):
     data = {
         key_send_to: send_to,
         'template_id': str(sample_email_template.id) if notification_type == 'email' else str(sample_template.id),
@@ -394,4 +397,4 @@ def test_post_notification_with_scheduled_for_raises_bad_request(client, sample_
     assert response.status_code == 400
     error_json = json.loads(response.get_data(as_text=True))
     assert error_json['errors'] == [
-        {"error": "BadRequestError", "message": 'Your service must be invited to schedule notifications via the API.'}]
+        {"error": "BadRequestError", "message": 'Cannot schedule notifications (this feature is invite-only)'}]