Merge branch 'master' into letters-billing-table

Conflicts:
	app/models.py

app/config.py

@@ -24,23 +24,6 @@ class QueueNames(object):
     NOTIFY = 'notify-internal-tasks'
     PROCESS_FTP = 'process-ftp-tasks'
 
-    @staticmethod
-    def old_queues():
-        return [
-            'db-sms',
-            'db-email',
-            'db-letter',
-            'priority',
-            'periodic',
-            'send-sms',
-            'send-email',
-            'research-mode',
-            'statistics',
-            'notify',
-            'retry',
-            'process-job'
-        ]
-
     @staticmethod
     def all_queues():
         return [
@@ -124,6 +107,7 @@ class Config(object):
     SMS_CHAR_COUNT_LIMIT = 495
     BRANDING_PATH = '/images/email-template/crests/'
     TEST_MESSAGE_FILENAME = 'Test message'
+    ONE_OFF_MESSAGE_FILENAME = 'Report'
     MAX_VERIFY_CODE_COUNT = 10
 
     NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
@@ -262,8 +246,6 @@ class Development(Config):
     NOTIFICATION_QUEUE_PREFIX = 'development'
     DEBUG = True
 
-    queues = QueueNames.all_queues() + QueueNames.old_queues()
-
     for queue in QueueNames.all_queues():
         Config.CELERY_QUEUES.append(
             Queue(queue, Exchange('default'), routing_key=queue)
@@ -283,9 +265,7 @@ class Test(Config):
     STATSD_HOST = "localhost"
     STATSD_PORT = 1000
 
-    queues = QueueNames.all_queues() + QueueNames.old_queues()
-
-    for queue in queues:
+    for queue in QueueNames.all_queues():
         Config.CELERY_QUEUES.append(
             Queue(queue, Exchange('default'), routing_key=queue)
         )

app/dao/inbound_sms_dao.py

@@ -0,0 +1,7 @@
+from app import db
+from app.dao.dao_utils import transactional
+
+
+@transactional
+def dao_create_inbound_sms(inbound_sms):
+    db.session.add(inbound_sms)

app/dao/jobs_dao.py

@@ -53,7 +53,8 @@ def dao_get_job_by_service_id_and_job_id(service_id, job_id):
 def dao_get_jobs_by_service_id(service_id, limit_days=None, page=1, page_size=50, statuses=None):
     query_filter = [
         Job.service_id == service_id,
-        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME']
+        Job.original_file_name != current_app.config['TEST_MESSAGE_FILENAME'],
+        Job.original_file_name != current_app.config['ONE_OFF_MESSAGE_FILENAME'],
     ]
     if limit_days is not None:
         query_filter.append(cast(Job.created_at, sql_date) >= days_ago(limit_days))

app/dao/notification_usage_dao.py

@@ -153,3 +153,70 @@ def rate_multiplier():
         (NotificationHistory.rate_multiplier == None, literal_column("'1'")),  # noqa
         (NotificationHistory.rate_multiplier != None, NotificationHistory.rate_multiplier),  # noqa
     ]), Integer())
+
+
+@statsd(namespace="dao")
+def get_total_billable_units_for_sent_sms_notifications_in_date_range(start_date, end_date, service_id):
+
+    billable_units = 0
+    total_cost = 0.0
+
+    rate_boundaries = discover_rate_bounds_for_billing_query(start_date, end_date)
+    for rate_boundary in rate_boundaries:
+        result = db.session.query(
+            func.sum(
+                NotificationHistory.billable_units * func.coalesce(NotificationHistory.rate_multiplier, 1)
+            ).label('billable_units')
+        ).filter(
+            NotificationHistory.service_id == service_id,
+            NotificationHistory.notification_type == 'sms',
+            NotificationHistory.created_at >= rate_boundary['start_date'],
+            NotificationHistory.created_at < rate_boundary['end_date'],
+            NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE)
+        )
+        billable_units_by_rate_boundry = result.scalar()
+        if billable_units_by_rate_boundry:
+            billable_units += int(billable_units_by_rate_boundry)
+            total_cost += int(billable_units_by_rate_boundry) * rate_boundary['rate']
+
+    return billable_units, total_cost
+
+
+def discover_rate_bounds_for_billing_query(start_date, end_date):
+    bounds = []
+    rates = get_rates_for_year(start_date, end_date, SMS_TYPE)
+
+    def current_valid_from(index):
+        return rates[index].valid_from
+
+    def next_valid_from(index):
+        return rates[index + 1].valid_from
+
+    def current_rate(index):
+        return rates[index].rate
+
+    def append_rate(rate_start_date, rate_end_date, rate):
+        bounds.append({
+            'start_date': rate_start_date,
+            'end_date': rate_end_date,
+            'rate': rate
+        })
+
+    if len(rates) == 1:
+        append_rate(start_date, end_date, current_rate(0))
+        return bounds
+
+    for i in range(len(rates)):
+        # first boundary
+        if i == 0:
+            append_rate(start_date, next_valid_from(i), current_rate(i))
+
+        # last boundary
+        elif i == (len(rates) - 1):
+            append_rate(current_valid_from(i), end_date, current_rate(i))
+
+        # other boundaries
+        else:
+            append_rate(current_valid_from(i), next_valid_from(i), current_rate(i))
+
+    return bounds

app/dao/services_dao.py

@@ -3,6 +3,7 @@ from datetime import date, datetime, timedelta
 
 from sqlalchemy import asc, func
 from sqlalchemy.orm import joinedload
+from flask import current_app
 
 from app import db
 from app.dao.dao_utils import (
@@ -66,6 +67,12 @@ def dao_fetch_service_by_id(service_id, only_active=False):
     return query.one()
 
 
+def dao_fetch_services_by_sms_sender(sms_sender):
+    return Service.query.filter(
+        Service.sms_sender == sms_sender
+    ).all()
+
+
 def dao_fetch_service_by_id_with_api_keys(service_id, only_active=False):
     query = Service.query.filter_by(
         id=service_id
@@ -131,6 +138,12 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
 @transactional
 @version_class(Service)
 def dao_create_service(service, user, service_id=None, service_permissions=[SMS_TYPE, EMAIL_TYPE]):
+    # the default property does not appear to work when there is a difference between the sqlalchemy schema and the
+    # db schema (ie: during a migration), so we have to set sms_sender manually here. After the GOVUK sms_sender
+    # migration is completed, this code should be able to be removed.
+    if not service.sms_sender:
+        service.sms_sender = current_app.config['FROM_NUMBER']
+
     from app.dao.permissions_dao import permission_dao
     service.users.append(user)
     permission_dao.add_default_service_permissions_for_user(user, service)

app/models.py

@@ -146,8 +146,10 @@ class DVLAOrganisation(db.Model):
 
 INTERNATIONAL_SMS_TYPE = 'international_sms'
 INBOUND_SMS_TYPE = 'inbound_sms'
+SCHEDULE_NOTIFICATIONS = 'schedule_notifications'
 
-SERVICE_PERMISSION_TYPES = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE]
+SERVICE_PERMISSION_TYPES = [EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE,
+                            SCHEDULE_NOTIFICATIONS]
 
 
 class ServicePermissionTypes(db.Model):
@@ -188,7 +190,7 @@ class Service(db.Model, Versioned):
     created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
     reply_to_email_address = db.Column(db.Text, index=False, unique=False, nullable=True)
     letter_contact_block = db.Column(db.Text, index=False, unique=False, nullable=True)
-    sms_sender = db.Column(db.String(11), nullable=True, default=lambda: current_app.config['FROM_NUMBER'])
+    sms_sender = db.Column(db.String(11), nullable=False, default=lambda: current_app.config['FROM_NUMBER'])
     organisation_id = db.Column(UUID(as_uuid=True), db.ForeignKey('organisation.id'), index=True, nullable=True)
     organisation = db.relationship('Organisation')
     dvla_organisation_id = db.Column(
@@ -681,6 +683,8 @@ NOTIFICATION_STATUS_TYPES = [
     NOTIFICATION_PERMANENT_FAILURE,
 ]
 
+NOTIFICATION_STATUS_TYPES_NON_BILLABLE = list(set(NOTIFICATION_STATUS_TYPES) - set(NOTIFICATION_STATUS_TYPES_BILLABLE))
+
 NOTIFICATION_STATUS_TYPES_ENUM = db.Enum(*NOTIFICATION_STATUS_TYPES, name='notify_status_type')
 
 
@@ -977,7 +981,7 @@ INVITED_USER_STATUS_TYPES = ['pending', 'accepted', 'cancelled']
 class ScheduledNotification(db.Model):
     __tablename__ = 'scheduled_notifications'
 
-    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4())
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     notification_id = db.Column(UUID(as_uuid=True), db.ForeignKey('notifications.id'), index=True, nullable=False)
     notification = db.relationship('Notification', uselist=False)
     scheduled_for = db.Column(db.DateTime, index=False, nullable=False)
@@ -1097,6 +1101,12 @@ class Rate(db.Model):
     rate = db.Column(db.Float(asdecimal=False), nullable=False)
     notification_type = db.Column(notification_types, index=True, nullable=False)
 
+    def __str__(self):
+        the_string = "{}".format(self.rate)
+        the_string += " {}".format(self.notification_type)
+        the_string += " {}".format(self.valid_from)
+        return the_string
+
 
 class JobStatistics(db.Model):
     __tablename__ = 'job_statistics'
@@ -1143,6 +1153,29 @@ class JobStatistics(db.Model):
         return the_string
 
 
+class InboundSms(db.Model):
+    __tablename__ = 'inbound_sms'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
+    service = db.relationship('Service', backref='inbound_sms')
+
+    notify_number = db.Column(db.String, nullable=False)  # the service's number, that the msg was sent to
+    user_number = db.Column(db.String, nullable=False)  # the end user's number, that the msg was sent from
+    provider_date = db.Column(db.DateTime)
+    provider_reference = db.Column(db.String)
+    _content = db.Column('content', db.String, nullable=False)
+
+    @property
+    def content(self):
+        return encryption.decrypt(self._content)
+
+    @content.setter
+    def content(self, content):
+        self._content = encryption.encrypt(content)
+
+
 class LetterRate(db.Model):
     __tablename__ = 'letter_rates'
 

app/notifications/receive_notifications.py

@@ -1,8 +1,15 @@
-from flask import Blueprint
-from flask import current_app
-from flask import request
+from urllib.parse import unquote
 
+import iso8601
+from flask import jsonify, Blueprint, current_app, request
+from notifications_utils.recipients import normalise_phone_number
+
+from app import statsd_client
+from app.dao.services_dao import dao_fetch_services_by_sms_sender
+from app.dao.inbound_sms_dao import dao_create_inbound_sms
+from app.models import InboundSms
 from app.errors import register_errors
+from app.utils import convert_bst_to_utc
 
 receive_notifications_blueprint = Blueprint('receive_notifications', __name__)
 register_errors(receive_notifications_blueprint)
@@ -10,8 +17,77 @@ register_errors(receive_notifications_blueprint)
 
 @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST'])
 def receive_mmg_sms():
+    """
+    {
+        'MSISDN': '447123456789'
+        'Number': '40604',
+        'Message': 'some+uri+encoded+message%3A',
+        'ID': 'SOME-MMG-SPECIFIC-ID',
+        'DateRecieved': '2017-05-21+11%3A56%3A11'
+    }
+    """
     post_data = request.get_json()
-    post_data.pop('MSISDN', None)
-    current_app.logger.info("Recieve notification form data: {}".format(post_data))
+    potential_services = dao_fetch_services_by_sms_sender(post_data['Number'])
 
-    return "RECEIVED"
+    if len(potential_services) != 1:
+        current_app.logger.error('Inbound number "{}" not associated with exactly one service'.format(
+            post_data['Number']
+        ))
+        statsd_client.incr('inbound.mmg.failed')
+        # since this is an issue with our service <-> number mapping, we should still tell MMG that we received
+        # succesfully
+        return 'RECEIVED', 200
+
+    statsd_client.incr('inbound.mmg.succesful')
+
+    service = potential_services[0]
+
+    inbound = create_inbound_mmg_sms_object(service, post_data)
+
+    current_app.logger.info('{} received inbound SMS with reference {}'.format(service.id, inbound.provider_reference))
+
+    return 'RECEIVED', 200
+
+
+def format_mmg_message(message):
+    return unquote(message.replace('+', ' '))
+
+
+def format_mmg_datetime(date):
+    """
+    We expect datetimes in format 2017-05-21+11%3A56%3A11 - ie, spaces replaced with pluses, and URI encoded
+    (the same as UTC)
+    """
+    orig_date = format_mmg_message(date)
+    parsed_datetime = iso8601.parse_date(orig_date).replace(tzinfo=None)
+    return convert_bst_to_utc(parsed_datetime)
+
+
+def create_inbound_mmg_sms_object(service, json):
+    message = format_mmg_message(json['Message'])
+    user_number = normalise_phone_number(json['MSISDN'])
+
+    provider_date = json.get('DateRecieved')
+    if provider_date:
+        provider_date = format_mmg_datetime(provider_date)
+
+    inbound = InboundSms(
+        service=service,
+        notify_number=service.sms_sender,
+        user_number=user_number,
+        provider_date=provider_date,
+        provider_reference=json.get('ID'),
+        content=message,
+    )
+    dao_create_inbound_sms(inbound)
+    return inbound
+
+
+@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST'])
+def receive_firetext_sms():
+    post_data = request.form
+    current_app.logger.info("Received Firetext notification form data: {}".format(post_data))
+
+    return jsonify({
+        "status": "ok"
+    }), 200

app/notifications/validators.py

@@ -6,7 +6,7 @@ from notifications_utils.recipients import (
 )
 
 from app.dao import services_dao
-from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE
+from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE, SCHEDULE_NOTIFICATIONS
 from app.service.utils import service_allowed_to_send_to
 from app.v2.errors import TooManyRequestsError, BadRequestError, RateLimitError
 from app import redis_store
@@ -92,6 +92,7 @@ def check_sms_content_char_count(content_count):
         raise BadRequestError(message=message)
 
 
-def service_can_schedule_notification(service):
-    # TODO: implement once the service permission works.
-    raise BadRequestError(message="Your service must be invited to schedule notifications via the API.")
+def service_can_schedule_notification(service, scheduled_for):
+    if scheduled_for:
+        if SCHEDULE_NOTIFICATIONS not in [p.permission for p in service.permissions]:
+            raise BadRequestError(message="Cannot schedule notifications (this feature is invite-only)")

app/service/rest.py

@@ -10,6 +10,7 @@ from flask import (
 )
 from sqlalchemy.orm.exc import NoResultFound
 
+from app import redis_store
 from app.dao import notification_usage_dao, notifications_dao
 from app.dao.dao_utils import dao_rollback
 from app.dao.api_key_dao import (
@@ -17,6 +18,8 @@ from app.dao.api_key_dao import (
     get_model_api_keys,
     get_unsigned_secret,
     expire_api_key)
+from app.dao.date_util import get_financial_year
+from app.dao.notification_usage_dao import get_total_billable_units_for_sent_sms_notifications_in_date_range
 from app.dao.services_dao import (
     dao_fetch_service_by_id,
     dao_fetch_all_services,
@@ -60,6 +63,7 @@ from app.schemas import (
     detailed_service_schema
 )
 from app.utils import pagination_links
+from notifications_utils.clients.redis import sms_billable_units_cache_key
 
 service_blueprint = Blueprint('service', __name__)
 
@@ -447,6 +451,39 @@ def get_monthly_template_stats(service_id):
         raise InvalidRequest('Year must be a number', status_code=400)
 
 
+@service_blueprint.route('/<uuid:service_id>/yearly-sms-billable-units')
+def get_yearly_sms_billable_units(service_id):
+    cache_key = sms_billable_units_cache_key(service_id)
+    cached_billable_sms_units = redis_store.get_all_from_hash(cache_key)
+    if cached_billable_sms_units:
+        return jsonify({
+            'billable_sms_units': int(cached_billable_sms_units[b'billable_units']),
+            'total_cost': float(cached_billable_sms_units[b'total_cost'])
+        })
+    else:
+        try:
+            start_date, end_date = get_financial_year(int(request.args.get('year')))
+        except (ValueError, TypeError) as e:
+            current_app.logger.exception(e)
+            return jsonify(result='error', message='No valid year provided'), 400
+
+        billable_units, total_cost = get_total_billable_units_for_sent_sms_notifications_in_date_range(
+            start_date,
+            end_date,
+            service_id)
+
+        cached_values = {
+            'billable_units': billable_units,
+            'total_cost': total_cost
+        }
+
+        redis_store.set_hash_and_expire(cache_key, cached_values, expire_in_seconds=60)
+        return jsonify({
+            'billable_sms_units': billable_units,
+            'total_cost': total_cost
+        })
+
+
 @service_blueprint.route('/<uuid:service_id>/yearly-usage')
 def get_yearly_billing_usage(service_id):
     try:

app/v2/notifications/notification_schemas.py

@@ -113,7 +113,7 @@ post_sms_request = {
         "phone_number": {"type": "string", "format": "phone_number"},
         "template_id": uuid,
         "personalisation": personalisation,
-        "scheduled_for": {"type": "string", "format": "datetime"}
+        "scheduled_for": {"type": ["string", "null"], "format": "datetime"}
     },
     "required": ["phone_number", "template_id"]
 }
@@ -141,7 +141,7 @@ post_sms_response = {
         "content": sms_content,
         "uri": {"type": "string", "format": "uri"},
         "template": template,
-        "scheduled_for": {"type": "string"}
+        "scheduled_for": {"type": ["string", "null"]}
     },
     "required": ["id", "content", "uri", "template"]
 }
@@ -157,7 +157,7 @@ post_email_request = {
         "email_address": {"type": "string", "format": "email_address"},
         "template_id": uuid,
         "personalisation": personalisation,
-        "scheduled_for": {"type": "string", "format": "datetime"}
+        "scheduled_for": {"type": ["string", "null"], "format": "datetime"}
     },
     "required": ["email_address", "template_id"]
 }
@@ -186,7 +186,7 @@ post_email_response = {
         "content": email_content,
         "uri": {"type": "string", "format": "uri"},
         "template": template,
-        "scheduled_for": {"type": "string"}
+        "scheduled_for": {"type": ["string", "null"]}
     },
     "required": ["id", "content", "uri", "template"]
 }

app/v2/notifications/post_notifications.py

@@ -35,9 +35,8 @@ def post_notification(notification_type):
         form = validate(request.get_json(), post_sms_request)
 
     scheduled_for = form.get("scheduled_for", None)
-    if scheduled_for:
-        if not service_can_schedule_notification(authenticated_service):
-            return
+    service_can_schedule_notification(authenticated_service, scheduled_for)
+
     check_rate_limiting(authenticated_service, api_user)
 
     form_send_to = form['phone_number'] if notification_type == SMS_TYPE else form['email_address']