Merge branch 'master' into remove-initial-update-sms-sender

app/__init__.py

@@ -124,7 +124,7 @@ def register_blueprint(application):
     application.register_blueprint(sms_callback_blueprint)
 
     # inbound sms
-    receive_notifications_blueprint.before_request(restrict_ip_sms)
+    receive_notifications_blueprint.before_request(requires_no_auth)
     application.register_blueprint(receive_notifications_blueprint)
 
     notifications_blueprint.before_request(requires_auth)
@@ -174,6 +174,7 @@ def register_blueprint(application):
 
 
 def register_v2_blueprints(application):
+    from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint as get_inbound_sms
     from app.v2.notifications.post_notifications import v2_notification_blueprint as post_notifications
     from app.v2.notifications.get_notifications import v2_notification_blueprint as get_notifications
     from app.v2.template.get_template import v2_template_blueprint as get_template
@@ -196,6 +197,9 @@ def register_v2_blueprints(application):
     post_template.before_request(requires_auth)
     application.register_blueprint(post_template)
 
+    get_inbound_sms.before_request(requires_auth)
+    application.register_blueprint(get_inbound_sms)
+
 
 def init_app(app):
     @app.before_request

app/billing/billing_schemas.py

@@ -8,7 +8,6 @@ create_or_update_free_sms_fragment_limit_schema = {
     "title": "Create",
     "properties": {
         "free_sms_fragment_limit": {"type": "integer", "minimum": 1},
-        "financial_year_start": {"type": "integer", "minimum": 2016}
     },
     "required": ["free_sms_fragment_limit"]
 }

app/celery/scheduled_tasks.py

@@ -11,6 +11,10 @@ from notifications_utils.s3 import s3upload
 
 from app.aws import s3
 from app import notify_celery
+from app.dao.services_dao import (
+    dao_fetch_monthly_historical_stats_by_template
+)
+from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template
 from app.performance_platform import total_sent_notifications, processing_time
 from app import performance_platform_client
 from app.dao.date_util import get_month_start_and_end_date_in_utc
@@ -402,3 +406,18 @@ def check_job_status():
             queue=QueueNames.JOBS
         )
         raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids))
+
+
+@notify_celery.task(name='daily-stats-template_usage_by_month')
+@statsd(namespace="tasks")
+def daily_stats_template_usage_by_month():
+    results = dao_fetch_monthly_historical_stats_by_template()
+
+    for result in results:
+        if result.template_id:
+            insert_or_update_stats_for_template(
+                result.template_id,
+                result.month,
+                result.year,
+                result.count
+            )

app/config.py

@@ -241,6 +241,11 @@ class Config(object):
             'task': 'check-job-status',
             'schedule': crontab(),
             'options': {'queue': QueueNames.PERIODIC}
+        },
+        'daily-stats-template_usage_by_month': {
+            'task': 'daily-stats-template_usage_by_month',
+            'schedule': crontab(hour=0, minute=50),
+            'options': {'queue': QueueNames.PERIODIC}
         }
     }
     CELERY_QUEUES = []

app/dao/inbound_sms_dao.py

@@ -2,7 +2,8 @@ from datetime import (
     timedelta,
     datetime
 )
-
+from flask import current_app
+from sqlalchemy import desc
 
 from app import db
 from app.dao.dao_utils import transactional
@@ -31,6 +32,28 @@ def dao_get_inbound_sms_for_service(service_id, limit=None, user_number=None):
     return q.all()
 
 
+def dao_get_paginated_inbound_sms_for_service(
+    service_id,
+    older_than=None,
+    page_size=None
+):
+    if page_size is None:
+        page_size = current_app.config['PAGE_SIZE']
+
+    filters = [InboundSms.service_id == service_id]
+
+    if older_than:
+        older_than_created_at = db.session.query(
+            InboundSms.created_at).filter(InboundSms.id == older_than).as_scalar()
+        filters.append(InboundSms.created_at < older_than_created_at)
+
+    query = InboundSms.query.filter(*filters)
+
+    return query.order_by(desc(InboundSms.created_at)).paginate(
+        per_page=page_size
+    ).items
+
+
 def dao_count_inbound_sms_for_service(service_id):
     return InboundSms.query.filter(
         InboundSms.service_id == service_id

app/dao/monthly_billing_dao.py

@@ -26,6 +26,7 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
     ).distinct().all()
 
 
+@statsd(namespace="dao")
 def create_or_update_monthly_billing(service_id, billing_month):
     start_date, end_date = get_month_start_and_end_date_in_utc(billing_month)
     _update_monthly_billing(service_id, start_date, end_date, SMS_TYPE)
@@ -47,6 +48,7 @@ def _monthly_billing_data_to_json(billing_data):
     return results
 
 
+@statsd(namespace="dao")
 @transactional
 def _update_monthly_billing(service_id, start_date, end_date, notification_type):
     billing_data = get_billing_data_for_month(

app/dao/notification_usage_dao.py

@@ -105,6 +105,7 @@ def is_between(date, start_date, end_date):
     return start_date <= date <= end_date
 
 
+@statsd(namespace="dao")
 def billing_data_per_month_query(rate, service_id, start_date, end_date, notification_type):
     month = get_london_month_from_utc_column(NotificationHistory.created_at)
     if notification_type == SMS_TYPE:

app/dao/services_dao.py

@@ -1,7 +1,7 @@
 import uuid
-from datetime import date, datetime, timedelta
+from datetime import date, datetime, timedelta, time
 
-from sqlalchemy import asc, func
+from sqlalchemy import asc, func, extract
 from sqlalchemy.orm import joinedload
 from flask import current_app
 
@@ -519,3 +519,25 @@ def dao_fetch_active_users_for_service(service_id):
     )
 
     return query.all()
+
+
+@statsd(namespace="dao")
+def dao_fetch_monthly_historical_stats_by_template():
+    month = get_london_month_from_utc_column(NotificationHistory.created_at)
+    year = func.date_trunc("year", NotificationHistory.created_at)
+    end_date = datetime.combine(date.today(), time.min)
+
+    return db.session.query(
+        NotificationHistory.template_id,
+        extract('month', month).label('month'),
+        extract('year', year).label('year'),
+        func.count().label('count')
+    ).filter(
+        NotificationHistory.created_at < end_date
+    ).group_by(
+        NotificationHistory.template_id,
+        month,
+        year
+    ).order_by(
+        NotificationHistory.template_id
+    ).all()

app/dao/stats_template_usage_by_month_dao.py

@@ -0,0 +1,24 @@
+from app import db
+from app.models import StatsTemplateUsageByMonth
+
+
+def insert_or_update_stats_for_template(template_id, month, year, count):
+    result = db.session.query(
+        StatsTemplateUsageByMonth
+    ).filter(
+        StatsTemplateUsageByMonth.template_id == template_id,
+        StatsTemplateUsageByMonth.month == month,
+        StatsTemplateUsageByMonth.year == year
+    ).update(
+        {
+            'count': count
+        }
+    )
+    if result == 0:
+        monthly_stats = StatsTemplateUsageByMonth(
+            template_id=template_id,
+            month=month,
+            year=year,
+            count=count
+        )
+        db.session.add(monthly_stats)

app/errors.py

@@ -94,7 +94,7 @@ def register_errors(blueprint):
         current_app.logger.exception(e)
         if hasattr(e, 'orig') and hasattr(e.orig, 'pgerror') and e.orig.pgerror and \
             ('duplicate key value violates unique constraint "services_name_key"' in e.orig.pgerror or
-                'duplicate key value violates unique constraint "services_email_from_key"' in e.orig.pgerror):
+             'duplicate key value violates unique constraint "services_email_from_key"' in e.orig.pgerror):
             return jsonify(
                 result='error',
                 message={'name': ["Duplicate service name '{}'".format(

app/models.py

@@ -9,7 +9,7 @@ from sqlalchemy.dialects.postgresql import (
     UUID,
     JSON
 )
-from sqlalchemy import UniqueConstraint, and_
+from sqlalchemy import UniqueConstraint, CheckConstraint, and_
 from sqlalchemy.orm import foreign, remote
 from notifications_utils.recipients import (
     validate_email_address,
@@ -97,7 +97,7 @@ class User(db.Model):
         nullable=True,
         onupdate=datetime.datetime.utcnow)
     _password = db.Column(db.String, index=False, unique=False, nullable=False)
-    mobile_number = db.Column(db.String, index=False, unique=False, nullable=False)
+    mobile_number = db.Column(db.String, index=False, unique=False, nullable=True)
     password_changed_at = db.Column(db.DateTime, index=False, unique=False, nullable=False,
                                     default=datetime.datetime.utcnow)
     logged_in_at = db.Column(db.DateTime, nullable=True)
@@ -107,6 +107,9 @@ class User(db.Model):
     current_session_id = db.Column(UUID(as_uuid=True), nullable=True)
     auth_type = db.Column(db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=SMS_AUTH_TYPE)
 
+    # either email auth or a mobile number must be provided
+    CheckConstraint("auth_type = 'email_auth' or mobile_number is not null")
+
     services = db.relationship(
         'Service',
         secondary='user_to_service',
@@ -1404,13 +1407,11 @@ class InboundSms(db.Model):
     def serialize(self):
         return {
             'id': str(self.id),
-            'created_at': self.created_at.isoformat(),
+            'created_at': self.created_at.strftime(DATETIME_FORMAT),
             'service_id': str(self.service_id),
             'notify_number': self.notify_number,
             'user_number': self.user_number,
             'content': self.content,
-            'provider_date': self.provider_date and self.provider_date.isoformat(),
-            'provider_reference': self.provider_reference
         }
 
 
@@ -1554,3 +1555,37 @@ class AuthType(db.Model):
     __tablename__ = 'auth_type'
 
     name = db.Column(db.String, primary_key=True)
+
+
+class StatsTemplateUsageByMonth(db.Model):
+    __tablename__ = "stats_template_usage_by_month"
+
+    template_id = db.Column(
+        UUID(as_uuid=True),
+        db.ForeignKey('templates.id'),
+        unique=False,
+        index=True,
+        nullable=False,
+        primary_key=True
+    )
+    month = db.Column(
+        db.Integer,
+        nullable=False,
+        index=True,
+        unique=False,
+        primary_key=True,
+        default=datetime.datetime.month
+    )
+    year = db.Column(
+        db.Integer,
+        nullable=False,
+        index=True,
+        unique=False,
+        primary_key=True,
+        default=datetime.datetime.year
+    )
+    count = db.Column(
+        db.Integer,
+        nullable=False,
+        default=0
+    )

app/notifications/receive_notifications.py

@@ -82,7 +82,11 @@ def receive_firetext_sms():
 
 
 def format_mmg_message(message):
-    return unquote(message.replace('+', ' '))
+    return unescape_string(unquote(message.replace('+', ' ')))
+
+
+def unescape_string(string):
+    return string.encode('raw_unicode_escape').decode('unicode_escape')
 
 
 def format_mmg_datetime(date):

app/notifications/validators.py

@@ -155,7 +155,7 @@ def check_service_sms_sender_id(service_id, sms_sender_id, notification_type):
             message = 'sms_sender_id is not a valid option for {} notification'.format(notification_type)
             raise BadRequestError(message=message)
         try:
-            dao_get_service_sms_senders_by_id(service_id, sms_sender_id)
+            return dao_get_service_sms_senders_by_id(service_id, sms_sender_id).sms_sender
         except NoResultFound:
             message = 'sms_sender_id {} does not exist in database for service id {}'\
                 .format(sms_sender_id, service_id)

app/schemas.py

@@ -105,6 +105,26 @@ class UserSchema(BaseSchema):
             "_password", "verify_codes")
         strict = True
 
+    @validates('name')
+    def validate_name(self, value):
+        if not value:
+            raise ValidationError('Invalid name')
+
+    @validates('email_address')
+    def validate_email_address(self, value):
+        try:
+            validate_email_address(value)
+        except InvalidEmailError as e:
+            raise ValidationError(str(e))
+
+    @validates('mobile_number')
+    def validate_mobile_number(self, value):
+        try:
+            if value is not None:
+                validate_phone_number(value, international=True)
+        except InvalidPhoneError as error:
+            raise ValidationError('Invalid phone number: {}'.format(error))
+
 
 class UserUpdateAttributeSchema(BaseSchema):
     auth_type = field_for(models.User, 'auth_type')
@@ -132,7 +152,8 @@ class UserUpdateAttributeSchema(BaseSchema):
     @validates('mobile_number')
     def validate_mobile_number(self, value):
         try:
-            validate_phone_number(value, international=True)
+            if value is not None:
+                validate_phone_number(value, international=True)
         except InvalidPhoneError as error:
             raise ValidationError('Invalid phone number: {}'.format(error))
 

app/user/rest.py

@@ -4,6 +4,7 @@ from datetime import datetime
 from urllib.parse import urlencode
 
 from flask import (jsonify, request, Blueprint, current_app, abort)
+from sqlalchemy.exc import IntegrityError
 
 from app.config import QueueNames
 from app.dao.users_dao import (
@@ -52,6 +53,19 @@ user_blueprint = Blueprint('user', __name__)
 register_errors(user_blueprint)
 
 
+@user_blueprint.errorhandler(IntegrityError)
+def handle_integrity_error(exc):
+    """
+    Handle integrity errors caused by the auth type/mobile number check constraint
+    """
+    if 'ck_users_mobile_or_email_auth' in str(exc):
+        # we don't expect this to trip, so still log error
+        current_app.logger.exception('Check constraint ck_users_mobile_or_email_auth triggered')
+        return jsonify(result='error', message='Mobile number must be set if auth_type is set to sms_auth'), 400
+
+    raise
+
+
 @user_blueprint.route('', methods=['POST'])
 def create_user():
     user_to_create, errors = user_schema.load(request.get_json())
@@ -63,23 +77,6 @@ def create_user():
     return jsonify(data=user_schema.dump(user_to_create).data), 201
 
 
-@user_blueprint.route('/<uuid:user_id>', methods=['PUT'])
-def update_user(user_id):
-    user_to_update = get_user_by_id(user_id=user_id)
-    req_json = request.get_json()
-    update_dct, errors = user_schema_load_json.load(req_json)
-    # TODO don't let password be updated in this PUT method (currently used by the forgot password flow)
-    pwd = req_json.get('password', None)
-    if pwd is not None:
-        if not pwd:
-            errors.update({'password': ['Invalid data for field']})
-            raise InvalidRequest(errors, status_code=400)
-        else:
-            reset_failed_login_count(user_to_update)
-    save_model_user(user_to_update, update_dict=update_dct, pwd=pwd)
-    return jsonify(data=user_schema.dump(user_to_update).data), 200
-
-
 @user_blueprint.route('/<uuid:user_id>', methods=['POST'])
 def update_user_attribute(user_id):
     user_to_update = get_user_by_id(user_id=user_id)
@@ -91,6 +88,17 @@ def update_user_attribute(user_id):
     return jsonify(data=user_schema.dump(user_to_update).data), 200
 
 
+@user_blueprint.route('/<uuid:user_id>/activate', methods=['POST'])
+def activate_user(user_id):
+    user = get_user_by_id(user_id=user_id)
+    if user.state == 'active':
+        raise InvalidRequest('User already active', status_code=400)
+
+    user.state = 'active'
+    save_model_user(user)
+    return jsonify(data=user_schema.dump(user).data), 200
+
+
 @user_blueprint.route('/<uuid:user_id>/reset-failed-login-count', methods=['POST'])
 def user_reset_failed_login_count(user_id):
     user_to_update = get_user_by_id(user_id=user_id)

app/v2/inbound_sms/__init__.py

@@ -0,0 +1,6 @@
+from flask import Blueprint
+from app.v2.errors import register_errors
+
+v2_inbound_sms_blueprint = Blueprint("v2_inbound_sms", __name__, url_prefix='/v2/received-text-messages')
+
+register_errors(v2_inbound_sms_blueprint)

app/v2/inbound_sms/get_inbound_sms.py

@@ -0,0 +1,44 @@
+from flask import jsonify, request, url_for, current_app
+
+from notifications_utils.recipients import validate_and_format_phone_number
+from notifications_utils.recipients import InvalidPhoneError
+
+from app import authenticated_service
+from app.dao import inbound_sms_dao
+from app.schema_validation import validate
+from app.v2.inbound_sms import v2_inbound_sms_blueprint
+from app.v2.inbound_sms.inbound_sms_schemas import get_inbound_sms_request
+
+
+@v2_inbound_sms_blueprint.route("", methods=['GET'])
+def get_inbound_sms():
+    data = validate(request.args.to_dict(), get_inbound_sms_request)
+
+    paginated_inbound_sms = inbound_sms_dao.dao_get_paginated_inbound_sms_for_service(
+        authenticated_service.id,
+        older_than=data.get('older_than', None),
+        page_size=current_app.config.get('API_PAGE_SIZE')
+    )
+
+    return jsonify(
+        received_text_messages=[i.serialize() for i in paginated_inbound_sms],
+        links=_build_links(paginated_inbound_sms)
+    ), 200
+
+
+def _build_links(inbound_sms_list):
+    _links = {
+        'current': url_for(
+            "v2_inbound_sms.get_inbound_sms",
+            _external=True,
+        ),
+    }
+
+    if inbound_sms_list:
+        _links['next'] = url_for(
+            "v2_inbound_sms.get_inbound_sms",
+            older_than=inbound_sms_list[-1].id,
+            _external=True,
+        )
+
+    return _links

app/v2/inbound_sms/inbound_sms_schemas.py

@@ -0,0 +1,70 @@
+from app.schema_validation.definitions import uuid
+
+
+get_inbound_sms_request = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "schema for query parameters allowed when getting list of received text messages",
+    "type": "object",
+    "properties": {
+        "older_than": uuid,
+    },
+    "additionalProperties": False,
+}
+
+
+get_inbound_sms_single_response = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "GET inbound sms schema response",
+    "type": "object",
+    "title": "GET response v2/inbound_sms",
+    "properties": {
+        "user_number": {"type": "string"},
+        "created_at": {
+            "format": "date-time",
+            "type": "string",
+            "description": "Date+time created at"
+        },
+        "service_id": uuid,
+        "id": uuid,
+        "notify_number": {"type": "string"},
+        "content": {"type": "string"},
+    },
+    "required": [
+        "id", "user_number", "created_at", "service_id",
+        "notify_number", "content"
+    ],
+    "additionalProperties": False,
+}
+
+get_inbound_sms_response = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "GET list of inbound sms response schema",
+    "type": "object",
+    "properties": {
+        "received_text_messages": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "$ref": "#/definitions/inbound_sms"
+            }
+        },
+        "links": {
+            "type": "object",
+            "properties": {
+                "current": {
+                    "type": "string"
+                },
+                "next": {
+                    "type": "string"
+                }
+            },
+            "additionalProperties": False,
+            "required": ["current"]
+        }
+    },
+    "required": ["received_text_messages", "links"],
+    "definitions": {
+        "inbound_sms": get_inbound_sms_single_response
+    },
+    "additionalProperties": False,
+}

app/v2/notifications/post_notifications.py

@@ -71,7 +71,7 @@ def post_notification(notification_type):
     check_rate_limiting(authenticated_service, api_user)
 
     check_service_email_reply_to_id(str(authenticated_service.id), service_email_reply_to_id, notification_type)
-    check_service_sms_sender_id(str(authenticated_service.id), service_sms_sender_id, notification_type)
+    sms_sender = check_service_sms_sender_id(str(authenticated_service.id), service_sms_sender_id, notification_type)
 
     template, template_with_content = validate_template(
         form['template_id'],
@@ -98,7 +98,7 @@ def post_notification(notification_type):
     if notification_type == SMS_TYPE:
         create_resp_partial = functools.partial(
             create_post_sms_response_from_notification,
-            from_number=authenticated_service.get_default_sms_sender()
+            from_number=sms_sender or authenticated_service.get_default_sms_sender()
         )
     elif notification_type == EMAIL_TYPE:
         create_resp_partial = functools.partial(