Merge branch 'master' into refactor-for-code-reuse

app/authentication/auth.py

@@ -16,10 +16,15 @@ class AuthError(Exception):
         self.code = code
 
     def to_dict_v2(self):
-        return {'code': self.code,
-                'message': self.short_message,
-                'fields': self.message,
-                'link': 'link to docs'}
+        return {
+            'status_code': self.code,
+            "errors": [
+                {
+                    "error": "AuthError",
+                    "message": self.short_message
+                }
+            ]
+        }
 
 
 def get_auth_token(req):
@@ -48,10 +53,19 @@ def requires_auth():
         return handle_admin_key(auth_token, current_app.config.get('ADMIN_CLIENT_SECRET'))
 
     try:
-        api_keys = get_model_api_keys(client)
+        service = dao_fetch_service_by_id(client)
     except DataError:
         raise AuthError("Invalid token: service id is not the right data type", 403)
-    for api_key in api_keys:
+    except NoResultFound:
+        raise AuthError("Invalid token: service not found", 403)
+
+    if not service.api_keys:
+        raise AuthError("Invalid token: service has no API keys", 403)
+
+    if not service.active:
+        raise AuthError("Invalid token: service is archived", 403)
+
+    for api_key in service.api_keys:
         try:
             get_decode_errors(auth_token, api_key.unsigned_secret)
         except TokenDecodeError:
@@ -62,15 +76,8 @@ def requires_auth():
 
         _request_ctx_stack.top.api_user = api_key
         return
-
-    try:
-        dao_fetch_service_by_id(client)
-    except NoResultFound:
-        raise AuthError("Invalid token: service not found", 403)
-
-    if not api_keys:
-        raise AuthError("Invalid token: service has no API keys", 403)
     else:
+        # service has API keys, but none matching the one the user provided
         raise AuthError("Invalid token: signature, api token is not valid", 403)
 
 

app/celery/tasks.py

@@ -62,10 +62,7 @@ def process_job(job_id):
             'job': str(job.id),
             'to': recipient,
             'row_number': row_number,
-            'personalisation': {
-                key: personalisation.get(key)
-                for key in template.placeholders
-            }
+            'personalisation': dict(personalisation)
         })
 
         if template.template_type == SMS_TYPE:

app/dao/dao_utils.py

@@ -9,7 +9,6 @@ def transactional(func):
     @wraps(func)
     def commit_or_rollback(*args, **kwargs):
         from flask import current_app
-        from app import db
         try:
             res = func(*args, **kwargs)
             db.session.commit()
@@ -27,7 +26,6 @@ def version_class(model_class, history_cls=None):
     def versioned(func):
         @wraps(func)
         def record_version(*args, **kwargs):
-            from app import db
             func(*args, **kwargs)
             history_objects = [create_hist(obj) for obj in
                                itertools.chain(db.session.new, db.session.dirty)

app/dao/services_dao.py

@@ -1,5 +1,5 @@
 import uuid
-from datetime import date
+from datetime import date, datetime
 
 from sqlalchemy import asc, func
 from sqlalchemy.orm import joinedload
@@ -28,30 +28,68 @@ from app.models import (
 from app.statsd_decorators import statsd
 
 
-def dao_fetch_all_services():
-    return Service.query.order_by(
+def dao_fetch_all_services(only_active=False):
+    query = Service.query.order_by(
         asc(Service.created_at)
     ).options(
         joinedload('users')
-    ).all()
+    )
+
+    if only_active:
+        query = query.filter(Service.active)
+
+    return query.all()
 
 
-def dao_fetch_service_by_id(service_id):
-    return Service.query.filter_by(
+def dao_fetch_service_by_id(service_id, only_active=False):
+    query = Service.query.filter_by(
         id=service_id
     ).options(
         joinedload('users')
-    ).one()
+    )
+
+    if only_active:
+        query = query.filter(Service.active)
+
+    return query.one()
 
 
-def dao_fetch_all_services_by_user(user_id):
-    return Service.query.filter(
+def dao_fetch_all_services_by_user(user_id, only_active=False):
+    query = Service.query.filter(
         Service.users.any(id=user_id)
     ).order_by(
         asc(Service.created_at)
     ).options(
         joinedload('users')
-    ).all()
+    )
+
+    if only_active:
+        query = query.filter(Service.active)
+
+    return query.all()
+
+
+@transactional
+@version_class(Service)
+@version_class(Template, TemplateHistory)
+@version_class(ApiKey)
+def dao_deactive_service(service_id):
+    # have to eager load templates and api keys so that we don't flush when we loop through them
+    # to ensure that db.session still contains the models when it comes to creating history objects
+    service = Service.query.options(
+        joinedload('templates'),
+        joinedload('api_keys'),
+    ).filter(Service.id == service_id).one()
+
+    service.active = False
+    service.name = '_archived_' + service.name
+    service.email_from = '_archived_' + service.email_from
+
+    for template in service.templates:
+        template.archived = True
+
+    for api_key in service.api_keys:
+        api_key.expiry_date = datetime.utcnow()
 
 
 def dao_fetch_service_by_id_and_user(service_id, user_id):
@@ -70,6 +108,7 @@ def dao_create_service(service, user):
     service.users.append(user)
     permission_dao.add_default_service_permissions_for_user(user, service)
     service.id = uuid.uuid4()  # must be set now so version history model can use same id
+    service.active = True
     service.research_mode = False
     db.session.add(service)
 

app/dao/users_dao.py

@@ -7,18 +7,26 @@ from app import db
 from app.models import (User, VerifyCode)
 
 
+def _remove_values_for_keys_if_present(dict, keys):
+    for key in keys:
+        dict.pop(key, None)
+
+
 def create_secret_code():
     return ''.join(map(str, random.sample(range(9), 5)))
 
 
+def save_user_attribute(usr, update_dict={}):
+    db.session.query(User).filter_by(id=usr.id).update(update_dict)
+    db.session.commit()
+
+
 def save_model_user(usr, update_dict={}, pwd=None):
     if pwd:
         usr.password = pwd
         usr.password_changed_at = datetime.utcnow()
     if update_dict:
-        if update_dict.get('id'):
-            del update_dict['id']
-            update_dict.pop('password_changed_at')
+        _remove_values_for_keys_if_present(update_dict, ['id', 'password_changed_at'])
         db.session.query(User).filter_by(id=usr.id).update(update_dict)
     else:
         db.session.add(usr)
@@ -74,7 +82,7 @@ def delete_user_verify_codes(user):
     db.session.commit()
 
 
-def get_model_users(user_id=None):
+def get_user_by_id(user_id=None):
     if user_id:
         return User.query.filter_by(id=user_id).one()
     return User.query.filter_by().all()

app/errors.py

@@ -10,7 +10,6 @@ from app.authentication.auth import AuthError
 
 class InvalidRequest(Exception):
     code = None
-    link = None
     fields = []
 
     def __init__(self, message, status_code):
@@ -26,10 +25,13 @@ class InvalidRequest(Exception):
         Version 2 of the public api error response.
         '''
         return {
-            "code": self.code,
-            "message": self.message,
-            "link": self.link,
-            "fields": self.fields
+            "status_code": self.status_code,
+            "errors": [
+                {
+                    "error": self.__class__.__name__,
+                    "message": self.message
+                }
+            ]
         }
 
     def __str__(self):

app/models.py

@@ -115,7 +115,7 @@ class Service(db.Model, Versioned):
         unique=False,
         nullable=True,
         onupdate=datetime.datetime.utcnow)
-    active = db.Column(db.Boolean, index=False, unique=False, nullable=False)
+    active = db.Column(db.Boolean, index=False, unique=False, nullable=False, default=True)
     message_limit = db.Column(db.BigInteger, index=False, unique=False, nullable=False)
     users = db.relationship(
         'User',
@@ -188,7 +188,7 @@ class ApiKey(db.Model, Versioned):
     name = db.Column(db.String(255), nullable=False)
     secret = db.Column(db.String(255), unique=True, nullable=False)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
-    service = db.relationship('Service', backref=db.backref('api_keys', lazy='dynamic'))
+    service = db.relationship('Service', backref='api_keys')
     key_type = db.Column(db.String(255), db.ForeignKey('key_types.name'), index=True, nullable=False)
     expiry_date = db.Column(db.DateTime)
     created_at = db.Column(
@@ -275,7 +275,7 @@ class Template(db.Model):
     content = db.Column(db.Text, index=False, unique=False, nullable=False)
     archived = db.Column(db.Boolean, index=False, nullable=False, default=False)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False, nullable=False)
-    service = db.relationship('Service', backref=db.backref('templates', lazy='dynamic'))
+    service = db.relationship('Service', backref='templates')
     subject = db.Column(db.Text, index=False, unique=False, nullable=True)
     created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
     created_by = db.relationship('User')

app/schema_validation/__init__.py

@@ -1,4 +1,5 @@
 import json
+
 from jsonschema import Draft4Validator, ValidationError
 
 
@@ -16,13 +17,11 @@ def build_error_message(errors, schema):
         field = "'{}' {}".format(e.path[0], e.schema.get('validationMessage')) if e.schema.get(
             'validationMessage') else e.message
         s = field.split("'")
-        field = {s[1]: s[2].strip()}
+        field = {"error": "ValidationError", "message": "{}{}".format(s[1], s[2])}
         fields.append(field)
     message = {
-        "code": "1001",
-        "message": "Validation error occurred - {}".format(schema['title']),
-        "link": "link to error documentation (not yet implemented)",
-        "fields": fields
+        "status_code": 400,
+        "errors": fields
     }
 
     return json.dumps(message)

app/schema_validation/definitions.py

@@ -6,7 +6,7 @@ If the definition is specific to a version put it in a definition file in the ve
 uuid = {
     "type": "string",
     "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
-    "validationMessage": "not a valid UUID",
+    "validationMessage": "is not a valid UUID",
     "code": "1001",  # yet to be implemented
     "link": "link to our error documentation not yet implemented"
 }

app/schemas.py

@@ -105,6 +105,42 @@ class UserSchema(BaseSchema):
         strict = True
 
 
+class UserUpdateAttributeSchema(BaseSchema):
+
+    class Meta:
+        model = models.User
+        exclude = (
+            'id', 'updated_at', 'created_at', 'user_to_service',
+            '_password', 'verify_codes', 'logged_in_at', 'password_changed_at',
+            'failed_login_count', 'state', 'platform_admin')
+        strict = True
+
+    @validates('name')
+    def validate_name(self, value):
+        if not value:
+            raise ValidationError('Invalid name')
+
+    @validates('email_address')
+    def validate_email_address(self, value):
+        try:
+            validate_email_address(value)
+        except InvalidEmailError as e:
+            raise ValidationError(e.message)
+
+    @validates('mobile_number')
+    def validate_mobile_number(self, value):
+        try:
+            validate_phone_number(value)
+        except InvalidPhoneError as error:
+            raise ValidationError('Invalid phone number: {}'.format(error))
+
+    @validates_schema(pass_original=True)
+    def check_unknown_fields(self, data, original_data):
+        for key in original_data:
+            if key not in self.fields:
+                raise ValidationError('Unknown field name {}'.format(key))
+
+
 class ProviderDetailsSchema(BaseSchema):
     class Meta:
         model = models.ProviderDetails
@@ -529,6 +565,7 @@ class UnarchivedTemplateSchema(BaseSchema):
 
 user_schema = UserSchema()
 user_schema_load_json = UserSchema(load_json=True)
+user_update_schema_load_json = UserUpdateAttributeSchema(load_json=True, partial=True)
 service_schema = ServiceSchema()
 service_schema_load_json = ServiceSchema(load_json=True)
 detailed_service_schema = DetailedServiceSchema()

app/service/rest.py

@@ -25,7 +25,8 @@ from app.dao.services_dao import (
     dao_fetch_stats_for_service,
     dao_fetch_todays_stats_for_service,
     dao_fetch_weekly_historical_stats_for_service,
-    dao_fetch_todays_stats_for_all_services
+    dao_fetch_todays_stats_for_all_services,
+    dao_deactive_service
 )
 from app.dao.service_whitelist_dao import (
     dao_fetch_service_whitelist,
@@ -34,7 +35,7 @@ from app.dao.service_whitelist_dao import (
 )
 from app.dao import notifications_dao
 from app.dao.provider_statistics_dao import get_fragment_count
-from app.dao.users_dao import get_model_users
+from app.dao.users_dao import get_user_by_id
 from app.errors import (
     register_errors,
     InvalidRequest
@@ -58,13 +59,16 @@ register_errors(service_blueprint)
 
 @service_blueprint.route('', methods=['GET'])
 def get_services():
+    only_active = request.args.get('only_active') == 'True'
+    detailed = request.args.get('detailed') == 'True'
     user_id = request.args.get('user_id', None)
+
     if user_id:
-        services = dao_fetch_all_services_by_user(user_id)
-    elif request.args.get('detailed') == 'True':
-        return jsonify(data=get_detailed_services())
+        services = dao_fetch_all_services_by_user(user_id, only_active)
+    elif detailed:
+        return jsonify(data=get_detailed_services(only_active))
     else:
-        services = dao_fetch_all_services()
+        services = dao_fetch_all_services(only_active)
     data = service_schema.dump(services, many=True).data
     return jsonify(data=data)
 
@@ -88,7 +92,7 @@ def create_service():
         errors = {'user_id': ['Missing data for required field.']}
         raise InvalidRequest(errors, status_code=400)
 
-    user = get_model_users(data['user_id'])
+    user = get_user_by_id(data['user_id'])
     data.pop('user_id', None)
     valid_service = service_schema.load(request.get_json()).data
     dao_create_service(valid_service, user)
@@ -148,7 +152,7 @@ def get_users_for_service(service_id):
 @service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['POST'])
 def add_user_to_service(service_id, user_id):
     service = dao_fetch_service_by_id(service_id)
-    user = get_model_users(user_id=user_id)
+    user = get_user_by_id(user_id=user_id)
 
     if user in service.users:
         error = 'User id: {} already part of service id: {}'.format(user_id, service_id)
@@ -163,7 +167,7 @@ def add_user_to_service(service_id, user_id):
 @service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['DELETE'])
 def remove_user_from_service(service_id, user_id):
     service = dao_fetch_service_by_id(service_id)
-    user = get_model_users(user_id=user_id)
+    user = get_user_by_id(user_id=user_id)
     if user not in service.users:
         error = 'User not found'
         raise InvalidRequest(error, status_code=404)
@@ -264,8 +268,8 @@ def get_detailed_service(service_id, today_only=False):
     return detailed_service_schema.dump(service).data
 
 
-def get_detailed_services():
-    services = {service.id: service for service in dao_fetch_all_services()}
+def get_detailed_services(only_active=False):
+    services = {service.id: service for service in dao_fetch_all_services(only_active)}
     stats = dao_fetch_todays_stats_for_all_services()
 
     for service_id, rows in itertools.groupby(stats, lambda x: x.service_id):
@@ -312,6 +316,19 @@ def update_whitelist(service_id):
         return '', 204
 
 
+@service_blueprint.route('/<uuid:service_id>/deactivate', methods=['POST'])
+def deactivate_service(service_id):
+    service = dao_fetch_service_by_id(service_id)
+
+    if not service.active:
+        # assume already inactive, don't change service name
+        return '', 204
+
+    dao_deactive_service(service.id)
+
+    return '', 204
+
+
 @service_blueprint.route('/<uuid:service_id>/billable-units')
 def get_billable_unit_count(service_id):
     try:

app/user/rest.py

@@ -4,7 +4,7 @@ from datetime import datetime
 from flask import (jsonify, request, Blueprint, current_app)
 from app import encryption, DATETIME_FORMAT
 from app.dao.users_dao import (
-    get_model_users,
+    get_user_by_id,
     save_model_user,
     create_user_code,
     get_user_code,
@@ -12,7 +12,8 @@ from app.dao.users_dao import (
     increment_failed_login_count,
     reset_failed_login_count,
     get_user_by_email,
-    create_secret_code
+    create_secret_code,
+    save_user_attribute
 )
 from app.dao.permissions_dao import permission_dao
 from app.dao.services_dao import dao_fetch_service_by_id
@@ -22,8 +23,10 @@ from app.schemas import (
     email_data_request_schema,
     user_schema,
     request_verify_code_schema,
+    permission_schema,
     user_schema_load_json,
-    permission_schema)
+    user_update_schema_load_json
+)
 
 from app.celery.tasks import (
     send_sms,
@@ -53,7 +56,7 @@ def create_user():
 
 @user.route('/<uuid:user_id>', methods=['PUT'])
 def update_user(user_id):
-    user_to_update = get_model_users(user_id=user_id)
+    user_to_update = get_user_by_id(user_id=user_id)
     req_json = request.get_json()
     update_dct, errors = user_schema_load_json.load(req_json)
     pwd = req_json.get('password', None)
@@ -66,9 +69,20 @@ def update_user(user_id):
     return jsonify(data=user_schema.dump(user_to_update).data), 200
 
 
+@user.route('/<uuid:user_id>', methods=['POST'])
+def update_user_attribute(user_id):
+    user_to_update = get_user_by_id(user_id=user_id)
+    req_json = request.get_json()
+    update_dct, errors = user_update_schema_load_json.load(req_json)
+    if errors:
+        raise InvalidRequest(errors, status_code=400)
+    save_user_attribute(user_to_update, update_dict=update_dct)
+    return jsonify(data=user_schema.dump(user_to_update).data), 200
+
+
 @user.route('/<uuid:user_id>/verify/password', methods=['POST'])
 def verify_user_password(user_id):
-    user_to_verify = get_model_users(user_id=user_id)
+    user_to_verify = get_user_by_id(user_id=user_id)
 
     txt_pwd = None
     try:
@@ -92,7 +106,7 @@ def verify_user_password(user_id):
 
 @user.route('/<uuid:user_id>/verify/code', methods=['POST'])
 def verify_user_code(user_id):
-    user_to_verify = get_model_users(user_id=user_id)
+    user_to_verify = get_user_by_id(user_id=user_id)
 
     txt_code = None
     resp_json = request.get_json()
@@ -120,7 +134,7 @@ def verify_user_code(user_id):
 
 @user.route('/<uuid:user_id>/sms-code', methods=['POST'])
 def send_user_sms_code(user_id):
-    user_to_send_to = get_model_users(user_id=user_id)
+    user_to_send_to = get_user_by_id(user_id=user_id)
     verify_code, errors = request_verify_code_schema.load(request.get_json())
 
     secret_code = create_secret_code()
@@ -149,7 +163,7 @@ def send_user_sms_code(user_id):
 
 @user.route('/<uuid:user_id>/change-email-verification', methods=['POST'])
 def send_user_confirm_new_email(user_id):
-    user_to_send_to = get_model_users(user_id=user_id)
+    user_to_send_to = get_user_by_id(user_id=user_id)
     email, errors = email_data_request_schema.load(request.get_json())
     if errors:
         raise InvalidRequest(message=errors, status_code=400)
@@ -178,7 +192,7 @@ def send_user_confirm_new_email(user_id):
 
 @user.route('/<uuid:user_id>/email-verification', methods=['POST'])
 def send_user_email_verification(user_id):
-    user_to_send_to = get_model_users(user_id=user_id)
+    user_to_send_to = get_user_by_id(user_id=user_id)
     secret_code = create_secret_code()
     create_user_code(user_to_send_to, secret_code, 'email')
 
@@ -230,7 +244,7 @@ def send_already_registered_email(user_id):
 @user.route('/<uuid:user_id>', methods=['GET'])
 @user.route('', methods=['GET'])
 def get_user(user_id=None):
-    users = get_model_users(user_id=user_id)
+    users = get_user_by_id(user_id=user_id)
     result = user_schema.dump(users, many=True) if isinstance(users, list) else user_schema.dump(users)
     return jsonify(data=result.data)
 
@@ -239,7 +253,7 @@ def get_user(user_id=None):
 def set_permissions(user_id, service_id):
     # TODO fix security hole, how do we verify that the user
     # who is making this request has permission to make the request.
-    user = get_model_users(user_id=user_id)
+    user = get_user_by_id(user_id=user_id)
     service = dao_fetch_service_by_id(service_id=service_id)
     permissions, errors = permission_schema.load(request.get_json(), many=True)
 

app/v2/errors.py

@@ -1,19 +1,14 @@
 import json
-
 from flask import jsonify, current_app
 from jsonschema import ValidationError
 from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
-
 from app.authentication.auth import AuthError
 from app.errors import InvalidRequest
 
 
 class TooManyRequestsError(InvalidRequest):
     status_code = 429
-    # code and link will be in a static file
-    code = "10429"
-    link = "link to docs"
     message_template = 'Exceeded send limits ({}) for today'
 
     def __init__(self, sending_limit):
@@ -22,8 +17,6 @@ class TooManyRequestsError(InvalidRequest):
 
 class BadRequestError(InvalidRequest):
     status_code = 400
-    code = 10400
-    link = "link to documentation"
     message = "An error occurred"
 
     def __init__(self, fields=[], message=None):
@@ -47,7 +40,8 @@ def register_errors(blueprint):
     @blueprint.errorhandler(DataError)
     def no_result_found(e):
         current_app.logger.exception(e)
-        return jsonify(message="No result found"), 404
+        return jsonify(status_code=404,
+                       errors=[{"error": e.__class__.__name__, "message": "No result found"}]), 404
 
     @blueprint.errorhandler(AuthError)
     def auth_error(error):
@@ -56,4 +50,5 @@ def register_errors(blueprint):
     @blueprint.errorhandler(Exception)
     def internal_server_error(error):
         current_app.logger.exception(error)
-        return jsonify(message='Internal server error'), 500
+        return jsonify(status_code=500,
+                       errors=[{"error": error.__class__.__name__, "message": 'Internal server error'}]), 500