Make user API client return JSON, not a model

The data flow of other bits of our application looks like this:
```
                         API (returns JSON)
                                  ⬇
          API client (returns a built in type, usually `dict`)
                                  ⬇
          Model (returns an instance, eg of type `Service`)
                                  ⬇
                         View (returns HTML)
```
The user API client was architected weirdly, in that it returned a model
directly, like this:

```
                         API (returns JSON)
                                  ⬇
    API client (returns a model, of type `User`, `InvitedUser`, etc)
                                  ⬇
                         View (returns HTML)
```

This mixing of different layers of the application is bad because it
makes it hard to write model code that doesn’t have circular
dependencies. As our application gets more complicated we will be
relying more on models to manage this complexity, so we should make it
easy, not hard to write them.

It also means that most of our mocking was of the User model, not just
the underlying JSON. So it would have been easy to introduce subtle bugs
to the user model, because it wasn’t being comprehensively tested. A lot
of the changed lines of code in this commit mean changing the tests to
mock only the JSON, which means that the model layer gets implicitly
tested.

For those reasons this commit changes the user API client to return
JSON, not an instance of `User` or other models.

app/models/__init__.py

@@ -25,3 +25,7 @@ class JSONModel():
             return next(thing for thing in things if thing['id'] == str(id))
         except StopIteration:
             abort(404)
+
+
+class InviteTokenError(Exception):
+    pass

app/models/organisation.py

@@ -131,20 +131,18 @@ class Organisation(JSONModel):
         return [s for s in self.services if not s['active'] or s['restricted']]
 
     @cached_property
-    def active_users(self):
-        # need to put this here to prevent cyclical import
-        from app.notify_client.user_api_client import user_api_client
-        return user_api_client.get_users_for_organisation(org_id=self.id)
+    def invited_users(self):
+        from app.models.user import OrganisationInvitedUsers
+        return OrganisationInvitedUsers(self.id)
 
     @cached_property
-    def invited_users(self):
-        # need to put this here to prevent cyclical import
-        from app.notify_client.org_invite_api_client import org_invite_api_client
-        return org_invite_api_client.get_invites_for_organisation(org_id=self.id)
+    def active_users(self):
+        from app.models.user import OrganisationUsers
+        return OrganisationUsers(self.id)
 
     @cached_property
     def team_members(self):
         return sorted(
-            self.active_users + [i for i in self.invited_users if i.status != 'accepted'],
-            key=lambda user: user.email_address,
+            self.invited_users + self.active_users,
+            key=lambda user: user.email_address.lower(),
         )

app/models/roles_and_permissions.py

@@ -0,0 +1,44 @@
+from itertools import chain
+
+roles = {
+    'send_messages': ['send_texts', 'send_emails', 'send_letters'],
+    'manage_templates': ['manage_templates'],
+    'manage_service': ['manage_users', 'manage_settings'],
+    'manage_api_keys': ['manage_api_keys'],
+    'view_activity': ['view_activity'],
+}
+
+# same dict as above, but flipped round
+roles_by_permission = {
+    permission: next(
+        role for role, permissions in roles.items() if permission in permissions
+    ) for permission in chain(*list(roles.values()))
+}
+
+all_permissions = set(roles_by_permission.values())
+
+permissions = (
+    ('view_activity', 'See dashboard'),
+    ('send_messages', 'Send messages'),
+    ('manage_templates', 'Add and edit templates'),
+    ('manage_service', 'Manage settings, team and usage'),
+    ('manage_api_keys', 'Manage API integration'),
+)
+
+
+def translate_permissions_from_db_to_admin_roles(permissions):
+    """
+    Given a list of database permissions, return a set of roles
+
+    look them up in roles_by_permission, falling back to just passing through from the api if they aren't in the dict
+    """
+    return {roles_by_permission.get(permission, permission) for permission in permissions}
+
+
+def translate_permissions_from_admin_roles_to_db(permissions):
+    """
+    Given a list of admin roles (ie: checkboxes on a permissions edit page for example), return a set of db permissions
+
+    Looks them up in the roles dict, falling back to just passing through if they're not recognised.
+    """
+    return set(chain.from_iterable(roles.get(permission, [permission]) for permission in permissions))

app/models/service.py

@@ -4,6 +4,7 @@ from werkzeug.utils import cached_property
 
 from app.models import JSONModel
 from app.models.organisation import Organisation
+from app.models.user import InvitedUsers, User, Users
 from app.notify_client.api_key_api_client import api_key_api_client
 from app.notify_client.billing_api_client import billing_api_client
 from app.notify_client.email_branding_client import email_branding_client
@@ -16,7 +17,6 @@ from app.notify_client.service_api_client import service_api_client
 from app.notify_client.template_folder_api_client import (
     template_folder_api_client,
 )
-from app.notify_client.user_api_client import user_api_client
 from app.utils import get_default_sms_sender
 
 
@@ -60,6 +60,10 @@ class Service(JSONModel):
         if 'permissions' not in self._dict:
             self.permissions = {'email', 'sms', 'letter'}
 
+    @classmethod
+    def from_id(cls, service_id):
+        return cls(service_api_client.get_service(service_id)['data'])
+
     def update(self, **kwargs):
         return service_api_client.update_service(self.id, **kwargs)
 
@@ -99,11 +103,11 @@ class Service(JSONModel):
 
     @cached_property
     def invited_users(self):
-        return invite_api_client.get_invites_for_service(service_id=self.id)
+        return InvitedUsers(self.id)
 
     @cached_property
     def active_users(self):
-        return user_api_client.get_users_for_service(service_id=self.id)
+        return Users(self.id)
 
     @cached_property
     def team_members(self):
@@ -114,16 +118,12 @@ class Service(JSONModel):
 
     @cached_property
     def has_team_members(self):
-        return (
-            user_api_client.get_count_of_users_with_permission(
-                self.id, 'manage_service'
-            ) + invite_api_client.get_count_of_invites_with_permission(
-                self.id, 'manage_service'
-            )
-        ) > 1
+        return len([
+            user for user in self.team_members
+            if user.has_permission_for_service(self.id, 'manage_service')
+        ]) > 1
 
     def cancel_invite(self, invited_user_id):
-
         if str(invited_user_id) not in {user.id for user in self.invited_users}:
             abort(404)
 
@@ -137,7 +137,7 @@ class Service(JSONModel):
         if str(user_id) not in {user.id for user in self.active_users}:
             abort(404)
 
-        return user_api_client.get_user(user_id)
+        return User.from_id(user_id)
 
     @cached_property
     def all_templates(self):

app/models/user.py

@@ -1,37 +1,19 @@
-from itertools import chain
-
-from flask import abort, request, session
-from flask_login import AnonymousUserMixin, UserMixin
+from flask import abort, current_app, request, session
+from flask_login import AnonymousUserMixin, UserMixin, login_user
+from notifications_python_client.errors import HTTPError
 from werkzeug.utils import cached_property
 
 from app.models.organisation import Organisation
-from app.notify_client.organisations_api_client import organisations_client
-from app.utils import is_gov_user
-
-roles = {
-    'send_messages': ['send_texts', 'send_emails', 'send_letters'],
-    'manage_templates': ['manage_templates'],
-    'manage_service': ['manage_users', 'manage_settings'],
-    'manage_api_keys': ['manage_api_keys'],
-    'view_activity': ['view_activity'],
-}
-
-# same dict as above, but flipped round
-roles_by_permission = {
-    permission: next(
-        role for role, permissions in roles.items() if permission in permissions
-    ) for permission in chain(*list(roles.values()))
-}
-
-all_permissions = set(roles_by_permission.values())
-
-permissions = (
-    ('view_activity', 'See dashboard'),
-    ('send_messages', 'Send messages'),
-    ('manage_templates', 'Add and edit templates'),
-    ('manage_service', 'Manage settings, team and usage'),
-    ('manage_api_keys', 'Manage API integration'),
+from app.models.roles_and_permissions import (
+    all_permissions,
+    translate_permissions_from_db_to_admin_roles,
 )
+from app.notify_client import InviteTokenError
+from app.notify_client.invite_api_client import invite_api_client
+from app.notify_client.org_invite_api_client import org_invite_api_client
+from app.notify_client.organisations_api_client import organisations_client
+from app.notify_client.user_api_client import user_api_client
+from app.utils import is_gov_user
 
 
 def _get_service_id_from_view_args():
@@ -42,26 +24,8 @@ def _get_org_id_from_view_args():
     return str(request.view_args.get('org_id', '')) or None
 
 
-def translate_permissions_from_db_to_admin_roles(permissions):
-    """
-    Given a list of database permissions, return a set of roles
-
-    look them up in roles_by_permission, falling back to just passing through from the api if they aren't in the dict
-    """
-    return {roles_by_permission.get(permission, permission) for permission in permissions}
-
-
-def translate_permissions_from_admin_roles_to_db(permissions):
-    """
-    Given a list of admin roles (ie: checkboxes on a permissions edit page for example), return a set of db permissions
-
-    Looks them up in the roles dict, falling back to just passing through if they're not recognised.
-    """
-    return set(chain.from_iterable(roles.get(permission, [permission]) for permission in permissions))
-
-
 class User(UserMixin):
-    def __init__(self, fields, max_failed_login_count=3):
+    def __init__(self, fields):
         self.id = fields.get('id')
         self.name = fields.get('name')
         self.email_address = fields.get('email_address')
@@ -71,13 +35,43 @@ class User(UserMixin):
         self.auth_type = fields.get('auth_type')
         self.failed_login_count = fields.get('failed_login_count')
         self.state = fields.get('state')
-        self.max_failed_login_count = max_failed_login_count
+        self.max_failed_login_count = current_app.config["MAX_FAILED_LOGIN_COUNT"]
         self.logged_in_at = fields.get('logged_in_at')
         self.platform_admin = fields.get('platform_admin')
         self.current_session_id = fields.get('current_session_id')
         self.services = fields.get('services', [])
         self.organisations = fields.get('organisations', [])
 
+    @classmethod
+    def from_id(cls, user_id):
+        return cls(user_api_client.get_user(user_id))
+
+    @classmethod
+    def from_email_address(cls, email_address):
+        return cls(user_api_client.get_user_by_email(email_address))
+
+    @classmethod
+    def from_email_address_or_none(cls, email_address):
+        response = user_api_client.get_user_by_email_or_none(email_address)
+        if response:
+            return cls(response)
+        return None
+
+    @staticmethod
+    def already_registered(email_address):
+        return bool(User.from_email_address_or_none(email_address))
+
+    @classmethod
+    def from_email_address_and_password_or_none(cls, email_address, password):
+        user = cls.from_email_address_or_none(email_address)
+        if not user:
+            return None
+        if user.locked:
+            return None
+        if not user_api_client.verify_password(user.id, password):
+            return None
+        return user
+
     def _set_permissions(self, permissions_by_service):
         """
         Permissions is a dict {'service_id': ['permission a', 'permission b', 'permission c']}
@@ -96,13 +90,61 @@ class User(UserMixin):
             in permissions_by_service.items()
         }
 
-    def get_id(self):
-        return self.id
+    def update(self, **kwargs):
+        response = user_api_client.update_user_attribute(self.id, **kwargs)
+        self.__init__(response)
+
+    def update_password(self, password):
+        response = user_api_client.update_password(self.id, password)
+        self.__init__(response)
+
+    def set_permissions(self, service_id, permissions, folder_permissions):
+        user_api_client.set_user_permissions(
+            self.id,
+            service_id,
+            permissions=permissions,
+            folder_permissions=folder_permissions,
+        )
 
     def logged_in_elsewhere(self):
         # if the current user (ie: db object) has no session, they've never logged in before
         return self.current_session_id is not None and session.get('current_session_id') != self.current_session_id
 
+    def activate(self):
+        if self.state == 'pending':
+            user_data = user_api_client.activate_user(self.id)
+            return self.__class__(user_data['data'])
+        else:
+            return self
+
+    def login(self):
+        login_user(self)
+
+    def sign_in(self):
+
+        session['user_details'] = {"email": self.email_address, "id": self.id}
+
+        if not self.is_active:
+            return False
+
+        if self.email_auth:
+            user_api_client.send_verify_code(self.id, 'email', None, request.args.get('next'))
+        if self.sms_auth:
+            user_api_client.send_verify_code(self.id, 'sms', self.mobile_number)
+
+        return True
+
+    @property
+    def sms_auth(self):
+        return self.auth_type == 'sms_auth'
+
+    @property
+    def email_auth(self):
+        return self.auth_type == 'email_auth'
+
+    def reset_failed_login_count(self):
+        user_api_client.reset_failed_login_count(self.id)
+
     @property
     def is_active(self):
         return self.state == 'active'
@@ -181,7 +223,8 @@ class User(UserMixin):
         if not self.belongs_to_service(service_id):
             abort(403)
 
-    def is_locked(self):
+    @property
+    def locked(self):
         return self.failed_login_count >= self.max_failed_login_count
 
     @property
@@ -225,9 +268,39 @@ class User(UserMixin):
             dct['password'] = self._password
         return dct
 
+    @classmethod
+    def register(
+        cls,
+        name,
+        email_address,
+        mobile_number,
+        password,
+        auth_type,
+    ):
+        return cls(user_api_client.register_user(
+            name,
+            email_address,
+            mobile_number or None,
+            password,
+            auth_type,
+        ))
+
     def set_password(self, pwd):
         self._password = pwd
 
+    def send_verify_email(self):
+        user_api_client.send_verify_email(self.id, self.email_address)
+
+    def send_verify_code(self, to=None):
+        user_api_client.send_verify_code(self.id, 'sms', to or self.mobile_number)
+
+    def send_already_registered_email(self):
+        user_api_client.send_already_registered_email(self.id, self.email_address)
+
+    def refresh_session_id(self):
+        self.current_session_id = user_api_client.get_user(self.id).get('current_session_id')
+        session['current_session_id'] = self.current_session_id
+
 
 class InvitedUser(object):
 
@@ -243,7 +316,7 @@ class InvitedUser(object):
                  folder_permissions):
         self.id = id
         self.service = str(service)
-        self.from_user = from_user
+        self._from_user = from_user
         self.email_address = email_address
         if isinstance(permissions, list):
             self.permissions = permissions
@@ -258,6 +331,63 @@ class InvitedUser(object):
         self.permissions = translate_permissions_from_db_to_admin_roles(self.permissions)
         self.folder_permissions = folder_permissions
 
+    @classmethod
+    def create(
+        cls,
+        invite_from_id,
+        service_id,
+        email_address,
+        permissions,
+        auth_type,
+        folder_permissions,
+    ):
+        return cls(**invite_api_client.create_invite(
+            invite_from_id,
+            service_id,
+            email_address,
+            permissions,
+            auth_type,
+            folder_permissions,
+        ))
+
+    def accept_invite(self):
+        invite_api_client.accept_invite(self.service, self.id)
+
+    def add_to_service(self):
+        user_api_client.add_user_to_service(
+            self.service,
+            self.id,
+            self.permissions,
+            self.folder_permissions,
+        )
+
+    @property
+    def from_user(self):
+        return User.from_id(self._from_user)
+
+    @property
+    def sms_auth(self):
+        return self.auth_type == 'sms_auth'
+
+    @property
+    def email_auth(self):
+        return self.auth_type == 'email_auth'
+
+    @classmethod
+    def from_token(cls, token):
+        try:
+            return cls(**invite_api_client.check_token(token))
+        except HTTPError as exception:
+            if exception.status_code == 400 and 'invitation' in exception.message:
+                raise InviteTokenError(exception.message['invitation'])
+            else:
+                raise exception
+
+    @classmethod
+    def from_session(cls):
+        invited_user = session.get('invited_user')
+        return cls(**invited_user) if invited_user else None
+
     def has_permissions(self, *permissions):
         if self.status == 'cancelled':
             return False
@@ -271,12 +401,12 @@ class InvitedUser(object):
     def __eq__(self, other):
         return ((self.id,
                 self.service,
-                self.from_user,
+                self._from_user,
                 self.email_address,
                 self.auth_type,
                 self.status) == (other.id,
                 other.service,
-                other.from_user,
+                other._from_user,
                 other.email_address,
                 other.auth_type,
                 other.status))
@@ -284,7 +414,7 @@ class InvitedUser(object):
     def serialize(self, permissions_as_string=False):
         data = {'id': self.id,
                 'service': self.service,
-                'from_user': self.from_user,
+                'from_user': self._from_user,
                 'email_address': self.email_address,
                 'status': self.status,
                 'created_at': str(self.created_at),
@@ -307,7 +437,7 @@ class InvitedOrgUser(object):
     def __init__(self, id, organisation, invited_by, email_address, status, created_at):
         self.id = id
         self.organisation = str(organisation)
-        self.invited_by = invited_by
+        self._invited_by = invited_by
         self.email_address = email_address
         self.status = status
         self.created_at = created_at
@@ -315,30 +445,97 @@ class InvitedOrgUser(object):
     def __eq__(self, other):
         return ((self.id,
                 self.organisation,
-                self.invited_by,
+                self._invited_by,
                 self.email_address,
                 self.status) == (other.id,
                 other.organisation,
-                other.invited_by,
+                other._invited_by,
                 other.email_address,
                 other.status))
 
+    @classmethod
+    def create(cls, invite_from_id, org_id, email_address):
+        return cls(**org_invite_api_client.create_invite(
+            invite_from_id, org_id, email_address
+        ))
+
+    @classmethod
+    def from_session(cls):
+        invited_org_user = session.get('invited_org_user')
+        return cls(**invited_org_user) if invited_org_user else None
+
     def serialize(self, permissions_as_string=False):
         data = {'id': self.id,
                 'organisation': self.organisation,
-                'invited_by': self.invited_by,
+                'invited_by': self._invited_by,
                 'email_address': self.email_address,
                 'status': self.status,
                 'created_at': str(self.created_at)
                 }
         return data
 
+    @property
+    def invited_by(self):
+        return User.from_id(self._invited_by)
+
+    @classmethod
+    def from_token(cls, token):
+        try:
+            return cls(**org_invite_api_client.check_token(token))
+        except HTTPError as exception:
+            if exception.status_code == 400 and 'invitation' in exception.message:
+                raise InviteTokenError(exception.message['invitation'])
+            else:
+                raise exception
+
+    def accept_invite(self):
+        org_invite_api_client.accept_invite(self.organisation, self.id)
+
+    def add_to_organisation(self):
+        user_api_client.add_user_to_organisation(self.organisation, self.id)
+
 
 class AnonymousUser(AnonymousUserMixin):
     # set the anonymous user so that if a new browser hits us we don't error http://stackoverflow.com/a/19275188
+
     def logged_in_elsewhere(self):
         return False
 
     @property
     def default_organisation(self):
         return Organisation(None)
+
+
+class Users():
+
+    client = user_api_client.get_users_for_service
+
+    def __init__(self, service_id):
+        self.users = self.client(service_id)
+
+    def __iter__(self):
+        return (User(user) for user in self.users)
+
+    def __add__(self, other):
+        return list(self) + list(other)
+
+
+class OrganisationUsers(Users):
+    client = user_api_client.get_users_for_organisation
+
+
+class InvitedUsers(Users):
+
+    client = invite_api_client.get_invites_for_service
+    model = InvitedUser
+
+    def __iter__(self):
+        return (
+            self.model(**invite) for invite in self.users
+            if invite['status'] != 'accepted'
+        )
+
+
+class OrganisationInvitedUsers(InvitedUsers):
+    client = org_invite_api_client.get_invites_for_organisation
+    model = InvitedOrgUser