Files
notifications-admin/app/notify_client/user_api_client.py

229 lines
8.8 KiB
Python
Raw Normal View History

from itertools import chain
from notifications_python_client.errors import HTTPError
Add Redis cache between admin and API Most of the time spent by the admin app to generate a page is spent waiting for the API. This is slow for three reasons: 1. Talking to the API means going out to the internet, then through nginx, the Flask app, SQLAlchemy, down to the database, and then serialising the result to JSON and making it into a HTTP response 2. Each call to the API is synchronous, therefore if a page needs 3 API calls to render then the second API call won’t be made until the first has finished, and the third won’t start until the second has finished 3. Every request for a service page in the admin app makes a minimum of two requests to the API (`GET /service/…` and `GET /user/…`) Hitting the database will always be the slowest part of an app like Notify. But this slowness is exacerbated by 2. and 3. Conversely every speedup made to 1. is multiplied by 2. and 3. So this pull request aims to make 1. a _lot_ faster by taking nginx, Flask, SQLAlchemy and the database out of the equation. It replaces them with Redis, which as an in-memory key/value store is a lot faster than Postgres. There is still the overhead of going across the network to talk to Redis, but the net improvement is vast. This commit only caches the `GET /service` response, but is written in such a way that we can easily expand to caching other responses down the line. The tradeoff here is that our code is more complex, and we risk introducing edge cases where a cache becomes stale. The mitigations against this are: - invalidating all caches after 24h so a stale cache doesn’t remain around indefinitely - being careful when we add new stuff to the service response --- Some indicative numbers, based on: - `GET http://localhost:6012/services/<service_id>/template/<template_id>` - with the admin app running locally - talking to Redis running locally - also talking to the API running locally, itself talking to a local Postgres instance - times measured with Chrome web inspector, average of 10 requests ╲ | No cache | Cache service | Cache service and user | Cache service, user and template -- | -- | -- | -- | -- **Request time** | 136ms | 97ms | 73ms | 37ms **Improvement** | 0% | 41% | 88% | 265% --- Estimates of how much storage this requires: - Services: 1,942 on production × 2kb = 4Mb - Users: 4,534 on production × 2kb = 9Mb - Templates: 7,079 on production × 4kb = 28Mb
2018-04-06 13:37:49 +01:00
from app.notify_client import NotifyAdminAPIClient, cache
from app.notify_client.models import (
User,
roles,
translate_permissions_from_admin_roles_to_db,
)
ALLOWED_ATTRIBUTES = {
'name',
'email_address',
'mobile_number',
'auth_type',
}
class UserApiClient(NotifyAdminAPIClient):
def __init__(self):
super().__init__("a" * 73, "b")
def init_app(self, app):
super().init_app(app)
2016-01-22 11:21:30 +00:00
self.max_failed_login_count = app.config["MAX_FAILED_LOGIN_COUNT"]
self.admin_url = app.config['ADMIN_BASE_URL']
def register_user(self, name, email_address, mobile_number, password, auth_type):
data = {
"name": name,
"email_address": email_address,
"mobile_number": mobile_number,
"password": password,
"auth_type": auth_type
}
user_data = self.post("/user", data)
2016-01-22 11:21:30 +00:00
return User(user_data['data'], max_failed_login_count=self.max_failed_login_count)
Cache `GET /user` response in Redis In the same way, and for the same reasons that we’re caching the service object. Here’s a sample of the data returned by the API – so we should make sure that any changes to this data invalidate the cache. If we ever change a user’s phone number (for example) directly in the database, then we will need to invalidate this cache manually. ```python {      'data':{         'organisations':[            '4c707b81-4c6d-4d33-9376-17f0de6e0405'       ],       'logged_in_at':'2018-04-10T11:41:03.781990Z',       'id':'2c45486e-177e-40b8-997d-5f4f81a461ca',       'email_address':'test@example.gov.uk',       'platform_admin':False,       'password_changed_at':'2018-01-01 10:10:10.100000',       'permissions':{            '42a9d4f2-1444-4e22-9133-52d9e406213f':[               'manage_api_keys',             'send_letters',             'manage_users',             'manage_templates',             'view_activity',             'send_texts',             'send_emails',             'manage_settings'          ],          'a928eef8-0f25-41ca-b480-0447f29b2c20':[               'manage_users',             'manage_templates',             'manage_settings',             'send_texts',             'send_emails',             'send_letters',             'manage_api_keys',             'view_activity'          ],       },       'state':'active',       'mobile_number':'07700900123',       'failed_login_count':0,       'name':'Example',       'services':[            '6078a8c0-52f5-4c4f-b724-d7d1ff2d3884',          '6afe3c1c-7fda-4d8d-aa8d-769c4bdf7803',       ],       'current_session_id':'fea2ade1-db0a-4c90-93e7-c64a877ce83e',       'auth_type':'sms_auth'    } } ```
2018-04-10 13:30:52 +01:00
def get_user(self, user_id):
return User(self._get_user(user_id)['data'], max_failed_login_count=self.max_failed_login_count)
@cache.set('user-{user_id}')
Cache `GET /user` response in Redis In the same way, and for the same reasons that we’re caching the service object. Here’s a sample of the data returned by the API – so we should make sure that any changes to this data invalidate the cache. If we ever change a user’s phone number (for example) directly in the database, then we will need to invalidate this cache manually. ```python {      'data':{         'organisations':[            '4c707b81-4c6d-4d33-9376-17f0de6e0405'       ],       'logged_in_at':'2018-04-10T11:41:03.781990Z',       'id':'2c45486e-177e-40b8-997d-5f4f81a461ca',       'email_address':'test@example.gov.uk',       'platform_admin':False,       'password_changed_at':'2018-01-01 10:10:10.100000',       'permissions':{            '42a9d4f2-1444-4e22-9133-52d9e406213f':[               'manage_api_keys',             'send_letters',             'manage_users',             'manage_templates',             'view_activity',             'send_texts',             'send_emails',             'manage_settings'          ],          'a928eef8-0f25-41ca-b480-0447f29b2c20':[               'manage_users',             'manage_templates',             'manage_settings',             'send_texts',             'send_emails',             'send_letters',             'manage_api_keys',             'view_activity'          ],       },       'state':'active',       'mobile_number':'07700900123',       'failed_login_count':0,       'name':'Example',       'services':[            '6078a8c0-52f5-4c4f-b724-d7d1ff2d3884',          '6afe3c1c-7fda-4d8d-aa8d-769c4bdf7803',       ],       'current_session_id':'fea2ade1-db0a-4c90-93e7-c64a877ce83e',       'auth_type':'sms_auth'    } } ```
2018-04-10 13:30:52 +01:00
def _get_user(self, user_id):
return self.get("/user/{}".format(user_id))
def get_user_by_email(self, email_address):
user_data = self.get('/user/email', params={'email': email_address})
return User(user_data['data'], max_failed_login_count=self.max_failed_login_count)
def get_user_by_email_or_none(self, email_address):
try:
return self.get_user_by_email(email_address)
except HTTPError as e:
if e.status_code == 404:
return None
2016-01-21 11:33:53 +00:00
def get_users(self):
users_data = self.get("/user")['data']
2016-01-21 11:33:53 +00:00
users = []
for user in users_data:
2016-01-22 11:21:30 +00:00
users.append(User(user, max_failed_login_count=self.max_failed_login_count))
2016-01-21 11:33:53 +00:00
return users
@cache.delete('user-{user_id}')
def update_user_attribute(self, user_id, **kwargs):
data = dict(kwargs)
disallowed_attributes = set(data.keys()) - ALLOWED_ATTRIBUTES
if disallowed_attributes:
raise TypeError('Not allowed to update user attributes: {}'.format(
", ".join(disallowed_attributes)
))
data = dict(**kwargs)
2016-11-10 12:10:01 +00:00
url = "/user/{}".format(user_id)
user_data = self.post(url, data=data)
2016-01-22 11:21:30 +00:00
return User(user_data['data'], max_failed_login_count=self.max_failed_login_count)
@cache.delete('user-{user_id}')
def reset_failed_login_count(self, user_id):
url = "/user/{}/reset-failed-login-count".format(user_id)
user_data = self.post(url, data={})
return User(user_data['data'], max_failed_login_count=self.max_failed_login_count)
@cache.delete('user-{user_id}')
def update_password(self, user_id, password):
data = {"_password": password}
url = "/user/{}/update-password".format(user_id)
user_data = self.post(url, data=data)
return User(user_data['data'], max_failed_login_count=self.max_failed_login_count)
@cache.delete('user-{user_id}')
2016-01-27 17:26:22 +00:00
def verify_password(self, user_id, password):
2016-01-21 11:33:53 +00:00
try:
2016-01-27 17:26:22 +00:00
url = "/user/{}/verify/password".format(user_id)
data = {"password": password}
2016-01-27 17:13:56 +00:00
self.post(url, data=data)
return True
2016-01-21 11:33:53 +00:00
except HTTPError as e:
if e.status_code == 400 or e.status_code == 404:
return False
def send_verify_code(self, user_id, code_type, to, next_string=None):
data = {'to': to}
if next_string:
data['next'] = next_string
if code_type == 'email':
data['email_auth_link_host'] = self.admin_url
endpoint = '/user/{0}/{1}-code'.format(user_id, code_type)
self.post(endpoint, data=data)
def send_verify_email(self, user_id, to):
data = {'to': to}
endpoint = '/user/{0}/email-verification'.format(user_id)
self.post(endpoint, data=data)
def send_already_registered_email(self, user_id, to):
data = {'email': to}
endpoint = '/user/{0}/email-already-registered'.format(user_id)
self.post(endpoint, data=data)
@cache.delete('user-{user_id}')
def check_verify_code(self, user_id, code, code_type):
data = {'code_type': code_type, 'code': code}
endpoint = '/user/{}/verify/code'.format(user_id)
try:
self.post(endpoint, data=data)
return True, ''
except HTTPError as e:
if e.status_code == 400 or e.status_code == 404:
if 'Code not found' in e.message:
return False, 'Code not found'
elif 'Code has expired' in e.message:
return False, 'Code has expired'
else:
# TODO what is the default message?
return False, 'Code not found'
raise e
def get_users_for_service(self, service_id):
endpoint = '/service/{}/users'.format(service_id)
resp = self.get(endpoint)
return [User(data) for data in resp['data']]
def get_count_of_users_with_permission(self, service_id, permission):
if permission not in roles.keys():
raise TypeError('{} is not a valid permission'.format(permission))
return len([
user for user in self.get_users_for_service(service_id)
if user.has_permission_for_service(service_id, permission)
])
2018-02-19 16:53:29 +00:00
def get_users_for_organisation(self, org_id):
endpoint = '/organisations/{}/users'.format(org_id)
resp = self.get(endpoint)
return [User(data) for data in resp['data']]
@cache.delete('service-{service_id}')
@cache.delete('user-{user_id}')
def add_user_to_service(self, service_id, user_id, permissions):
# permissions passed in are the combined admin roles, not db permissions
endpoint = '/service/{}/users/{}'.format(service_id, user_id)
data = [{'permission': x} for x in translate_permissions_from_admin_roles_to_db(permissions)]
self.post(endpoint, data=data)
@cache.delete('user-{user_id}')
2018-02-19 16:53:29 +00:00
def add_user_to_organisation(self, org_id, user_id):
resp = self.post('/organisations/{}/users/{}'.format(org_id, user_id), data={})
return User(resp['data'], max_failed_login_count=self.max_failed_login_count)
@cache.delete('user-{user_id}')
def set_user_permissions(self, user_id, service_id, permissions):
# permissions passed in are the combined admin roles, not db permissions
data = [{'permission': x} for x in translate_permissions_from_admin_roles_to_db(permissions)]
endpoint = '/user/{}/service/{}/permission'.format(user_id, service_id)
2016-03-07 18:18:52 +00:00
self.post(endpoint, data=data)
def send_reset_password_url(self, email_address):
endpoint = '/user/reset-password'
data = {'email': email_address}
self.post(endpoint, data=data)
def find_users_by_full_or_partial_email(self, email_address):
endpoint = '/user/find-users-by-email'
data = {'email': email_address}
users = self.post(endpoint, data=data)
return users
2018-02-19 16:53:29 +00:00
def is_email_already_in_use(self, email_address):
if self.get_user_by_email_or_none(email_address):
2018-02-19 16:53:29 +00:00
return True
return False
def activate_user(self, user):
if user.state == 'pending':
Cache `GET /user` response in Redis In the same way, and for the same reasons that we’re caching the service object. Here’s a sample of the data returned by the API – so we should make sure that any changes to this data invalidate the cache. If we ever change a user’s phone number (for example) directly in the database, then we will need to invalidate this cache manually. ```python {      'data':{         'organisations':[            '4c707b81-4c6d-4d33-9376-17f0de6e0405'       ],       'logged_in_at':'2018-04-10T11:41:03.781990Z',       'id':'2c45486e-177e-40b8-997d-5f4f81a461ca',       'email_address':'test@example.gov.uk',       'platform_admin':False,       'password_changed_at':'2018-01-01 10:10:10.100000',       'permissions':{            '42a9d4f2-1444-4e22-9133-52d9e406213f':[               'manage_api_keys',             'send_letters',             'manage_users',             'manage_templates',             'view_activity',             'send_texts',             'send_emails',             'manage_settings'          ],          'a928eef8-0f25-41ca-b480-0447f29b2c20':[               'manage_users',             'manage_templates',             'manage_settings',             'send_texts',             'send_emails',             'send_letters',             'manage_api_keys',             'view_activity'          ],       },       'state':'active',       'mobile_number':'07700900123',       'failed_login_count':0,       'name':'Example',       'services':[            '6078a8c0-52f5-4c4f-b724-d7d1ff2d3884',          '6afe3c1c-7fda-4d8d-aa8d-769c4bdf7803',       ],       'current_session_id':'fea2ade1-db0a-4c90-93e7-c64a877ce83e',       'auth_type':'sms_auth'    } } ```
2018-04-10 13:30:52 +01:00
user_data = self._activate_user(user.id)
return User(user_data['data'], max_failed_login_count=self.max_failed_login_count)
else:
return user
@cache.delete('user-{user_id}')
Cache `GET /user` response in Redis In the same way, and for the same reasons that we’re caching the service object. Here’s a sample of the data returned by the API – so we should make sure that any changes to this data invalidate the cache. If we ever change a user’s phone number (for example) directly in the database, then we will need to invalidate this cache manually. ```python {      'data':{         'organisations':[            '4c707b81-4c6d-4d33-9376-17f0de6e0405'       ],       'logged_in_at':'2018-04-10T11:41:03.781990Z',       'id':'2c45486e-177e-40b8-997d-5f4f81a461ca',       'email_address':'test@example.gov.uk',       'platform_admin':False,       'password_changed_at':'2018-01-01 10:10:10.100000',       'permissions':{            '42a9d4f2-1444-4e22-9133-52d9e406213f':[               'manage_api_keys',             'send_letters',             'manage_users',             'manage_templates',             'view_activity',             'send_texts',             'send_emails',             'manage_settings'          ],          'a928eef8-0f25-41ca-b480-0447f29b2c20':[               'manage_users',             'manage_templates',             'manage_settings',             'send_texts',             'send_emails',             'send_letters',             'manage_api_keys',             'view_activity'          ],       },       'state':'active',       'mobile_number':'07700900123',       'failed_login_count':0,       'name':'Example',       'services':[            '6078a8c0-52f5-4c4f-b724-d7d1ff2d3884',          '6afe3c1c-7fda-4d8d-aa8d-769c4bdf7803',       ],       'current_session_id':'fea2ade1-db0a-4c90-93e7-c64a877ce83e',       'auth_type':'sms_auth'    } } ```
2018-04-10 13:30:52 +01:00
def _activate_user(self, user_id):
return self.post("/user/{}/activate".format(user_id), data=None)
def send_change_email_verification(self, user_id, new_email):
endpoint = '/user/{}/change-email-verification'.format(user_id)
data = {'email': new_email}
self.post(endpoint, data)
def get_organisations_and_services_for_user(self, user):
endpoint = '/user/{}/organisations-and-services'.format(user.id)
return self.get(endpoint)
def get_services_for_user(self, user):
orgs_and_services_for_user = self.get_organisations_and_services_for_user(user)
all_services = orgs_and_services_for_user['services_without_organisations'] + next(chain(
org['services'] for org in orgs_and_services_for_user['organisations']
), [])
return sorted(all_services, key=lambda service: service['name'])
def get_service_ids_for_user(self, user):
return {
service['id'] for service in self.get_services_for_user(user)
}
def user_belongs_to_service(self, user, service_id):
return service_id in self.get_service_ids_for_user(user)