Merge pull request #2014 from alphagov/redis-spike-CHS

Use Redis to cache API calls in admin app

app/config.py

@@ -74,6 +74,9 @@ class Config(object):
     ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
     CHECK_PROXY_HEADER = False
 
+    REDIS_URL = os.environ.get('REDIS_URL')
+    REDIS_ENABLED = os.environ.get('REDIS_ENABLED') == '1'
+
 
 class Development(Config):
     NOTIFY_LOG_PATH = 'application.log'

app/notify_client/__init__.py

@@ -2,6 +2,7 @@ from flask_login import current_user
 from flask import has_request_context, request, abort
 from notifications_python_client.base import BaseAPIClient
 from notifications_python_client import __version__
+from notifications_utils.clients.redis.redis_client import RedisClient
 
 
 def _attach_current_user(data):
@@ -12,11 +13,16 @@ def _attach_current_user(data):
 
 
 class NotifyAdminAPIClient(BaseAPIClient):
+
+    redis_client = RedisClient()
+
     def init_app(self, app):
         self.base_url = app.config['API_HOST_NAME']
         self.service_id = app.config['ADMIN_CLIENT_USER_NAME']
         self.api_key = app.config['ADMIN_CLIENT_SECRET']
         self.route_secret = app.config['ROUTE_SECRET_KEY_1']
+        self.redis_client.init_app(app)
+        self.redis_client.redis_store.decode_responses = True
 
     def generate_headers(self, api_token):
         headers = {

app/notify_client/cache.py

@@ -0,0 +1,52 @@
+import json
+from datetime import timedelta
+from functools import wraps
+
+TTL = int(timedelta(hours=24).total_seconds())
+
+
+def _make_key(prefix, args, key_from_args):
+
+    if key_from_args is None:
+        key_from_args = [0]
+
+    return '-'.join(
+        [prefix] + [args[index] for index in key_from_args]
+    )
+
+
+def set(prefix, key_from_args=None):
+
+    def _set(client_method):
+
+        @wraps(client_method)
+        def new_client_method(client_instance, *args, **kwargs):
+            redis_key = _make_key(prefix, args, key_from_args)
+            cached = client_instance.redis_client.get(redis_key)
+            if cached:
+                return json.loads(cached.decode('utf-8'))
+            api_response = client_method(client_instance, *args, **kwargs)
+            client_instance.redis_client.set(
+                redis_key,
+                json.dumps(api_response),
+                ex=TTL,
+            )
+            return api_response
+
+        return new_client_method
+    return _set
+
+
+def delete(prefix, key_from_args=None):
+
+    def _delete(client_method):
+
+        @wraps(client_method)
+        def new_client_method(client_instance, *args, **kwargs):
+            redis_key = _make_key(prefix, args, key_from_args)
+            client_instance.redis_client.delete(redis_key)
+            return client_method(client_instance, *args, **kwargs)
+
+        return new_client_method
+
+    return _delete

app/notify_client/organisations_api_client.py

@@ -1,4 +1,4 @@
-from app.notify_client import NotifyAdminAPIClient, _attach_current_user
+from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache
 
 
 class OrganisationsClient(NotifyAdminAPIClient):
@@ -27,6 +27,7 @@ class OrganisationsClient(NotifyAdminAPIClient):
     def get_service_organisation(self, service_id):
         return self.get(url="/service/{}/organisation".format(service_id))
 
+    @cache.delete('service')
     def update_service_organisation(self, service_id, org_id):
         data = {
             'service_id': service_id

app/notify_client/service_api_client.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from app.notify_client import NotifyAdminAPIClient, _attach_current_user
+from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache
 
 
 class ServiceAPIClient(NotifyAdminAPIClient):
@@ -33,6 +33,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
         data = _attach_current_user(data)
         return self.post("/service", data)['data']['id']
 
+    @cache.set('service')
     def get_service(self, service_id):
         return self._get_service(service_id, detailed=False, today_only=False)
 
@@ -72,6 +73,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
         params_dict['only_active'] = True
         return self.get_services(params_dict)
 
+    @cache.delete('service')
     def update_service(
         self,
         service_id,
@@ -108,18 +110,23 @@ class ServiceAPIClient(NotifyAdminAPIClient):
         endpoint = "/service/{0}".format(service_id)
         return self.post(endpoint, data)
 
+    # This method is not cached because it calls through to one which is
     def update_service_with_properties(self, service_id, properties):
         return self.update_service(service_id, **properties)
 
+    @cache.delete('service')
     def archive_service(self, service_id):
         return self.post('/service/{}/archive'.format(service_id), data=None)
 
+    @cache.delete('service')
     def suspend_service(self, service_id):
         return self.post('/service/{}/suspend'.format(service_id), data=None)
 
+    @cache.delete('service')
     def resume_service(self, service_id):
         return self.post('/service/{}/resume'.format(service_id), data=None)
 
+    @cache.delete('service')
     def remove_user_from_service(self, service_id, user_id):
         """
         Remove a user from a service
@@ -258,6 +265,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
     def get_whitelist(self, service_id):
         return self.get(url='/service/{}/whitelist'.format(service_id))
 
+    @cache.delete('service')
     def update_whitelist(self, service_id, data):
         return self.put(url='/service/{}/whitelist'.format(service_id), data=data)
 
@@ -295,6 +303,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             '/service/{}/inbound-sms/summary'.format(service_id)
         )
 
+    @cache.delete('service')
     def create_service_inbound_api(self, service_id, url, bearer_token, user_id):
         data = {
             "url": url,
@@ -303,6 +312,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
         }
         return self.post("/service/{}/inbound-api".format(service_id), data)
 
+    @cache.delete('service')
     def update_service_inbound_api(self, service_id, url, bearer_token, user_id, inbound_api_id):
         data = {
             "url": url,
@@ -334,6 +344,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             )
         )
 
+    @cache.delete('service')
     def add_reply_to_email_address(self, service_id, email_address, is_default=False):
         return self.post(
             "/service/{}/email-reply-to".format(service_id),
@@ -343,6 +354,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             }
         )
 
+    @cache.delete('service')
     def update_reply_to_email_address(self, service_id, reply_to_email_id, email_address, is_default=False):
         return self.post(
             "/service/{}/email-reply-to/{}".format(
@@ -361,6 +373,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
     def get_letter_contact(self, service_id, letter_contact_id):
         return self.get("/service/{}/letter-contact/{}".format(service_id, letter_contact_id))
 
+    @cache.delete('service')
     def add_letter_contact(self, service_id, contact_block, is_default=False):
         return self.post(
             "/service/{}/letter-contact".format(service_id),
@@ -370,6 +383,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             }
         )
 
+    @cache.delete('service')
     def update_letter_contact(self, service_id, letter_contact_id, contact_block, is_default=False):
         return self.post(
             "/service/{}/letter-contact/{}".format(
@@ -395,6 +409,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             "/service/{}/sms-sender/{}".format(service_id, sms_sender_id)
         )
 
+    @cache.delete('service')
     def add_sms_sender(self, service_id, sms_sender, is_default=False, inbound_number_id=None):
         data = {
             "sms_sender": sms_sender,
@@ -404,6 +419,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             data["inbound_number_id"] = inbound_number_id
         return self.post("/service/{}/sms-sender".format(service_id), data=data)
 
+    @cache.delete('service')
     def update_sms_sender(self, service_id, sms_sender_id, sms_sender, is_default=False):
         return self.post(
             "/service/{}/sms-sender/{}".format(service_id, sms_sender_id),
@@ -420,6 +436,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             )
         )['data']
 
+    @cache.delete('service')
     def update_service_callback_api(self, service_id, url, bearer_token, user_id, callback_api_id):
         data = {
             "url": url,
@@ -429,6 +446,7 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             data['bearer_token'] = bearer_token
         return self.post("/service/{}/delivery-receipt-api/{}".format(service_id, callback_api_id), data)
 
+    @cache.delete('service')
     def create_service_callback_api(self, service_id, url, bearer_token, user_id):
         data = {
             "url": url,

app/notify_client/user_api_client.py

@@ -1,6 +1,6 @@
 from notifications_python_client.errors import HTTPError
 
-from app.notify_client import NotifyAdminAPIClient
+from app.notify_client import NotifyAdminAPIClient, cache
 from app.notify_client.models import (
     User,
     roles,
@@ -147,6 +147,7 @@ class UserApiClient(NotifyAdminAPIClient):
         resp = self.get(endpoint)
         return [User(data) for data in resp['data']]
 
+    @cache.delete('service')
     def add_user_to_service(self, service_id, user_id, permissions):
         # permissions passed in are the combined admin roles, not db permissions
         endpoint = '/service/{}/users/{}'.format(service_id, user_id)