Merge pull request #735 from alphagov/caching-with-redis

Caching with redis

app/__init__.py

@@ -1,19 +1,21 @@
-import uuid
 import os
+import uuid
 
-from flask import request, url_for, g, jsonify
 from flask import Flask, _request_ctx_stack
+from flask import request, url_for, g, jsonify
 from flask.ext.sqlalchemy import SQLAlchemy
 from flask_marshmallow import Marshmallow
 from monotonic import monotonic
-from werkzeug.local import LocalProxy
 from notifications_utils import logging
+from werkzeug.local import LocalProxy
+
 from app.celery.celery import NotifyCelery
 from app.clients import Clients
-from app.clients.sms.mmg import MMGClient
+from app.clients.email.aws_ses import AwsSesClient
+from app.clients.redis.redis_client import RedisClient
 from app.clients.sms.firetext import FiretextClient
 from app.clients.sms.loadtesting import LoadtestingClient
-from app.clients.email.aws_ses import AwsSesClient
+from app.clients.sms.mmg import MMGClient
 from app.clients.statsd.statsd_client import StatsdClient
 from app.encryption import Encryption
 
@@ -30,6 +32,7 @@ mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
 encryption = Encryption()
 statsd_client = StatsdClient()
+redis_store = RedisClient()
 
 clients = Clients()
 
@@ -55,6 +58,7 @@ def create_app(app_name=None):
     aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
     notify_celery.init_app(application)
     encryption.init_app(application)
+    redis_store.init_app(application)
     clients.init_app(sms_clients=[firetext_client, mmg_client, loadtest_client], email_clients=[aws_ses_client])
 
     register_blueprint(application)

app/celery/tasks.py

@@ -27,6 +27,8 @@ from app.models import (
 from app.notifications.process_notifications import persist_notification
 from app.service.utils import service_allowed_to_send_to
 from app.statsd_decorators import statsd
+from app import redis_store
+from app.clients.redis import daily_limit_cache_key
 
 
 @notify_celery.task(name="process-job")
@@ -152,14 +154,16 @@ def send_sms(self,
 
     except SQLAlchemyError as e:
         current_app.logger.exception(
-            "RETRY: send_sms notification for job {} row number {}".format(notification.get('job', None),
-                                                                           notification.get('row_number', None)), e)
+            "RETRY: send_sms notification for job {} row number {}".format(
+                notification.get('job', None),
+                notification.get('row_number', None)), e)
         try:
             raise self.retry(queue="retry", exc=e)
         except self.MaxRetriesExceededError:
             current_app.logger.exception(
-                "RETRY FAILED: task send_sms failed for notification".format(notification.get('job', None),
-                                                                             notification.get('row_number', None)), e)
+                "RETRY FAILED: task send_sms failed for notification".format(
+                    notification.get('job', None),
+                    notification.get('row_number', None)), e)
 
 
 @notify_celery.task(bind=True, name="send-email", max_retries=5, default_retry_delay=300)
@@ -207,5 +211,6 @@ def send_email(self, service_id,
             raise self.retry(queue="retry", exc=e)
         except self.MaxRetriesExceededError:
             current_app.logger.error(
-                "RETRY FAILED: task send_email failed for notification".format(notification.get('job', None),
-                                                                               notification.get('row_number', None)), e)
+                "RETRY FAILED: task send_email failed for notification".format(
+                    notification.get('job', None),
+                    notification.get('row_number', None)), e)

app/clients/redis/__init__.py

@@ -0,0 +1,5 @@
+from datetime import datetime
+
+
+def daily_limit_cache_key(service_id):
+    return "{}-{}-{}".format(str(service_id), datetime.utcnow().strftime("%Y-%m-%d"), "count")

app/clients/redis/redis_client.py

@@ -0,0 +1,40 @@
+from flask.ext.redis import FlaskRedis
+from flask import current_app
+
+
+class RedisClient:
+    redis_store = FlaskRedis()
+    active = False
+
+    def init_app(self, app):
+        self.active = app.config.get('REDIS_ENABLED')
+        if self.active:
+            self.redis_store.init_app(app)
+
+    def set(self, key, value, ex=None, px=None, nx=False, xx=False, raise_exception=False):
+        if self.active:
+            try:
+                self.redis_store.set(key, value, ex, px, nx, xx)
+            except Exception as e:
+                current_app.logger.exception(e)
+                if raise_exception:
+                    raise e
+
+    def incr(self, key, raise_exception=False):
+        if self.active:
+            try:
+                return self.redis_store.incr(key)
+            except Exception as e:
+                current_app.logger.exception(e)
+                if raise_exception:
+                    raise e
+
+    def get(self, key, raise_exception=False):
+        if self.active:
+            try:
+                return self.redis_store.get(key)
+            except Exception as e:
+                current_app.logger.exception(e)
+                if raise_exception:
+                    raise e
+        return None

app/errors.py

@@ -6,6 +6,7 @@ from sqlalchemy.exc import SQLAlchemyError, DataError
 from sqlalchemy.orm.exc import NoResultFound
 from marshmallow import ValidationError
 from app.authentication.auth import AuthError
+from app.notifications import SendNotificationToQueueError
 
 
 class InvalidRequest(Exception):
@@ -83,6 +84,11 @@ def register_errors(blueprint):
         current_app.logger.exception(e)
         return jsonify(result='error', message="No result found"), 404
 
+    @blueprint.errorhandler(SendNotificationToQueueError)
+    def failed_to_create_notification(e):
+        current_app.logger.exception(e)
+        return jsonify(result='error', message=e.message), 500
+
     @blueprint.errorhandler(SQLAlchemyError)
     def db_error(e):
         current_app.logger.exception(e)

app/notifications/__init__.py

@@ -0,0 +1,7 @@
+
+
+class SendNotificationToQueueError(Exception):
+    status_code = 500
+
+    def __init__(self):
+        self.message = "Failed to create the notification"

app/notifications/process_notifications.py

@@ -4,12 +4,13 @@ from flask import current_app
 from notifications_utils.renderers import PassThrough
 from notifications_utils.template import Template
 
-from app import DATETIME_FORMAT
+from app import DATETIME_FORMAT, redis_store
 from app.celery import provider_tasks
+from app.clients import redis
 from app.dao.notifications_dao import dao_create_notification, dao_delete_notifications_and_history_by_id
 from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE
 from app.notifications.validators import check_sms_content_char_count
-from app.v2.errors import BadRequestError
+from app.v2.errors import BadRequestError, SendNotificationToQueueError
 
 
 def create_content_for_notification(template, personalisation):
@@ -63,6 +64,7 @@ def persist_notification(template_id,
         client_reference=reference
     )
     dao_create_notification(notification)
+    redis_store.incr(redis.daily_limit_cache_key(service_id))
     return notification
 
 
@@ -79,10 +81,10 @@ def send_notification_to_queue(notification, research_mode):
                 [str(notification.id)],
                 queue='send-email' if not research_mode else 'research-mode'
             )
-    except Exception:
-        current_app.logger.exception("Failed to send to SQS exception")
+    except Exception as e:
+        current_app.logger.exception(e)
         dao_delete_notifications_and_history_by_id(notification.id)
-        raise
+        raise SendNotificationToQueueError()
 
     current_app.logger.info(
         "{} {} created at {}".format(notification.notification_type, notification.id, notification.created_at)

app/notifications/rest.py

@@ -36,6 +36,8 @@ from app.schemas import (
 )
 from app.service.utils import service_allowed_to_send_to
 from app.utils import pagination_links
+from app import redis_store
+from app.clients import redis
 
 notifications = Blueprint('notifications', __name__)
 
@@ -44,6 +46,7 @@ from app.errors import (
     InvalidRequest
 )
 
+
 register_errors(notifications)
 
 
@@ -204,6 +207,7 @@ def get_notification_statistics_for_day():
 
 @notifications.route('/notifications/<string:notification_type>', methods=['POST'])
 def send_notification(notification_type):
+
     if notification_type not in ['sms', 'email']:
         assert False
 
@@ -242,7 +246,6 @@ def send_notification(notification_type):
 
     notification_id = create_uuid() if saved_notification is None else saved_notification.id
     notification.update({"template_version": template.version})
-
     return jsonify(
         data=get_notification_return_data(
             notification_id,

app/notifications/validators.py

@@ -4,13 +4,18 @@ from app.dao import services_dao
 from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM
 from app.service.utils import service_allowed_to_send_to
 from app.v2.errors import TooManyRequestsError, BadRequestError
+from app import redis_store
+from app.clients import redis
 
 
 def check_service_message_limit(key_type, service):
-    if all((key_type != KEY_TYPE_TEST,
-            service.restricted)):
-        service_stats = services_dao.fetch_todays_total_message_count(service.id)
-        if service_stats >= service.message_limit:
+    if key_type != KEY_TYPE_TEST:
+        cache_key = redis.daily_limit_cache_key(service.id)
+        service_stats = redis_store.get(cache_key)
+        if not service_stats:
+            service_stats = services_dao.fetch_todays_total_message_count(service.id)
+            redis_store.set(cache_key, service_stats, ex=3600)
+        if int(service_stats) >= service.message_limit:
             raise TooManyRequestsError(service.message_limit)
 
 

app/v2/errors.py

@@ -5,6 +5,7 @@ from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
 from app.authentication.auth import AuthError
 from app.errors import InvalidRequest
+from app.notifications import SendNotificationToQueueError
 
 
 class TooManyRequestsError(InvalidRequest):
@@ -47,6 +48,13 @@ def register_errors(blueprint):
     def auth_error(error):
         return jsonify(error.to_dict_v2()), error.code
 
+    @blueprint.errorhandler(SendNotificationToQueueError)
+    def failed_to_create_notification(error):
+        current_app.logger.exception(error)
+        return jsonify(
+            status_code=500,
+            errors=[{"error": error.__class__.__name__, "message": error.message}]), 500
+
     @blueprint.errorhandler(Exception)
     def internal_server_error(error):
         current_app.logger.exception(error)