add metrics for redis timings

app/notifications/process_notifications.py

@@ -34,6 +34,15 @@ from app.dao.notifications_dao import (
 from app.v2.errors import BadRequestError
 
 
+from gds_metrics import Histogram
+
+
+REDIS_GET_AND_INCR_DAILY_LIMIT_DURATION_SECONDS = Histogram(
+    'redis_get_and_incr_daily_limit_duration_seconds',
+    'Time taken to get and possibly incremement the daily limit cache key',
+)
+
+
 def create_content_for_notification(template, personalisation):
     template_object = template._as_utils_template_with_personalisation(personalisation)
     check_placeholders(template_object)
@@ -115,6 +124,7 @@ def persist_notification(
     if not simulated:
         dao_create_notification(notification)
         if key_type != KEY_TYPE_TEST:
+            with REDIS_GET_AND_INCR_DAILY_LIMIT_DURATION_SECONDS.time():
                 if redis_store.get(redis.daily_limit_cache_key(service.id)):
                     redis_store.incr(redis.daily_limit_cache_key(service.id))
 

app/notifications/validators.py

@@ -22,12 +22,21 @@ from app.utils import get_public_notify_type_text
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_letter_contact_dao import dao_get_letter_contact_by_id
 
+from gds_metrics.metrics import Histogram
+
+
+REDIS_EXCEEDED_RATE_LIMIT_DURATION_SECONDS = Histogram(
+    'redis_exceeded_rate_limit_duration_seconds',
+    'Time taken to check rate limit',
+)
+
 
 def check_service_over_api_rate_limit(service, api_key):
     if current_app.config['API_RATE_LIMIT_ENABLED'] and current_app.config['REDIS_ENABLED']:
         cache_key = rate_limit_cache_key(service.id, api_key.key_type)
         rate_limit = service.rate_limit
         interval = 60
+        with REDIS_EXCEEDED_RATE_LIMIT_DURATION_SECONDS.time():
             if redis_store.exceeded_rate_limit(cache_key, rate_limit, interval):
                 current_app.logger.info("service {} has been rate limited for throughput".format(service.id))
                 raise RateLimitError(rate_limit, interval, api_key.key_type)

gunicorn_config.py

@@ -7,7 +7,7 @@ from gds_metrics.gunicorn import child_exit  # noqa
 
 workers = 4
 worker_class = "eventlet"
-worker_connections = 256
+worker_connections = 30
 errorlog = "/home/vcap/logs/gunicorn_error.log"
 bind = "0.0.0.0:{}".format(os.getenv("PORT"))
 statsd_host = "{}:8125".format(os.getenv("STATSD_HOST"))