Merge pull request #1818 from alphagov/redis-templates

add new redis template usage per day key

app/commands.py

@@ -8,8 +8,10 @@ import flask
 from click_datetime import Datetime as click_dt
 from flask import current_app
 from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy import func
+from notifications_utils.statsd_decorators import statsd
 
-from app import db, DATETIME_FORMAT, encryption
+from app import db, DATETIME_FORMAT, encryption, redis_store
 from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
 from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.celery.letters_pdf_tasks import create_letters_pdf
@@ -28,8 +30,11 @@ from app.dao.services_dao import (
 from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
 from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE, Notification
 from app.performance_platform.processing_time import (send_processing_time_for_start_and_end)
-from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
-from notifications_utils.statsd_decorators import statsd
+from app.utils import (
+    cache_key_for_service_template_usage_per_day,
+    get_london_midnight_in_utc,
+    get_midnight_for_day_before,
+)
 
 
 @click.group(name='command', help='Additional commands')
@@ -489,3 +494,37 @@ def migrate_data_to_ft_billing(start_date, end_date):
 
         total_updated += result.rowcount
     print('Total inserted/updated records = {}'.format(total_updated))
+
+
+@notify_command()
+@click.option('-s', '--service_id', required=True, type=click.UUID)
+@click.option('-d', '--day', required=True, type=click_dt(format='%Y-%m-%d'))
+def populate_redis_template_usage(service_id, day):
+    """
+    Recalculate and replace the stats in redis for a day.
+    To be used if redis data is lost for some reason.
+    """
+    assert current_app.config['REDIS_ENABLED']
+    usage = {
+        str(row.template_id): row.count
+        for row in db.session.query(
+            Notification.template_id,
+            func.count().label('count')
+        ).filter(
+            Notification.service_id == service_id,
+        ).group_by(
+            Notification.template_id
+        )
+    }
+    current_app.logger.info('Populating usage dict for service {} day {}: {}'.format(
+        service_id,
+        day,
+        usage.items())
+    )
+    if usage:
+        key = cache_key_for_service_template_usage_per_day(service_id, day)
+        redis_store.set_hash_and_expire(
+            key,
+            usage,
+            current_app.config['EXPIRE_CACHE_EIGHT_DAYS']
+        )

app/config.py

@@ -95,7 +95,8 @@ class Config(object):
     # URL of redis instance
     REDIS_URL = os.getenv('REDIS_URL')
     REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
-    EXPIRE_CACHE_IN_SECONDS = 600
+    EXPIRE_CACHE_TEN_MINUTES = 600
+    EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
 
     # Performance platform
     PERFORMANCE_PLATFORM_ENABLED = False

app/notifications/process_notifications.py

@@ -29,7 +29,13 @@ from app.dao.notifications_dao import (
 )
 
 from app.v2.errors import BadRequestError
-from app.utils import get_template_instance, cache_key_for_service_template_counter, convert_bst_to_utc
+from app.utils import (
+    cache_key_for_service_template_counter,
+    cache_key_for_service_template_usage_per_day,
+    convert_bst_to_utc,
+    convert_utc_to_bst,
+    get_template_instance,
+)
 
 
 def create_content_for_notification(template, personalisation):
@@ -108,12 +114,24 @@ def persist_notification(
                 redis_store.incr(redis.daily_limit_cache_key(service.id))
             if redis_store.get_all_from_hash(cache_key_for_service_template_counter(service.id)):
                 redis_store.increment_hash_value(cache_key_for_service_template_counter(service.id), template_id)
+
+            increment_template_usage_cache(service.id, template_id, notification_created_at)
+
         current_app.logger.info(
             "{} {} created at {}".format(notification_type, notification_id, notification_created_at)
         )
     return notification
 
 
+def increment_template_usage_cache(service_id, template_id, created_at):
+    key = cache_key_for_service_template_usage_per_day(service_id, convert_utc_to_bst(created_at))
+    redis_store.increment_hash_value(key, template_id)
+    # set key to expire in eight days - we don't know if we've just created the key or not, so must assume that we
+    # have and reset the expiry. Eight days is longer than any notification is in the notifications table, so we'll
+    # always capture the full week's numbers
+    redis_store.expire(key, current_app.config['EXPIRE_CACHE_EIGHT_DAYS'])
+
+
 def send_notification_to_queue(notification, research_mode, queue=None):
     if research_mode or notification.key_type == KEY_TYPE_TEST:
         queue = QueueNames.RESEARCH_MODE

app/template_statistics/rest.py

@@ -79,7 +79,22 @@ def get_template_statistics_for_7_days(limit_days, service_id):
         if cache_values:
             redis_store.set_hash_and_expire(cache_key,
                                             cache_values,
-                                            current_app.config.get('EXPIRE_CACHE_IN_SECONDS', 600))
+                                            current_app.config['EXPIRE_CACHE_TEN_MINUTES'])
     else:
         stats = dao_get_templates_for_cache(template_stats_by_id.items())
     return stats
+
+    # TODO: can only switch to this code when redis has been populated (either through time passing or a manual step)
+    # from collections import Counter
+    # from notifications_utils.redis_client import RedisException
+    # template_stats_by_id = Counter()
+    # for day in last_7_days:
+    #     # "<SERVICE_ID>-template-usage-{YYYY-MM-DD}"
+    #     key = cache_key_for_service_templates_used_per_day(service_id, limit_days)
+    #     try:
+    #         template_stats_by_id += Counter(redis_store.get_all_from_hash(key, raise_exception=True))
+    #     except RedisException:
+    #         # TODO: ????
+    #
+    # # TODO: streamline db query and avoid weird unions if possible.
+    # return dao_get_templates_for_cache(template_stats_by_id.items())

app/utils.py

@@ -79,6 +79,10 @@ def cache_key_for_service_template_counter(service_id, limit_days=7):
     return "{}-template-counter-limit-{}-days".format(service_id, limit_days)
 
 
+def cache_key_for_service_template_usage_per_day(service_id, datetime):
+    return "service-{}-template-usage-{}".format(service_id, datetime.date().isoformat())
+
+
 def get_public_notify_type_text(notify_type, plural=False):
     from app.models import SMS_TYPE
     notify_type_text = notify_type