add command to backfill template usage

The command takes a service id and a day, grabs the historical data for
that day (potentially out of notification_history), and pops it in
redis (for eight days, same as if it were written to manually).

also, prefix template usage key with "service" to make clear that it's
a service id, and not an individual template id.

app/commands.py

@@ -8,8 +8,10 @@ import flask
 from click_datetime import Datetime as click_dt
 from flask import current_app
 from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy import func
+from notifications_utils.statsd_decorators import statsd
 
-from app import db, DATETIME_FORMAT, encryption
+from app import db, DATETIME_FORMAT, encryption, redis_store
 from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
 from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.celery.letters_pdf_tasks import create_letters_pdf
@@ -28,8 +30,11 @@ from app.dao.services_dao import (
 from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
 from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE, Notification
 from app.performance_platform.processing_time import (send_processing_time_for_start_and_end)
-from app.utils import get_midnight_for_day_before, get_london_midnight_in_utc
-from notifications_utils.statsd_decorators import statsd
+from app.utils import (
+    cache_key_for_service_template_usage_per_day,
+    get_london_midnight_in_utc,
+    get_midnight_for_day_before,
+)
 
 
 @click.group(name='command', help='Additional commands')
@@ -489,3 +494,37 @@ def migrate_data_to_ft_billing(start_date, end_date):
 
         total_updated += result.rowcount
     print('Total inserted/updated records = {}'.format(total_updated))
+
+
+@notify_command()
+@click.option('-s', '--service_id', required=True, type=click.UUID)
+@click.option('-d', '--day', required=True, type=click_dt(format='%Y-%m-%d'))
+def populate_redis_template_usage(service_id, day):
+    """
+    Recalculate and replace the stats in redis for a day.
+    To be used if redis data is lost for some reason.
+    """
+    assert current_app.config['REDIS_ENABLED']
+    usage = {
+        str(row.template_id): row.count
+        for row in db.session.query(
+            Notification.template_id,
+            func.count().label('count')
+        ).filter(
+            Notification.service_id == service_id,
+        ).group_by(
+            Notification.template_id
+        )
+    }
+    current_app.logger.info('Populating usage dict for service {} day {}: {}'.format(
+        service_id,
+        day,
+        usage.items())
+    )
+    if usage:
+        key = cache_key_for_service_template_usage_per_day(service_id, day)
+        redis_store.set_hash_and_expire(
+            key,
+            usage,
+            current_app.config['EXPIRE_CACHE_EIGHT_DAYS']
+        )

app/utils.py

@@ -80,7 +80,7 @@ def cache_key_for_service_template_counter(service_id, limit_days=7):
 
 
 def cache_key_for_service_template_usage_per_day(service_id, datetime):
-    return "{}-template-usage-{}".format(service_id, datetime.date().isoformat())
+    return "service-{}-template-usage-{}".format(service_id, datetime.date().isoformat())
 
 
 def get_public_notify_type_text(notify_type, plural=False):

tests/app/notifications/test_process_notification.py

@@ -226,7 +226,7 @@ def test_persist_notification_doesnt_touch_cache_for_old_keys_that_dont_exist(sa
     )
     mock_incr.assert_not_called()
     mock_incr_hash_value.assert_called_once_with(
-        str(sample_template.service_id) + "-template-usage-2016-01-01",
+        "service-{}-template-usage-2016-01-01".format(sample_template.service_id),
         sample_template.id
     )
 
@@ -252,8 +252,8 @@ def test_persist_notification_increments_cache_if_key_exists(sample_template, sa
 
     mock_incr.assert_called_once_with(str(sample_template.service_id) + "-2016-01-01-count", )
     assert mock_incr_hash_value.mock_calls == [
-        call(str(sample_template.service_id) + "-template-counter-limit-7-days", sample_template.id),
-        call(str(sample_template.service_id) + "-template-usage-2016-01-01", sample_template.id),
+        call("{}-template-counter-limit-7-days".format(sample_template.service_id), sample_template.id),
+        call("service-{}-template-usage-2016-01-01".format(sample_template.service_id), sample_template.id),
     ]
 
 
@@ -493,10 +493,10 @@ def test_persist_notification_increments_and_expires_redis_template_usage(
             key_type=sample_api_key.key_type,
         )
     mock_incr_hash_value.assert_called_once_with(
-        '{}-template-usage-{}'.format(str(sample_template.service_id), day_in_key),
+        'service-{}-template-usage-{}'.format(str(sample_template.service_id), day_in_key),
         sample_template.id
     )
     mock_expire.assert_called_once_with(
-        '{}-template-usage-{}'.format(str(sample_template.service_id), day_in_key),
+        'service-{}-template-usage-{}'.format(str(sample_template.service_id), day_in_key),
         current_app.config['EXPIRE_CACHE_EIGHT_DAYS']
     )