wip

- adds simple explicit instructions for how to install and run
  redis
- gets celery running in docker able to talk to redis running
  natively on your machine as per
  https://docs.docker.com/desktop/mac/networking/#i-want-to-connect-from-a-container-to-a-service-on-the-host

README.md

@@ -61,10 +61,18 @@ export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/
 
 ### Redis
 
-To switch redis on you'll need to install it locally. On a OSX we've used brew for this. To use redis caching you need to switch it on by changing the config for development:
+You can run redis locally using
 
-        REDIS_ENABLED = True
+```
+brew install redis
+redis-server
+```
 
+To get the API to use redis locally you need to change the config for development:
+
+```
+REDIS_ENABLED = True
+```
 
 ##  To run the application
 

app/config.py

@@ -114,7 +114,7 @@ class Config(object):
 
     # URL of redis instance
     REDIS_URL = os.getenv('REDIS_URL')
-    REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
+    REDIS_ENABLED = True
     EXPIRE_CACHE_TEN_MINUTES = 600
     EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
 
@@ -428,7 +428,7 @@ class Development(Config):
     NOTIFY_EMAIL_DOMAIN = "notify.tools"
 
     SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI', 'postgresql://localhost/notification_api')
-    REDIS_URL = 'redis://localhost:6379/0'
+    REDIS_URL = os.getenv('REDIS_URL', 'redis://localhost:6379/0')
 
     ANTIVIRUS_ENABLED = os.getenv('ANTIVIRUS_ENABLED') == '1'
 

app/dao/fact_notification_status_dao.py

@@ -176,6 +176,43 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
     ).all()
 
 
+def fetch_notification_status_for_service_for_today_and_7_previous_days_attempt2(service_id, by_template=False, limit_days=7):
+    start_date = midnight_n_days_ago(limit_days)
+    now = datetime.utcnow()
+    stats_for_7_days = db.session.query(
+        FactNotificationStatus.notification_type.label('notification_type'),
+        FactNotificationStatus.notification_status.label('status'),
+        *([FactNotificationStatus.template_id.label('template_id')] if by_template else []),
+        FactNotificationStatus.notification_count.label('count')
+    ).filter(
+        FactNotificationStatus.service_id == service_id,
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.key_type != KEY_TYPE_TEST
+    )
+
+    all_stats_table = stats_for_7_days.subquery()
+
+    query = db.session.query(
+        *([
+            Template.name.label("template_name"),
+            Template.is_precompiled_letter,
+            all_stats_table.c.template_id
+        ] if by_template else []),
+        all_stats_table.c.notification_type,
+        all_stats_table.c.status,
+        func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
+    )
+
+    if by_template:
+        query = query.filter(all_stats_table.c.template_id == Template.id)
+
+    return query.group_by(
+        *([Template.name, Template.is_precompiled_letter, all_stats_table.c.template_id] if by_template else []),
+        all_stats_table.c.notification_type,
+        all_stats_table.c.status,
+    ).all()
+
+
 def fetch_notification_status_totals_for_all_services(start_date, end_date):
     stats = db.session.query(
         FactNotificationStatus.notification_type.label('notification_type'),

app/notifications/process_notifications.py

@@ -148,6 +148,7 @@ def persist_notification(
     # if simulated create a Notification model to return but do not persist the Notification to the dB
     if not simulated:
         dao_create_notification(notification)
+
         if key_type != KEY_TYPE_TEST and current_app.config['REDIS_ENABLED']:
             cache_key = redis.daily_limit_cache_key(service.id)
             if redis_store.get(cache_key) is None:
@@ -158,12 +159,35 @@ def persist_notification(
                 redis_store.set(cache_key, 1, ex=86400)
             else:
                 redis_store.incr(cache_key)
+
+            # sending may not always be true, check this
+            cache_key, subkey = notification_count_cache_key(
+                service.id, notification_created_at, notification_type, template_id,'sending'
+            )
+            if redis_store.get_all_from_hash(cache_key) is None:
+                # if cache does not exist set the cache to 1 with an expiry of 8 days,
+                # The cache should be set by the time we create the notification
+                # but in case it is this will make sure the expiry is set to 8 days,
+                # where if we let the incr method create the cache it will be set a ttl.
+                # fix this comment as it might be missing a word?
+                redis_store.set_hash_and_expire(cache_key, {subkey: 1}, ex=691200)
+            else:
+                redis_store.increment_hash_value(cache_key, subkey)
+
         current_app.logger.info(
             "{} {} created at {}".format(notification_type, notification_id, notification_created_at)
         )
     return notification
 
 
+def notification_count_cache_key(
+    service_id, created_at_utc, notification_type, template_id, simplified_status
+):
+    # this is not actually turning it in to bst, will need to fix this
+    bst_date = created_at_utc.strftime("%Y-%m-%d")
+    return "service-{}:bst-date-{}:{}".format(str(service_id), bst_date, notification_type), "template-{}:{}".format(template_id, simplified_status) 
+
+
 def send_notification_to_queue_detached(
     key_type, notification_type, notification_id, research_mode, queue=None
 ):

app/template_statistics/rest.py

@@ -1,10 +1,12 @@
 from flask import Blueprint, jsonify, request
 
+from app import redis_store
 from app.dao.fact_notification_status_dao import (
     fetch_notification_status_for_service_for_today_and_7_previous_days,
+    fetch_notification_status_for_service_for_today_and_7_previous_days_attempt2,
 )
 from app.dao.notifications_dao import dao_get_last_date_template_was_used
-from app.dao.templates_dao import dao_get_template_by_id_and_service_id
+from app.dao.templates_dao import dao_get_template_by_id_and_service_id, dao_get_template_by_id
 from app.errors import InvalidRequest, register_errors
 from app.utils import DATETIME_FORMAT
 
@@ -27,11 +29,54 @@ def get_template_statistics_for_service_by_day(service_id):
 
     if whole_days < 0 or whole_days > 7:
         raise InvalidRequest({'whole_days': ['whole_days must be between 0 and 7']}, status_code=400)
-    data = fetch_notification_status_for_service_for_today_and_7_previous_days(
+    data = fetch_notification_status_for_service_for_today_and_7_previous_days_attempt2(
         service_id, by_template=True, limit_days=whole_days
     )
 
-    return jsonify(data=[
+    # service_templates = dao_get_all_templates_for_service(service_id)
+
+    # # for template in service_templates:
+    # redis_data = redis_store.get_all_from_hash(f"service-{service_id}:bst-date-2022-02-14:email:sending")
+    #     for template_id, value in redis_data:
+    #         x = {
+    #             'count': value,
+    #             'template_id': template_id,
+    #             'template_name': row.template_name,
+    #             'template_type': notification_type,
+    #             'is_precompiled_letter': 
+    #             'status': sending
+    #         }
+
+
+
+    parsed_redis_data = []
+    for notification_type in ['email', 'text', 'letter']:
+        redis_data = redis_store.get_all_from_hash(f"service-{service_id}:bst-date-2022-02-14:{notification_type}")
+        print(redis_data)
+        if redis_data:
+            for key, value in redis_data.items():
+                key = key.decode("utf-8")
+                template_id, status = key.split(":")
+                print(template_id)
+                # print(template_id)
+                # print(type(template_id))
+                # print('here')
+                template_id = template_id.replace("template-", "")
+                # print(type(template_id))
+                template = dao_get_template_by_id(template_id)
+                x = {
+                    'count': int(value),
+                    'template_id': template_id,
+                    'template_name': template.name,
+                    'template_type': notification_type,
+                    'is_precompiled_letter': False,
+                    'status': status
+                }
+                print(x)
+                parsed_redis_data.append(x)
+
+
+    result = [
         {
             'count': row.count,
             'template_id': str(row.template_id),
@@ -41,7 +86,17 @@ def get_template_statistics_for_service_by_day(service_id):
             'status': row.status
         }
         for row in data
-    ])
+    ]
+
+    result = result + parsed_redis_data
+
+    # service-id:xxxxxx-xxxxx:bst-date:2022-02-09:email:template-stats
+    #     template-id:yyyyyy-yyyyyy:sending
+    #     template-id:yyyyyy-yyyyyy:delivered
+    #     template-id:zzzzzz-zzzzzz:sending
+
+
+    return jsonify(data=result)
 
 
 @template_statistics.route('/last-used/<uuid:template_id>')

scripts/run_with_docker.sh

@@ -9,11 +9,13 @@ source environment.sh
 AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-"$(aws configure get aws_access_key_id)"}
 AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"$(aws configure get aws_secret_access_key)"}
 : "${SQLALCHEMY_DATABASE_URI:=postgresql://postgres@host.docker.internal/notification_api}"
+REDIS_URL="redis://host.docker.internal:6379"
 
 docker run -it --rm \
   -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
   -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
   -e SQLALCHEMY_DATABASE_URI=$SQLALCHEMY_DATABASE_URI \
+  -e REDIS_URL=$REDIS_URL \
   -v $(pwd):/home/vcap/app \
   ${DOCKER_IMAGE_NAME} \
   ${@}