mirror of
https://github.com/GSA/notifications-api.git
synced 2025-12-25 18:02:40 -05:00
8e73961f65
We've run into issues with redis expiring keys while we try and write to them - short lived redis TTLs aren't really sustainable for keys where we mutate the state. Template usage is a hash contained in redis where we increment a count keyed by template_id each time a message is sent for that template. But if the key expires, hincrby (redis command for incrementing a value in a hash) will re-create an empty hash. This is no good, as we need the hash to be populated with the last seven days worth of data, which we then increment further. We can't tell whether the hincrby created the key, so a different approach entirely was needed: * New redis key: <service_id>-template-usage-<YYYY-MM-DD>. Note: This YYYY-MM-DD is BTC time so it lines up nicely with ft_billing table * Incremented to from process_notification - if it doesn't exist yet, it'll be created then. * Expiry set to 8 days every time it's incremented to. Then, at read time, we'll just read the last eight days of keys from Redis, and sum them up. This works because we're only ever incrementing from that one place - never setting wholesale, never recreating the data from scratch. So we know that if the data is in redis, then it is good and accurate data. One thing we *don't* know and *cannot* reason about is what no key in redis means. It could be either of: * This is the first message that the service has sent today. * The key was deleted from redis for some reason. Since we set the TTL to so long, we'll never be writing to a key that previously expired. But if there is a redis (or operator) error and the key is deleted, then we'll have bad data - after any data loss we'll have to rebuild the data.
101 lines
3.7 KiB
Python
101 lines
3.7 KiB
Python
from flask import (
|
|
Blueprint,
|
|
jsonify,
|
|
request,
|
|
current_app)
|
|
|
|
from app import redis_store
|
|
from app.dao.notifications_dao import (
|
|
dao_get_template_usage,
|
|
dao_get_last_template_usage
|
|
)
|
|
from app.dao.templates_dao import (
|
|
dao_get_templates_for_cache,
|
|
dao_get_template_by_id_and_service_id
|
|
)
|
|
|
|
from app.schemas import notification_with_template_schema
|
|
from app.utils import cache_key_for_service_template_counter
|
|
from app.errors import register_errors, InvalidRequest
|
|
|
|
template_statistics = Blueprint('template-statistics',
|
|
__name__,
|
|
url_prefix='/service/<service_id>/template-statistics')
|
|
|
|
register_errors(template_statistics)
|
|
|
|
|
|
@template_statistics.route('')
|
|
def get_template_statistics_for_service_by_day(service_id):
|
|
if request.args.get('limit_days'):
|
|
try:
|
|
limit_days = int(request.args['limit_days'])
|
|
except ValueError as e:
|
|
error = '{} is not an integer'.format(request.args['limit_days'])
|
|
message = {'limit_days': [error]}
|
|
raise InvalidRequest(message, status_code=400)
|
|
else:
|
|
limit_days = None
|
|
|
|
if limit_days == 7:
|
|
stats = get_template_statistics_for_7_days(limit_days, service_id)
|
|
else:
|
|
stats = dao_get_template_usage(service_id, limit_days=limit_days)
|
|
|
|
def serialize(data):
|
|
return {
|
|
'count': data.count,
|
|
'template_id': str(data.template_id),
|
|
'template_name': data.name,
|
|
'template_type': data.template_type,
|
|
'is_precompiled_letter': data.is_precompiled_letter
|
|
}
|
|
|
|
return jsonify(data=[serialize(row) for row in stats])
|
|
|
|
|
|
@template_statistics.route('/<template_id>')
|
|
def get_template_statistics_for_template_id(service_id, template_id):
|
|
template = dao_get_template_by_id_and_service_id(template_id, service_id)
|
|
if not template:
|
|
message = 'No template found for id {}'.format(template_id)
|
|
errors = {'template_id': [message]}
|
|
raise InvalidRequest(errors, status_code=404)
|
|
|
|
data = None
|
|
notification = dao_get_last_template_usage(template_id, template.template_type)
|
|
if notification:
|
|
data = notification_with_template_schema.dump(notification).data
|
|
|
|
return jsonify(data=data)
|
|
|
|
|
|
def get_template_statistics_for_7_days(limit_days, service_id):
|
|
cache_key = cache_key_for_service_template_counter(service_id)
|
|
template_stats_by_id = redis_store.get_all_from_hash(cache_key)
|
|
if not template_stats_by_id:
|
|
stats = dao_get_template_usage(service_id, limit_days=limit_days)
|
|
cache_values = dict([(x.template_id, x.count) for x in stats])
|
|
if cache_values:
|
|
redis_store.set_hash_and_expire(cache_key,
|
|
cache_values,
|
|
current_app.config['EXPIRE_CACHE_TEN_MINUTES'])
|
|
else:
|
|
stats = dao_get_templates_for_cache(template_stats_by_id.items())
|
|
return stats
|
|
|
|
# TODO: can only switch to this code when redis has been populated (either through time passing or a manual step)
|
|
# from collections import Counter
|
|
# from notifications_utils.redis_client import RedisException
|
|
# template_stats_by_id = Counter()
|
|
# for day in last_7_days:
|
|
# # "<SERVICE_ID>-template-usage-{YYYY-MM-DD}"
|
|
# key = cache_key_for_service_templates_used_per_day(service_id, limit_days)
|
|
# try:
|
|
# template_stats_by_id += Counter(redis_store.get_all_from_hash(key, raise_exception=True))
|
|
# except RedisException:
|
|
# # TODO: ????
|
|
#
|
|
# # TODO: streamline db query and avoid weird unions if possible.
|
|
# return dao_get_templates_for_cache(template_stats_by_id.items())
|