darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-02 17:31:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2303 from alphagov/ft-status-template-statistics

Browse Source 
Change template statistics endpoint to use fact_notification_status_dao
This commit is contained in:
Alexey Bezhan
2019-01-17 15:04:20 +00:00
committed by GitHub
parent 276a9a3828 ac3832a918
commit 90dc69b6bc
14 changed files with 112 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
app/commands.py
View File

@@ -1,4 +1,3 @@
import sys
import functools
import uuid
from datetime import datetime, timedelta
@@ -9,10 +8,9 @@ import flask
from click_datetime import Datetime as click_dt
from flask import current_app, json
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy import func
from notifications_utils.statsd_decorators import statsd
from app import db, DATETIME_FORMAT, encryption, redis_store
from app import db, DATETIME_FORMAT, encryption
from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
from app.celery.service_callback_tasks import send_delivery_status_to_service
from app.celery.letters_pdf_tasks import create_letters_pdf
@@ -34,11 +32,7 @@ from app.dao.services_dao import (
from app.dao.users_dao import delete_model_user, delete_user_verify_codes
from app.models import PROVIDERS, User, Notification
from app.performance_platform.processing_time import send_processing_time_for_start_and_end
from app.utils import (
cache_key_for_service_template_usage_per_day,
get_london_midnight_in_utc,
get_midnight_for_day_before,
)
from app.utils import get_london_midnight_in_utc, get_midnight_for_day_before
@click.group(name='command', help='Additional commands')
@@ -430,50 +424,6 @@ def migrate_data_to_ft_billing(start_date, end_date):
current_app.logger.info('Total inserted/updated records = {}'.format(total_updated))
@notify_command()
@click.option('-s', '--service_id', required=True, type=click.UUID)
@click.option('-d', '--day', required=True, type=click_dt(format='%Y-%m-%d'))
def populate_redis_template_usage(service_id, day):
"""
Recalculate and replace the stats in redis for a day.
To be used if redis data is lost for some reason.
"""
if not current_app.config['REDIS_ENABLED']:
current_app.logger.error('Cannot populate redis template usage - redis not enabled')
sys.exit(1)
# the day variable is set by click to be midnight of that day
start_time = get_london_midnight_in_utc(day)
end_time = get_london_midnight_in_utc(day + timedelta(days=1))
usage = {
str(row.template_id): row.count
for row in db.session.query(
Notification.template_id,
func.count().label('count')
).filter(
Notification.service_id == service_id,
Notification.created_at >= start_time,
Notification.created_at < end_time
).group_by(
Notification.template_id
)
}
current_app.logger.info('Populating usage dict for service {} day {}: {}'.format(
service_id,
day,
usage.items())
)
if usage:
key = cache_key_for_service_template_usage_per_day(service_id, day)
redis_store.set_hash_and_expire(
key,
usage,
current_app.config['EXPIRE_CACHE_EIGHT_DAYS'],
raise_exception=True
)
@notify_command(name='rebuild-ft-billing-for-day')
@click.option('-s', '--service_id', required=False, type=click.UUID)
@click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True,

22
app/dao/fact_notification_status_dao.py
View File

@@ -107,12 +107,13 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id):
).all()
def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, limit_days=7):
def fetch_notification_status_for_service_for_today_and_7_previous_days(service_id, by_template=False, limit_days=7):
start_date = midnight_n_days_ago(limit_days)
now = datetime.utcnow()
stats_for_7_days = db.session.query(
FactNotificationStatus.notification_type.label('notification_type'),
FactNotificationStatus.notification_status.label('status'),
*([FactNotificationStatus.template_id.label('template_id')] if by_template else []),
FactNotificationStatus.notification_count.label('count')
).filter(
FactNotificationStatus.service_id == service_id,
@@ -123,6 +124,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
stats_for_today = db.session.query(
Notification.notification_type.cast(db.Text),
Notification.status,
*([Notification.template_id] if by_template else []),
func.count().label('count')
).filter(
Notification.created_at >= get_london_midnight_in_utc(now),
@@ -130,14 +132,28 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_
Notification.key_type != KEY_TYPE_TEST
).group_by(
Notification.notification_type,
*([Notification.template_id] if by_template else []),
Notification.status
)
all_stats_table = stats_for_7_days.union_all(stats_for_today).subquery()
return db.session.query(
query = db.session.query(
*([
Template.name.label("template_name"),
Template.is_precompiled_letter,
all_stats_table.c.template_id
] if by_template else []),
all_stats_table.c.notification_type,
all_stats_table.c.status,
func.cast(func.sum(all_stats_table.c.count), Integer).label('count'),
).group_by(
)
if by_template:
query = query.filter(all_stats_table.c.template_id == Template.id)
return query.group_by(
*([Template.name, Template.is_precompiled_letter, all_stats_table.c.template_id] if by_template else []),
all_stats_table.c.notification_type,
all_stats_table.c.status,
).all()

34
app/dao/notifications_dao.py
View File

@@ -30,7 +30,6 @@ from app.models import (
Notification,
NotificationHistory,
ScheduledNotification,
Template,
KEY_TYPE_TEST,
LETTER_TYPE,
NOTIFICATION_CREATED,
@@ -51,39 +50,6 @@ from app.utils import get_london_midnight_in_utc
from app.utils import midnight_n_days_ago, escape_special_characters
@statsd(namespace="dao")
def dao_get_template_usage(service_id, day):
start = get_london_midnight_in_utc(day)
end = get_london_midnight_in_utc(day + timedelta(days=1))
notifications_aggregate_query = db.session.query(
func.count().label('count'),
Notification.template_id
).filter(
Notification.created_at >= start,
Notification.created_at < end,
Notification.service_id == service_id,
Notification.key_type != KEY_TYPE_TEST,
).group_by(
Notification.template_id
).subquery()
query = db.session.query(
Template.id,
Template.name,
Template.template_type,
Template.is_precompiled_letter,
func.coalesce(notifications_aggregate_query.c.count, 0).label('count')
).outerjoin(
notifications_aggregate_query,
notifications_aggregate_query.c.template_id == Template.id
).filter(
Template.service_id == service_id
).order_by(Template.name)
return query.all()
@statsd(namespace="dao")
def dao_get_last_template_usage(template_id, template_type, service_id):
# By adding the service_id to the filter the performance of the query is greatly improved.

15
app/dao/templates_dao.py
View File

@@ -129,18 +129,3 @@ def dao_get_template_versions(service_id, template_id):
).order_by(
desc(TemplateHistory.version)
).all()
def dao_get_multiple_template_details(template_ids):
query = db.session.query(
Template.id,
Template.template_type,
Template.name,
Template.is_precompiled_letter
).filter(
Template.id.in_(template_ids)
).order_by(
Template.name
)
return query.all()

21
app/notifications/process_notifications.py
View File

@@ -9,7 +9,7 @@ from notifications_utils.recipients import (
validate_and_format_phone_number,
format_email_address
)
from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
from notifications_utils.timezones import convert_bst_to_utc
from app import redis_store
from app.celery import provider_tasks
@@ -33,11 +33,7 @@ from app.dao.notifications_dao import (
)
from app.v2.errors import BadRequestError
from app.utils import (
cache_key_for_service_template_counter,
cache_key_for_service_template_usage_per_day,
get_template_instance,
)
from app.utils import get_template_instance
def create_content_for_notification(template, personalisation):
@@ -126,10 +122,6 @@ def persist_notification(
if key_type != KEY_TYPE_TEST:
if redis_store.get(redis.daily_limit_cache_key(service.id)):
redis_store.incr(redis.daily_limit_cache_key(service.id))
if redis_store.get_all_from_hash(cache_key_for_service_template_counter(service.id)):
redis_store.increment_hash_value(cache_key_for_service_template_counter(service.id), template_id)
increment_template_usage_cache(service.id, template_id, notification_created_at)
current_app.logger.info(
"{} {} created at {}".format(notification_type, notification_id, notification_created_at)
@@ -137,15 +129,6 @@ def persist_notification(
return notification
def increment_template_usage_cache(service_id, template_id, created_at):
key = cache_key_for_service_template_usage_per_day(service_id, convert_utc_to_bst(created_at))
redis_store.increment_hash_value(key, template_id)
# set key to expire in eight days - we don't know if we've just created the key or not, so must assume that we
# have and reset the expiry. Eight days is longer than any notification is in the notifications table, so we'll
# always capture the full week's numbers
redis_store.expire(key, current_app.config['EXPIRE_CACHE_EIGHT_DAYS'])
def send_notification_to_queue(notification, research_mode, queue=None):
if research_mode or notification.key_type == KEY_TYPE_TEST:
queue = QueueNames.RESEARCH_MODE

84
app/template_statistics/rest.py
View File

@@ -1,24 +1,10 @@
from flask import (
Blueprint,
jsonify,
request,
current_app
)
from app import redis_store
from app.dao.notifications_dao import (
dao_get_template_usage,
dao_get_last_template_usage
)
from app.dao.templates_dao import (
dao_get_multiple_template_details,
dao_get_template_by_id_and_service_id
)
from flask import Blueprint, jsonify, request
from app.dao.notifications_dao import dao_get_last_template_usage
from app.dao.templates_dao import dao_get_template_by_id_and_service_id
from app.dao.fact_notification_status_dao import fetch_notification_status_for_service_for_today_and_7_previous_days
from app.schemas import notification_with_template_schema
from app.utils import cache_key_for_service_template_usage_per_day, last_n_days
from app.errors import register_errors, InvalidRequest
from collections import Counter
template_statistics = Blueprint('template_statistics',
__name__,
@@ -39,8 +25,21 @@ def get_template_statistics_for_service_by_day(service_id):
if whole_days < 0 or whole_days > 7:
raise InvalidRequest({'whole_days': ['whole_days must be between 0 and 7']}, status_code=400)
data = fetch_notification_status_for_service_for_today_and_7_previous_days(
service_id, by_template=True, limit_days=whole_days
)
return jsonify(data=_get_template_statistics_for_last_n_days(service_id, whole_days))
return jsonify(data=[
{
'count': row.count,
'template_id': str(row.template_id),
'template_name': row.template_name,
'template_type': row.notification_type,
'is_precompiled_letter': row.is_precompiled_letter,
'status': row.status
}
for row in data
])
@template_statistics.route('/<template_id>')
@@ -53,50 +52,3 @@ def get_template_statistics_for_template_id(service_id, template_id):
data = notification_with_template_schema.dump(notification).data
return jsonify(data=data)
def _get_template_statistics_for_last_n_days(service_id, whole_days):
template_stats_by_id = Counter()
# 0 whole_days = last 1 days (ie since midnight today) = today.
# 7 whole days = last 8 days (ie since midnight this day last week) = a week and a bit
for day in last_n_days(whole_days + 1):
# "{SERVICE_ID}-template-usage-{YYYY-MM-DD}"
key = cache_key_for_service_template_usage_per_day(service_id, day)
stats = redis_store.get_all_from_hash(key)
if stats:
stats = {
k.decode('utf-8'): int(v) for k, v in stats.items()
}
else:
# key didn't exist (or redis was down) - lets populate from DB.
stats = {
str(row.id): row.count for row in dao_get_template_usage(service_id, day=day)
}
# if there is data in db, but not in redis - lets put it in redis so we don't have to do
# this calc again next time. If there isn't any data, we can't put it in redis.
# Zero length hashes aren't a thing in redis. (There'll only be no data if the service has no templates)
# Nothing is stored if redis is down.
if stats:
redis_store.set_hash_and_expire(
key,
stats,
current_app.config['EXPIRE_CACHE_EIGHT_DAYS']
)
template_stats_by_id += Counter(stats)
# attach count from stats to name/type/etc from database
template_details = dao_get_multiple_template_details(template_stats_by_id.keys())
return [
{
'count': template_stats_by_id[str(template.id)],
'template_id': str(template.id),
'template_name': template.name,
'template_type': template.template_type,
'is_precompiled_letter': template.is_precompiled_letter
}
for template in template_details
# we don't want to return templates with no count to the front-end,
# but they're returned from the DB and might be put in redis like that (if there was no data that day)
if template_stats_by_id[str(template.id)] != 0
]

11
app/utils.py
View File

@@ -68,17 +68,6 @@ def get_london_month_from_utc_column(column):
)
def cache_key_for_service_template_counter(service_id, limit_days=7):
return "{}-template-counter-limit-{}-days".format(service_id, limit_days)
def cache_key_for_service_template_usage_per_day(service_id, datetime):
"""
You should pass a BST datetime into this function
"""
return "service-{}-template-usage-{}".format(service_id, datetime.date().isoformat())
def get_public_notify_type_text(notify_type, plural=False):
from app.models import (SMS_TYPE, UPLOAD_DOCUMENT, PRECOMPILED_LETTER)
notify_type_text = notify_type