mirror of
https://github.com/GSA/notifications-api.git
synced 2026-08-05 14:10:47 -04:00
refactor template stats endpoint to read from new redis keys
New redis keys are partitioned per service per day. New process is as follows: * require a count of days to filter by. Currently admin always gives 7. * for each day, check and see if there's anything in redis. There won't be if either a) redis is/was down or b) the service didn't send any notifications that day - if there isn't, go to the database and get a count out. * combine all these stats together * get the names/template types etc out of the DB at the end.
This commit is contained in:
Leo Hemsted
@@ -10,14 +10,13 @@ from app.dao.templates_dao import (
|
||||
dao_get_all_templates_for_service,
|
||||
dao_update_template,
|
||||
dao_get_template_versions,
|
||||
dao_get_templates_for_cache,
|
||||
dao_get_multiple_template_details,
|
||||
dao_redact_template, dao_update_template_reply_to
|
||||
)
|
||||
from app.models import (
|
||||
Template,
|
||||
TemplateHistory,
|
||||
TemplateRedacted,
|
||||
PRECOMPILED_TEMPLATE_NAME
|
||||
TemplateRedacted
|
||||
)
|
||||
|
||||
from tests.app.conftest import sample_template as create_sample_template
|
||||
@@ -481,77 +480,16 @@ def test_get_template_versions_is_empty_for_hidden_templates(notify_db, notify_d
|
||||
assert len(versions) == 0
|
||||
|
||||
|
||||
def test_get_templates_by_ids_successful(notify_db, notify_db_session):
|
||||
template_1 = create_sample_template(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
template_name='Sample Template 1',
|
||||
template_type="sms",
|
||||
content="Template content"
|
||||
)
|
||||
template_2 = create_sample_template(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
template_name='Sample Template 2',
|
||||
template_type="sms",
|
||||
content="Template content"
|
||||
)
|
||||
create_sample_template(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
template_name='Sample Template 3',
|
||||
template_type="email",
|
||||
content="Template content"
|
||||
)
|
||||
sample_cache_dict = {str.encode(str(template_1.id)): str.encode('2'),
|
||||
str.encode(str(template_2.id)): str.encode('3')}
|
||||
cache = [[k, v] for k, v in sample_cache_dict.items()]
|
||||
templates = dao_get_templates_for_cache(cache)
|
||||
assert len(templates) == 2
|
||||
assert [(template_1.id, template_1.template_type, template_1.name, False, 2),
|
||||
(template_2.id, template_2.template_type, template_2.name, False, 3)] == templates
|
||||
def test_get_multiple_template_details_returns_templates_for_list_of_ids(sample_service):
|
||||
t1 = create_template(sample_service)
|
||||
t2 = create_template(sample_service)
|
||||
create_template(sample_service) # t3
|
||||
|
||||
res = dao_get_multiple_template_details([t1.id, t2.id])
|
||||
|
||||
def test_get_letter_templates_by_ids_successful(notify_db, notify_db_session):
|
||||
template_1 = create_sample_template(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
template_name=PRECOMPILED_TEMPLATE_NAME,
|
||||
template_type="letter",
|
||||
content="Template content",
|
||||
hidden=True
|
||||
)
|
||||
template_2 = create_sample_template(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
template_name='Sample Template 2',
|
||||
template_type="letter",
|
||||
content="Template content"
|
||||
)
|
||||
sample_cache_dict = {str.encode(str(template_1.id)): str.encode('2'),
|
||||
str.encode(str(template_2.id)): str.encode('3')}
|
||||
cache = [[k, v] for k, v in sample_cache_dict.items()]
|
||||
templates = dao_get_templates_for_cache(cache)
|
||||
assert len(templates) == 2
|
||||
assert [(template_1.id, template_1.template_type, template_1.name, True, 2),
|
||||
(template_2.id, template_2.template_type, template_2.name, False, 3)] == templates
|
||||
|
||||
|
||||
def test_get_templates_by_ids_successful_for_one_cache_item(notify_db, notify_db_session):
|
||||
template_1 = create_sample_template(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
template_name='Sample Template 1',
|
||||
template_type="sms",
|
||||
content="Template content"
|
||||
)
|
||||
sample_cache_dict = {str.encode(str(template_1.id)): str.encode('2')}
|
||||
cache = [[k, v] for k, v in sample_cache_dict.items()]
|
||||
templates = dao_get_templates_for_cache(cache)
|
||||
assert len(templates) == 1
|
||||
assert [(template_1.id, template_1.template_type, template_1.name, False, 2)] == templates
|
||||
|
||||
|
||||
def test_get_templates_by_ids_returns_empty_list():
|
||||
assert dao_get_templates_for_cache({}) == []
|
||||
assert dao_get_templates_for_cache(None) == []
|
||||
assert {x.id for x in res} == {t1.id, t2.id}
|
||||
# make sure correct properties are on each row
|
||||
assert res[0].id
|
||||
assert res[0].template_type
|
||||
assert res[0].name
|
||||
assert not res[0].is_precompiled_letter
|
||||
|
||||
@@ -1,281 +1,129 @@
|
||||
from datetime import datetime, timedelta
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from freezegun import freeze_time
|
||||
|
||||
from app.dao.templates_dao import dao_update_template
|
||||
|
||||
from tests import create_authorization_header
|
||||
from tests.app.conftest import (
|
||||
sample_template as create_sample_template,
|
||||
sample_notification,
|
||||
sample_notification_history,
|
||||
sample_email_template
|
||||
)
|
||||
from tests.app.conftest import sample_notification
|
||||
|
||||
|
||||
def test_get_all_template_statistics_with_bad_arg_returns_400(client, sample_service):
|
||||
auth_header = create_authorization_header()
|
||||
# get_template_statistics_for_service_by_day
|
||||
|
||||
response = client.get(
|
||||
'/service/{}/template-statistics'.format(sample_service.id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header],
|
||||
query_string={'limit_days': 'blurk'}
|
||||
@pytest.mark.parametrize('query_string', [
|
||||
{},
|
||||
{'limit_days': 0},
|
||||
{'limit_days': 8},
|
||||
{'limit_days': 3.5},
|
||||
{'limit_days': 'blurk'},
|
||||
])
|
||||
def get_template_statistics_for_service_by_day_with_bad_arg_returns_400(admin_request, query_string):
|
||||
json_resp = admin_request.get(
|
||||
'template_statistics.get_template_statistics_for_service_by_day',
|
||||
service_id=uuid.uuid4(),
|
||||
**query_string,
|
||||
_expected_status=400
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
json_resp = json.loads(response.get_data(as_text=True))
|
||||
assert json_resp['result'] == 'error'
|
||||
assert json_resp['message'] == {'limit_days': ['blurk is not an integer']}
|
||||
assert 'limit_days' in json_resp['message']
|
||||
|
||||
|
||||
@freeze_time('2016-08-18')
|
||||
def test_get_template_statistics_for_service(notify_db, notify_db_session, client, mocker):
|
||||
email, sms = set_up_notifications(notify_db, notify_db_session)
|
||||
|
||||
mocked_redis = mocker.patch('app.redis_store.get_all_from_hash')
|
||||
|
||||
auth_header = create_authorization_header()
|
||||
|
||||
response = client.get(
|
||||
'/service/{}/template-statistics'.format(email.service_id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header]
|
||||
def test_get_template_statistics_for_service_by_day_returns_template_info(admin_request, mocker, sample_notification):
|
||||
json_resp = admin_request.get(
|
||||
'template_statistics.get_template_statistics_for_service_by_day',
|
||||
service_id=sample_notification.service_id,
|
||||
limit_days=1
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
json_resp = json.loads(response.get_data(as_text=True))
|
||||
assert len(json_resp['data']) == 2
|
||||
assert json_resp['data'][0]['count'] == 3
|
||||
assert json_resp['data'][0]['template_id'] == str(email.id)
|
||||
assert json_resp['data'][0]['template_name'] == email.name
|
||||
assert json_resp['data'][0]['template_type'] == email.template_type
|
||||
assert json_resp['data'][1]['count'] == 3
|
||||
assert json_resp['data'][1]['template_id'] == str(sms.id)
|
||||
assert json_resp['data'][1]['template_name'] == sms.name
|
||||
assert json_resp['data'][1]['template_type'] == sms.template_type
|
||||
assert len(json_resp) == 1
|
||||
|
||||
mocked_redis.assert_not_called()
|
||||
assert json_resp['data'][0]['count'] == 1
|
||||
assert json_resp['data'][0]['template_id'] == str(sample_notification.template_id)
|
||||
assert json_resp['data'][0]['template_name'] == 'Template Name'
|
||||
assert json_resp['data'][0]['template_type'] == 'sms'
|
||||
|
||||
|
||||
@freeze_time('2016-08-18')
|
||||
def test_get_template_statistics_for_service_limited_1_day(notify_db, notify_db_session, client,
|
||||
mocker):
|
||||
email, sms = set_up_notifications(notify_db, notify_db_session)
|
||||
mock_redis = mocker.patch('app.redis_store.get_all_from_hash')
|
||||
def test_get_template_statistics_for_service_by_day_gets_out_of_redis_if_available(admin_request, mocker):
|
||||
assert False
|
||||
|
||||
auth_header = create_authorization_header()
|
||||
|
||||
response = client.get(
|
||||
'/service/{}/template-statistics'.format(email.service_id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header],
|
||||
query_string={'limit_days': 1}
|
||||
def test_get_template_statistics_for_service_by_day_goes_to_db_if_not_in_redis(admin_request, mocker):
|
||||
assert False
|
||||
|
||||
|
||||
def test_get_template_statistics_for_service_by_day_gets_stats_for_correct_days(admin_request, mocker):
|
||||
assert False
|
||||
|
||||
|
||||
def test_get_template_statistics_for_service_by_day_returns_empty_list_if_no_templates(
|
||||
admin_request,
|
||||
mocker,
|
||||
sample_service
|
||||
):
|
||||
json_resp = admin_request.get(
|
||||
'template_statistics.get_template_statistics_for_service_by_day',
|
||||
service_id=sample_service.id,
|
||||
limit_days=1
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
json_resp = json.loads(response.get_data(as_text=True))['data']
|
||||
assert len(json_resp) == 2
|
||||
|
||||
assert json_resp[0]['count'] == 1
|
||||
assert json_resp[0]['template_id'] == str(email.id)
|
||||
assert json_resp[0]['template_name'] == email.name
|
||||
assert json_resp[0]['template_type'] == email.template_type
|
||||
assert json_resp[1]['count'] == 1
|
||||
assert json_resp[1]['template_id'] == str(sms.id)
|
||||
assert json_resp[1]['template_name'] == sms.name
|
||||
assert json_resp[1]['template_type'] == sms.template_type
|
||||
|
||||
mock_redis.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_values", [False, True])
|
||||
@freeze_time('2016-08-18')
|
||||
def test_get_template_statistics_for_service_limit_7_days(notify_db, notify_db_session, client,
|
||||
mocker,
|
||||
cache_values):
|
||||
email, sms = set_up_notifications(notify_db, notify_db_session)
|
||||
mock_cache_values = {str.encode(str(sms.id)): str.encode('3'),
|
||||
str.encode(str(email.id)): str.encode('3')} if cache_values else None
|
||||
mocked_redis_get = mocker.patch('app.redis_store.get_all_from_hash', return_value=mock_cache_values)
|
||||
mocked_redis_set = mocker.patch('app.redis_store.set_hash_and_expire')
|
||||
|
||||
auth_header = create_authorization_header()
|
||||
response_for_a_week = client.get(
|
||||
'/service/{}/template-statistics'.format(email.service_id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header],
|
||||
query_string={'limit_days': 7}
|
||||
)
|
||||
|
||||
assert response_for_a_week.status_code == 200
|
||||
json_resp = json.loads(response_for_a_week.get_data(as_text=True))
|
||||
assert len(json_resp['data']) == 2
|
||||
assert json_resp['data'][0]['count'] == 3
|
||||
assert json_resp['data'][0]['template_name'] == 'New Email Template Name'
|
||||
assert json_resp['data'][1]['count'] == 3
|
||||
assert json_resp['data'][1]['template_name'] == 'New SMS Template Name'
|
||||
|
||||
mocked_redis_get.assert_called_once_with("{}-template-counter-limit-7-days".format(email.service_id))
|
||||
if cache_values:
|
||||
mocked_redis_set.assert_not_called()
|
||||
else:
|
||||
mocked_redis_set.assert_called_once_with("{}-template-counter-limit-7-days".format(email.service_id),
|
||||
{sms.id: 3, email.id: 3}, 600)
|
||||
|
||||
|
||||
@freeze_time('2016-08-18')
|
||||
def test_get_template_statistics_for_service_limit_30_days(notify_db, notify_db_session, client,
|
||||
mocker):
|
||||
email, sms = set_up_notifications(notify_db, notify_db_session)
|
||||
mock_redis = mocker.patch('app.redis_store.get_all_from_hash')
|
||||
|
||||
auth_header = create_authorization_header()
|
||||
|
||||
response_for_a_month = client.get(
|
||||
'/service/{}/template-statistics'.format(email.service_id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header],
|
||||
query_string={'limit_days': 30}
|
||||
)
|
||||
|
||||
assert response_for_a_month.status_code == 200
|
||||
json_resp = json.loads(response_for_a_month.get_data(as_text=True))
|
||||
assert len(json_resp['data']) == 2
|
||||
assert json_resp['data'][0]['count'] == 3
|
||||
assert json_resp['data'][0]['template_name'] == 'New Email Template Name'
|
||||
assert json_resp['data'][1]['count'] == 3
|
||||
assert json_resp['data'][1]['template_name'] == 'New SMS Template Name'
|
||||
|
||||
mock_redis.assert_not_called()
|
||||
|
||||
|
||||
@freeze_time('2016-08-18')
|
||||
def test_get_template_statistics_for_service_no_limit(notify_db, notify_db_session, client,
|
||||
mocker):
|
||||
email, sms = set_up_notifications(notify_db, notify_db_session)
|
||||
mock_redis = mocker.patch('app.redis_store.get_all_from_hash')
|
||||
auth_header = create_authorization_header()
|
||||
response_for_all = client.get(
|
||||
'/service/{}/template-statistics'.format(email.service_id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header]
|
||||
)
|
||||
assert response_for_all.status_code == 200
|
||||
json_resp = json.loads(response_for_all.get_data(as_text=True))
|
||||
assert len(json_resp['data']) == 2
|
||||
assert json_resp['data'][0]['count'] == 3
|
||||
assert json_resp['data'][0]['template_name'] == 'New Email Template Name'
|
||||
assert json_resp['data'][1]['count'] == 3
|
||||
assert json_resp['data'][1]['template_name'] == 'New SMS Template Name'
|
||||
|
||||
mock_redis.assert_not_called()
|
||||
|
||||
|
||||
def set_up_notifications(notify_db, notify_db_session):
|
||||
sms = create_sample_template(notify_db, notify_db_session)
|
||||
email = sample_email_template(notify_db, notify_db_session)
|
||||
today = datetime.now()
|
||||
a_week_ago = datetime.now() - timedelta(days=7)
|
||||
a_month_ago = datetime.now() - timedelta(days=30)
|
||||
sample_notification(notify_db, notify_db_session, created_at=a_month_ago, template=sms)
|
||||
sample_notification(notify_db, notify_db_session, created_at=a_month_ago, template=email)
|
||||
email.name = 'Updated Email Template Name'
|
||||
dao_update_template(email)
|
||||
sms.name = 'Updated SMS Template Name'
|
||||
dao_update_template(sms)
|
||||
sample_notification(notify_db, notify_db_session, created_at=a_week_ago, template=sms)
|
||||
sample_notification(notify_db, notify_db_session, created_at=a_week_ago, template=email)
|
||||
email.name = 'New Email Template Name'
|
||||
dao_update_template(email)
|
||||
sms.name = 'New SMS Template Name'
|
||||
dao_update_template(sms)
|
||||
sample_notification(notify_db, notify_db_session, created_at=today, template=sms)
|
||||
sample_notification(notify_db, notify_db_session, created_at=today, template=email)
|
||||
return email, sms
|
||||
|
||||
|
||||
@freeze_time('2016-08-18')
|
||||
def test_returns_empty_list_if_no_templates_used(client, sample_service, mocker):
|
||||
auth_header = create_authorization_header()
|
||||
mock_redis = mocker.patch('app.redis_store.set_hash_and_expire')
|
||||
|
||||
response = client.get(
|
||||
'/service/{}/template-statistics'.format(sample_service.id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header]
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
json_resp = json.loads(response.get_data(as_text=True))
|
||||
assert len(json_resp['data']) == 0
|
||||
mock_redis.assert_not_called()
|
||||
|
||||
# get_template_statistics_for_template
|
||||
|
||||
|
||||
def test_get_template_statistics_by_id_returns_last_notification(
|
||||
def test_get_template_statistics_for_template_returns_last_notification(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
client):
|
||||
admin_request):
|
||||
sample_notification(notify_db, notify_db_session)
|
||||
sample_notification(notify_db, notify_db_session)
|
||||
notification_3 = sample_notification(notify_db, notify_db_session)
|
||||
|
||||
auth_header = create_authorization_header()
|
||||
|
||||
response = client.get(
|
||||
'/service/{}/template-statistics/{}'.format(notification_3.service_id, notification_3.template_id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header],
|
||||
json_resp = admin_request.get(
|
||||
'template_statistics.get_template_statistics_for_template_id',
|
||||
service_id=notification_3.service_id,
|
||||
template_id=notification_3.template_id
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
json_resp = json.loads(response.get_data(as_text=True))['data']
|
||||
assert json_resp['id'] == str(notification_3.id)
|
||||
assert json_resp['data']['id'] == str(notification_3.id)
|
||||
|
||||
|
||||
def test_get_template_statistics_for_template_returns_empty_if_no_statistics(
|
||||
client,
|
||||
admin_request,
|
||||
sample_template,
|
||||
):
|
||||
auth_header = create_authorization_header()
|
||||
|
||||
response = client.get(
|
||||
'/service/{}/template-statistics/{}'.format(sample_template.service_id, sample_template.id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header],
|
||||
json_resp = admin_request.get(
|
||||
'template_statistics.get_template_statistics_for_template_id',
|
||||
service_id=sample_template.service_id,
|
||||
template_id=sample_template.id
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
json_resp = json.loads(response.get_data(as_text=True))
|
||||
assert not json_resp['data']
|
||||
|
||||
|
||||
def test_get_template_statistics_raises_error_for_nonexistent_template(
|
||||
client,
|
||||
def test_get_template_statistics_for_template_raises_error_for_nonexistent_template(
|
||||
admin_request,
|
||||
sample_service,
|
||||
fake_uuid
|
||||
):
|
||||
auth_header = create_authorization_header()
|
||||
|
||||
response = client.get(
|
||||
'/service/{}/template-statistics/{}'.format(sample_service.id, fake_uuid),
|
||||
headers=[('Content-Type', 'application/json'), auth_header],
|
||||
json_resp = admin_request.get(
|
||||
'template_statistics.get_template_statistics_for_template_id',
|
||||
service_id=sample_service.id,
|
||||
template_id=fake_uuid,
|
||||
_expected_status=404
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
json_resp = json.loads(response.get_data(as_text=True))
|
||||
assert json_resp['message'] == 'No result found'
|
||||
assert json_resp['result'] == 'error'
|
||||
|
||||
|
||||
def test_get_template_statistics_by_id_returns_empty_for_old_notification(
|
||||
notify_db,
|
||||
notify_db_session,
|
||||
client,
|
||||
sample_template
|
||||
def test_get_template_statistics_for_template_returns_empty_for_old_notification(
|
||||
admin_request,
|
||||
sample_notification_history
|
||||
):
|
||||
sample_notification_history(notify_db, notify_db_session, sample_template)
|
||||
|
||||
auth_header = create_authorization_header()
|
||||
|
||||
response = client.get(
|
||||
'/service/{}/template-statistics/{}'.format(sample_template.service.id, sample_template.id),
|
||||
headers=[('Content-Type', 'application/json'), auth_header],
|
||||
json_resp = admin_request.get(
|
||||
'template_statistics.get_template_statistics_for_template_id',
|
||||
service_id=sample_notification_history.service_id,
|
||||
template_id=sample_notification_history.template_id
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
json_resp = json.loads(response.get_data(as_text=True))['data']
|
||||
assert not json_resp
|
||||
assert not json_resp['data']
|
||||
|
||||
@@ -8,7 +8,8 @@ from app.utils import (
|
||||
get_midnight_for_day_before,
|
||||
convert_utc_to_bst,
|
||||
convert_bst_to_utc,
|
||||
days_ago
|
||||
days_ago,
|
||||
last_n_days
|
||||
)
|
||||
|
||||
|
||||
@@ -66,3 +67,16 @@ def test_convert_bst_to_utc():
|
||||
def test_days_ago(current_time, expected_datetime):
|
||||
with freeze_time(current_time):
|
||||
assert days_ago(1) == expected_datetime
|
||||
|
||||
|
||||
def test_last_n_days():
|
||||
with freeze_time('2018-03-27 12:00'):
|
||||
res = last_n_days(5)
|
||||
|
||||
assert res == [
|
||||
datetime(2018, 3, 23, 0, 0),
|
||||
datetime(2018, 3, 24, 0, 0),
|
||||
datetime(2018, 3, 25, 0, 0),
|
||||
datetime(2018, 3, 26, 0, 0),
|
||||
datetime(2018, 3, 27, 0, 0)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user