darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-23 08:51:30 -05:00
Code Issues Packages Projects Releases Wiki Activity

Added a redis cache for the template usage stats.

Browse Source 
Cache expires every 10 minutes, but will help with the every 2 second query, especially when a job is running.
There is some clean up and qa to do for this yet
This commit is contained in:
Rebecca Law
2017-02-13 18:47:29 +00:00
parent b2267ae5fc
commit 458adefcb8
9 changed files with 351 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/__init__.py
View File

@@ -65,6 +65,8 @@ def create_app(app_name=None):
aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
notify_celery.init_app(application)
encryption.init_app(application)
print(os.environ['REDIS_URL'])
print(application.config['REDIS_ENABLED'])
redis_store.init_app(application)
performance_platform_client.init_app(application)
clients.init_app(sms_clients=[firetext_client, mmg_client, loadtest_client], email_clients=[aws_ses_client])
@@ -176,3 +178,7 @@ def process_user_agent(user_agent_string):
return "non-notify-user-agent"
else:
return "unknown"
def cache_key_for_service_template_counter(service_id, limit_days=7):
return "{}-template-counter-limit-{}-days".format(service_id, limit_days)

2
app/config.py
View File

@@ -49,6 +49,7 @@ class Config(object):
# URL of redis instance
REDIS_URL = os.getenv('REDIS_URL')
REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
EXPIRE_CACHE_IN_SECONDS = 600
# Performance platform
PERFORMANCE_PLATFORM_ENABLED = os.getenv('PERFORMANCE_PLATFORM_ENABLED') == '1'
@@ -185,6 +186,7 @@ class Development(Config):
Queue('research-mode', Exchange('default'), routing_key='research-mode')
]
API_HOST_NAME = "http://localhost:6011"
REDIS_ENABLED = True
class Test(Config):

39
app/dao/templates_dao.py
View File

@@ -1,6 +1,7 @@
import uuid
from sqlalchemy import (asc, desc)
import sqlalchemy
from sqlalchemy import (desc, cast, String, text)
from app import db
from app.models import (Template, TemplateHistory)
@@ -56,3 +57,39 @@ def dao_get_template_versions(service_id, template_id):
).order_by(
desc(TemplateHistory.version)
).all()
# def dao_get_templates_by_for_cache(cache):
# if not cache or len(cache) == 0:
# return []
# # First create a subquery that is a union select of the cache values
# # Then join templates to the subquery
# cache_queries = [
# db.session.query(sqlalchemy.sql.expression.bindparam("template_id" + str(i),
# template_id).label('template_id'),
# sqlalchemy.sql.expression.bindparam("count" + str(i), count).label('count'))
# for i, (template_id, count) in enumerate(cache)]
# cache_subq = cache_queries[0].union(*cache_queries[1:]).subquery()
# query = db.session.query(Template.id.label('template_id'),
# Template.template_type,
# Template.name,
# cache_subq.c.count.label('count')
# ).join(cache_subq,
# cast(Template.id, String) == cast(cache_subq.c.template_id, String)
# ).order_by(Template.name)
#
# return query.all()
def dao_get_templates_by_for_cache(cache):
if not cache or len(cache) == 0:
return []
txt = "( " + " Union all ".join(
"select '{}'::text as template_id, {} as count".format(x.decode(),
y.decode()) for x, y in cache) + " ) as cache"
txt = "Select t.id as template_id, t.template_type, t.name, cache.count from templates t, " + \
txt + " where t.id::text = cache.template_id order by t.name"
stmt = text(txt)
return db.session.execute(stmt).fetchall()

3
app/notifications/process_notifications.py
View File

@@ -2,7 +2,7 @@ from datetime import datetime
from flask import current_app
from app import redis_store
from app import redis_store, cache_key_for_service_template_counter
from app.celery import provider_tasks
from notifications_utils.clients import redis
from app.dao.notifications_dao import dao_create_notification, dao_delete_notifications_and_history_by_id
@@ -63,6 +63,7 @@ def persist_notification(template_id,
if not simulated:
dao_create_notification(notification)
redis_store.incr(redis.daily_limit_cache_key(service.id))
redis_store.increment_hash_value(cache_key_for_service_template_counter(service.id), template_id)
current_app.logger.info(
"{} {} created at {}".format(notification.notification_type, notification.id, notification.created_at)
)

31
app/template_statistics/rest.py
View File

@@ -1,14 +1,16 @@
from flask import (
Blueprint,
jsonify,
request
)
request,
current_app)
from app import redis_store
from app.dao.notifications_dao import (
dao_get_template_usage,
dao_get_last_template_usage)
from app.dao.templates_dao import dao_get_templates_by_for_cache
from app.schemas import notifications_filter_schema, NotificationWithTemplateSchema, notification_with_template_schema
from app.schemas import notification_with_template_schema
template_statistics = Blueprint('template-statistics',
__name__,
@@ -30,7 +32,15 @@ def get_template_statistics_for_service_by_day(service_id):
raise InvalidRequest(message, status_code=400)
else:
limit_days = None
if limit_days == 7:
stats = get_template_statistics_for_7_days(limit_days, service_id)
print(stats)
# [(UUID('c2a331f8-e0b9-43de-9dd2-88300511a1d7'), 'Create with priority', 'sms', 1)]
else:
stats = dao_get_template_usage(service_id, limit_days=limit_days)
print(stats)
def serialize(data):
return {
@@ -52,3 +62,18 @@ def get_template_statistics_for_template_id(service_id, template_id):
raise InvalidRequest(errors, status_code=404)
data = notification_with_template_schema.dump(notification).data
return jsonify(data=data)
def get_template_statistics_for_7_days(limit_days, service_id):
cache_key = "{}-template-counter-limit-7-days".format(service_id)
template_stats_by_id = redis_store.get_all_from_hash(cache_key)
if not template_stats_by_id:
print("populate cache")
stats = dao_get_template_usage(service_id, limit_days=limit_days)
cache_values = dict([(x.template_id, x.count) for x in stats])
redis_store.set_hash_and_expire(cache_key, cache_values, current_app.config.get('EXPIRE_CACHE_IN_SECONDS', 600))
current_app.logger.info('use redis-client: {}'.format(cache_key))
else:
print("template_stats_by_id: {}".format(template_stats_by_id))
stats = dao_get_templates_by_for_cache(template_stats_by_id.items())
return stats

54
tests/app/dao/test_templates_dao.py
View File

@@ -4,7 +4,8 @@ from app.dao.templates_dao import (
dao_get_template_by_id_and_service_id,
dao_get_all_templates_for_service,
dao_update_template,
dao_get_template_versions)
dao_get_template_versions,
dao_get_templates_by_for_cache)
from tests.app.conftest import sample_template as create_sample_template
from app.models import Template, TemplateHistory
import pytest
@@ -265,3 +266,54 @@ def test_get_template_versions(sample_template):
from app.schemas import template_history_schema
v = template_history_schema.load(versions, many=True)
assert len(v) == 2
def test_get_templates_by_ids_successful(notify_db, notify_db_session):
template_1 = create_sample_template(
notify_db,
notify_db_session,
template_name='Sample Template 1',
template_type="sms",
content="Template content"
)
template_2 = create_sample_template(
notify_db,
notify_db_session,
template_name='Sample Template 2',
template_type="sms",
content="Template content"
)
create_sample_template(
notify_db,
notify_db_session,
template_name='Sample Template 3',
template_type="email",
content="Template content"
)
sample_cache_dict = {str.encode(str(template_1.id)): str.encode('2'),
str.encode(str(template_2.id)): str.encode('3')}
cache = [[k, v] for k, v in sample_cache_dict.items()]
templates = dao_get_templates_by_for_cache(cache)
assert len(templates) == 2
assert [(template_1.id, template_1.template_type, template_1.name, 2),
(template_2.id, template_2.template_type, template_2.name, 3)] == templates
def test_get_templates_by_ids_successful_for_one_cache_item(notify_db, notify_db_session):
template_1 = create_sample_template(
notify_db,
notify_db_session,
template_name='Sample Template 1',
template_type="sms",
content="Template content"
)
sample_cache_dict = {str.encode(str(template_1.id)): str.encode('2')}
cache = [[k, v] for k, v in sample_cache_dict.items()]
templates = dao_get_templates_by_for_cache(cache)
assert len(templates) == 1
assert [(template_1.id, template_1.template_type, template_1.name, 2)] == templates
def test_get_templates_by_ids_returns_empty_list():
assert dao_get_templates_by_for_cache({}) == []
assert dao_get_templates_by_for_cache(None) == []

5
tests/app/notifications/rest/test_send_notification.py
View File

@@ -670,18 +670,17 @@ def test_should_persist_notification(notify_api, sample_template,
@pytest.mark.parametrize('template_type',
['sms', 'email'])
def test_should_delete_notification_and_return_error_if_sqs_fails(
notify_api,
client,
sample_email_template,
sample_template,
fake_uuid,
mocker,
template_type):
with notify_api.test_request_context(), notify_api.test_client() as client:
mocked = mocker.patch(
'app.celery.provider_tasks.deliver_{}.apply_async'.format(template_type),
side_effect=Exception("failed to talk to SQS")
)
m1 = mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
template = sample_template if template_type == 'sms' else sample_email_template
to = sample_template.service.created_by.mobile_number if template_type == 'sms' \
else sample_email_template.service.created_by.email_address

8
tests/app/notifications/test_process_notification.py
View File

@@ -7,6 +7,7 @@ from sqlalchemy.exc import SQLAlchemyError
from freezegun import freeze_time
from collections import namedtuple
from app import cache_key_for_service_template_counter
from app.models import Template, Notification, NotificationHistory
from app.notifications import SendNotificationToQueueError
from app.notifications.process_notifications import (create_content_for_notification,
@@ -114,7 +115,8 @@ def test_exception_thown_by_redis_store_get_should_not_be_fatal(sample_template,
def test_cache_is_not_incremented_on_failure_to_persist_notification(sample_api_key, mocker):
mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.incr')
mocked_redis = mocker.patch('app.redis_store.incr')
mocked_redis_hash = mocker.patch('app.redis_store.increment_hash_value')
with pytest.raises(SQLAlchemyError):
persist_notification(template_id=None,
template_version=None,
@@ -125,6 +127,7 @@ def test_cache_is_not_incremented_on_failure_to_persist_notification(sample_api_
api_key_id=sample_api_key.id,
key_type=sample_api_key.key_type)
mocked_redis.assert_not_called()
mocked_redis_hash.assert_not_called()
@freeze_time("2016-01-01 11:09:00.061258")
@@ -132,6 +135,7 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker)
assert Notification.query.count() == 0
assert NotificationHistory.query.count() == 0
mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.incr')
mocked_redis_hash = mocker.patch('app.notifications.process_notifications.redis_store.increment_hash_value')
n_id = uuid.uuid4()
created_at = datetime.datetime(2016, 11, 11, 16, 8, 18)
persist_notification(template_id=sample_job.template.id,
@@ -154,6 +158,8 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker)
assert persisted_notification.job_row_number == 10
assert persisted_notification.created_at == created_at
mocked_redis.assert_called_once_with(str(sample_job.service_id) + "-2016-01-01-count")
mocked_redis_hash.assert_called_once_with(cache_key_for_service_template_counter(sample_job.service_id),
sample_job.template.id)
assert persisted_notification.client_reference == "ref from client"
assert persisted_notification.reference is None

193
tests/app/template_statistics/test_rest.py
View File

@@ -1,16 +1,14 @@
from datetime import datetime, timedelta
import json
import pytest
from freezegun import freeze_time
from tests import create_authorization_header
from tests.app.conftest import sample_template as create_sample_template, sample_template, sample_notification, \
sample_email_template
from tests.app.conftest import (sample_template as create_sample_template, sample_notification, sample_email_template)
def test_get_all_template_statistics_with_bad_arg_returns_400(notify_api, sample_service):
with notify_api.test_request_context():
with notify_api.test_client() as client:
def test_get_all_template_statistics_with_bad_arg_returns_400(client, sample_service):
auth_header = create_authorization_header()
response = client.get(
@@ -26,80 +24,82 @@ def test_get_all_template_statistics_with_bad_arg_returns_400(notify_api, sample
@freeze_time('2016-08-18')
def test_get_template_statistics_for_service(notify_db, notify_db_session, notify_api, sample_service):
sms = sample_template(notify_db, notify_db_session, service=sample_service)
email = sample_email_template(notify_db, notify_db_session, service=sample_service)
today = datetime.now()
sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=sms)
sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=sms)
sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=email)
sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=email)
def test_get_template_statistics_for_service(notify_db, notify_db_session, client, mocker):
email, sms = set_up_notifications(notify_db, notify_db_session)
mocked_redis = mocker.patch('app.redis_store.get_all_from_hash')
with notify_api.test_request_context():
with notify_api.test_client() as client:
auth_header = create_authorization_header()
response = client.get(
'/service/{}/template-statistics'.format(sample_service.id),
'/service/{}/template-statistics'.format(email.service_id),
headers=[('Content-Type', 'application/json'), auth_header]
)
assert response.status_code == 200
json_resp = json.loads(response.get_data(as_text=True))
assert len(json_resp['data']) == 2
assert json_resp['data'][0]['count'] == 2
assert json_resp['data'][0]['count'] == 3
assert json_resp['data'][0]['template_id'] == str(email.id)
assert json_resp['data'][0]['template_name'] == email.name
assert json_resp['data'][0]['template_type'] == email.template_type
assert json_resp['data'][1]['count'] == 2
assert json_resp['data'][1]['count'] == 3
assert json_resp['data'][1]['template_id'] == str(sms.id)
assert json_resp['data'][1]['template_name'] == sms.name
assert json_resp['data'][1]['template_type'] == sms.template_type
mocked_redis.assert_not_called()
@freeze_time('2016-08-18')
def test_get_template_statistics_for_service_limited_by_day(notify_db, notify_db_session, notify_api, sample_service):
sms = sample_template(notify_db, notify_db_session, service=sample_service)
email = sample_email_template(notify_db, notify_db_session, service=sample_service)
today = datetime.now()
a_week_ago = datetime.now() - timedelta(days=7)
a_month_ago = datetime.now() - timedelta(days=30)
sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=sms)
sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=email)
sample_notification(notify_db, notify_db_session, created_at=a_week_ago, service=sample_service, template=sms)
sample_notification(notify_db, notify_db_session, created_at=a_week_ago, service=sample_service, template=email)
sample_notification(notify_db, notify_db_session, created_at=a_month_ago, service=sample_service, template=sms)
sample_notification(notify_db, notify_db_session, created_at=a_month_ago, service=sample_service, template=email)
def test_get_template_statistics_for_service_limited_1_day(notify_db, notify_db_session, client,
mocker):
email, sms = set_up_notifications(notify_db, notify_db_session)
mock_redis = mocker.patch('app.redis_store.get_all_from_hash')
with notify_api.test_request_context():
with notify_api.test_client() as client:
auth_header = create_authorization_header()
response = client.get(
'/service/{}/template-statistics'.format(sample_service.id),
'/service/{}/template-statistics'.format(email.service_id),
headers=[('Content-Type', 'application/json'), auth_header],
query_string={'limit_days': 1}
)
assert response.status_code == 200
json_resp = json.loads(response.get_data(as_text=True))
assert len(json_resp['data']) == 2
assert json_resp['data'][0]['count'] == 1
assert json_resp['data'][0]['template_id'] == str(email.id)
assert json_resp['data'][0]['template_name'] == email.name
assert json_resp['data'][0]['template_type'] == email.template_type
assert json_resp['data'][1]['count'] == 1
assert json_resp['data'][1]['template_id'] == str(sms.id)
assert json_resp['data'][1]['template_name'] == sms.name
assert json_resp['data'][1]['template_type'] == sms.template_type
json_resp = json.loads(response.get_data(as_text=True))['data']
assert len(json_resp) == 2
assert json_resp[0]['count'] == 1
assert json_resp[0]['template_id'] == str(email.id)
assert json_resp[0]['template_name'] == email.name
assert json_resp[0]['template_type'] == email.template_type
assert json_resp[1]['count'] == 1
assert json_resp[1]['template_id'] == str(sms.id)
assert json_resp[1]['template_name'] == sms.name
assert json_resp[1]['template_type'] == sms.template_type
mock_redis.assert_not_called()
@pytest.mark.parametrize("cache_values", [False, True])
@freeze_time('2016-08-18')
def test_get_template_statistics_for_service_limit_7_days(notify_db, notify_db_session, client,
mocker,
cache_values):
email, sms = set_up_notifications(notify_db, notify_db_session)
mock_cache_values = {str.encode(str(sms.id)): str.encode('2'),
str.encode(str(email.id)): str.encode('2')} if cache_values else None
mocked_redis_get = mocker.patch('app.redis_store.get_all_from_hash', return_value=mock_cache_values)
mocked_redis_set = mocker.patch('app.redis_store.set_hash_and_expire')
auth_header = create_authorization_header()
response_for_a_week = client.get(
'/service/{}/template-statistics'.format(sample_service.id),
'/service/{}/template-statistics'.format(email.service_id),
headers=[('Content-Type', 'application/json'), auth_header],
query_string={'limit_days': 7}
)
assert response.status_code == 200
assert response_for_a_week.status_code == 200
json_resp = json.loads(response_for_a_week.get_data(as_text=True))
assert len(json_resp['data']) == 2
assert json_resp['data'][0]['count'] == 2
@@ -107,8 +107,24 @@ def test_get_template_statistics_for_service_limited_by_day(notify_db, notify_db
assert json_resp['data'][1]['count'] == 2
assert json_resp['data'][1]['template_name'] == 'Template Name'
mocked_redis_get.assert_called_once_with("{}-template-counter-limit-7-days".format(email.service_id))
if cache_values:
mocked_redis_set.assert_not_called()
else:
mocked_redis_set.assert_called_once_with("{}-template-counter-limit-7-days".format(email.service_id),
{sms.id: 2, email.id: 2}, 600)
@freeze_time('2016-08-18')
def test_get_template_statistics_for_service_limit_30_days(notify_db, notify_db_session, client,
mocker):
email, sms = set_up_notifications(notify_db, notify_db_session)
mock_redis = mocker.patch('app.redis_store.get_all_from_hash')
auth_header = create_authorization_header()
response_for_a_month = client.get(
'/service/{}/template-statistics'.format(sample_service.id),
'/service/{}/template-statistics'.format(email.service_id),
headers=[('Content-Type', 'application/json'), auth_header],
query_string={'limit_days': 30}
)
@@ -121,11 +137,19 @@ def test_get_template_statistics_for_service_limited_by_day(notify_db, notify_db
assert json_resp['data'][1]['count'] == 3
assert json_resp['data'][1]['template_name'] == 'Template Name'
mock_redis.assert_not_called()
@freeze_time('2016-08-18')
def test_get_template_statistics_for_service_no_limit(notify_db, notify_db_session, client,
mocker):
email, sms = set_up_notifications(notify_db, notify_db_session)
mock_redis = mocker.patch('app.redis_store.get_all_from_hash')
auth_header = create_authorization_header()
response_for_all = client.get(
'/service/{}/template-statistics'.format(sample_service.id),
'/service/{}/template-statistics'.format(email.service_id),
headers=[('Content-Type', 'application/json'), auth_header]
)
assert response_for_all.status_code == 200
json_resp = json.loads(response_for_all.get_data(as_text=True))
assert len(json_resp['data']) == 2
@@ -134,11 +158,26 @@ def test_get_template_statistics_for_service_limited_by_day(notify_db, notify_db
assert json_resp['data'][1]['count'] == 3
assert json_resp['data'][1]['template_name'] == 'Template Name'
mock_redis.assert_not_called()
def set_up_notifications(notify_db, notify_db_session):
sms = create_sample_template(notify_db, notify_db_session)
email = sample_email_template(notify_db, notify_db_session)
today = datetime.now()
a_week_ago = datetime.now() - timedelta(days=7)
a_month_ago = datetime.now() - timedelta(days=30)
sample_notification(notify_db, notify_db_session, created_at=today, template=sms)
sample_notification(notify_db, notify_db_session, created_at=today, template=email)
sample_notification(notify_db, notify_db_session, created_at=a_week_ago, template=sms)
sample_notification(notify_db, notify_db_session, created_at=a_week_ago, template=email)
sample_notification(notify_db, notify_db_session, created_at=a_month_ago, template=sms)
sample_notification(notify_db, notify_db_session, created_at=a_month_ago, template=email)
return email, sms
@freeze_time('2016-08-18')
def test_returns_empty_list_if_no_templates_used(notify_api, sample_service):
with notify_api.test_request_context():
with notify_api.test_client() as client:
def test_returns_empty_list_if_no_templates_used(client, sample_service):
auth_header = create_authorization_header()
response = client.get(
@@ -154,38 +193,15 @@ def test_returns_empty_list_if_no_templates_used(notify_api, sample_service):
def test_get_template_statistics_by_id_returns_last_notification(
notify_db,
notify_db_session,
notify_api,
sample_service):
client):
sample_notification(notify_db, notify_db_session)
sample_notification(notify_db, notify_db_session)
notification_3 = sample_notification(notify_db, notify_db_session)
template = create_sample_template(
notify_db,
notify_db_session,
template_name='Sample Template 1',
service=sample_service
)
notification_1 = sample_notification(
notify_db,
notify_db_session,
service=sample_service,
template=template)
notification_2 = sample_notification(
notify_db,
notify_db_session,
service=sample_service,
template=template)
notification_3 = sample_notification(
notify_db,
notify_db_session,
service=sample_service,
template=template)
with notify_api.test_request_context():
with notify_api.test_client() as client:
auth_header = create_authorization_header()
response = client.get(
'/service/{}/template-statistics/{}'.format(sample_service.id, template.id),
'/service/{}/template-statistics/{}'.format(notification_3.service_id, notification_3.template_id),
headers=[('Content-Type', 'application/json'), auth_header],
)
@@ -195,28 +211,17 @@ def test_get_template_statistics_by_id_returns_last_notification(
def test_get_template_statistics_for_template_returns_empty_if_no_statistics(
notify_db,
notify_db_session,
notify_api,
sample_service
client,
sample_template,
):
template = create_sample_template(
notify_db,
notify_db_session,
template_name='Sample Template 1',
service=sample_service
)
with notify_api.test_request_context():
with notify_api.test_client() as client:
auth_header = create_authorization_header()
response = client.get(
'/service/{}/template-statistics/{}'.format(sample_service.id, template.id),
'/service/{}/template-statistics/{}'.format(sample_template.service_id, sample_template.id),
headers=[('Content-Type', 'application/json'), auth_header],
)
assert response.status_code == 404
json_resp = json.loads(response.get_data(as_text=True))
assert json_resp['result'] == 'error'
assert json_resp['message']['template_id'] == ['No template found for id {}'.format(template.id)]
assert json_resp['message']['template_id'] == ['No template found for id {}'.format(sample_template.id)]