darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-07-16 19:30:51 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #735 from alphagov/caching-with-redis

Browse Source 
Caching with redis
This commit is contained in:
minglis
2016-11-25 09:06:12 +00:00
committed by GitHub
parent 8fa337927c 2a2ef34339
commit 2e1578656b
17 changed files with 394 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
tests/app/celery/test_tasks.py
View File

@@ -631,27 +631,28 @@ def test_should_not_send_sms_if_team_key_and_recipient_not_in_team(notify_db, no
def test_should_use_email_template_and_persist(sample_email_template_with_placeholders, sample_api_key, mocker):
notification = _notification_json(
sample_email_template_with_placeholders,
'my_email@my_email.com',
{"name": "Jo"},
row_number=1)
mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
with freeze_time("2016-01-01 12:00:00.000000"):
notification = _notification_json(
sample_email_template_with_placeholders,
'my_email@my_email.com',
{"name": "Jo"},
row_number=1)
mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
notification_id = uuid.uuid4()
notification_id = uuid.uuid4()
with freeze_time("2016-01-01 11:09:00.00000"):
now = datetime.utcnow()
with freeze_time("2016-01-01 11:09:00.00000"):
now = datetime.utcnow()
with freeze_time("2016-01-01 11:10:00.00000"):
send_email(
sample_email_template_with_placeholders.service_id,
notification_id,
encryption.encrypt(notification),
now.strftime(DATETIME_FORMAT),
api_key_id=str(sample_api_key.id),
key_type=sample_api_key.key_type
)
with freeze_time("2016-01-01 11:10:00.00000"):
send_email(
sample_email_template_with_placeholders.service_id,
notification_id,
encryption.encrypt(notification),
now.strftime(DATETIME_FORMAT),
api_key_id=str(sample_api_key.id),
key_type=sample_api_key.key_type
)
persisted_notification = Notification.query.one()
assert persisted_notification.to == 'my_email@my_email.com'

81
tests/app/clients/test_redis_client.py Normal file
View File

@@ -0,0 +1,81 @@
import pytest
from unittest.mock import Mock
from app.clients.redis.redis_client import RedisClient
from app.clients.redis import daily_limit_cache_key
from freezegun import freeze_time
@pytest.fixture(scope='function')
def enabled_redis_client(notify_api, mocker):
notify_api.config['REDIS_ENABLED'] = True
return build_redis_client(notify_api, mocker)
@pytest.fixture(scope='function')
def disabled_redis_client(notify_api, mocker):
notify_api.config['REDIS_ENABLED'] = False
return build_redis_client(notify_api, mocker)
def build_redis_client(notify_api, mocker):
redis_client = RedisClient()
redis_client.init_app(notify_api)
mocker.patch.object(redis_client.redis_store, 'get', return_value=100)
mocker.patch.object(redis_client.redis_store, 'set')
return redis_client
def test_should_not_raise_exception_if_raise_set_to_false(notify_api):
notify_api.config['REDIS_ENABLED'] = True
redis_client = RedisClient()
redis_client.init_app(notify_api)
redis_client.redis_store.get = Mock(side_effect=Exception())
redis_client.redis_store.set = Mock(side_effect=Exception())
redis_client.redis_store.incr = Mock(side_effect=Exception())
assert redis_client.get('test') is None
assert redis_client.set('test', 'test') is None
assert redis_client.incr('test') is None
def test_should_raise_exception_if_raise_set_to_true(notify_api):
notify_api.config['REDIS_ENABLED'] = True
redis_client = RedisClient()
redis_client.init_app(notify_api)
redis_client.redis_store.get = Mock(side_effect=Exception('get failed'))
redis_client.redis_store.set = Mock(side_effect=Exception('set failed'))
redis_client.redis_store.incr = Mock(side_effect=Exception('inc failed'))
with pytest.raises(Exception) as e:
redis_client.get('test', raise_exception=True)
assert str(e.value) == 'get failed'
with pytest.raises(Exception) as e:
redis_client.set('test', 'test', raise_exception=True)
assert str(e.value) == 'set failed'
with pytest.raises(Exception) as e:
redis_client.incr('test', raise_exception=True)
assert str(e.value) == 'inc failed'
def test_should_not_call_set_if_not_enabled(disabled_redis_client):
assert not disabled_redis_client.set('key', 'value')
disabled_redis_client.redis_store.set.assert_not_called()
def test_should_call_set_if_enabled(enabled_redis_client):
enabled_redis_client.set('key', 'value')
enabled_redis_client.redis_store.set.assert_called_with('key', 'value', None, None, False, False)
def test_should_not_call_get_if_not_enabled(disabled_redis_client):
disabled_redis_client.set('key', 'value')
disabled_redis_client.redis_store.get.assert_not_called()
def test_should_call_get_if_enabled(enabled_redis_client):
assert enabled_redis_client.get('key') == 100
enabled_redis_client.redis_store.get.assert_called_with('key')
def test_should_build_cache_key_service_and_action(sample_service):
with freeze_time("2016-01-01 12:00:00.000000"):
assert daily_limit_cache_key(sample_service.id) == '{}-2016-01-01-count'.format(sample_service.id)

17
tests/app/notifications/rest/test_send_notification.py
View File

@@ -343,7 +343,7 @@ def test_should_allow_valid_email_notification(notify_api, sample_email_template
@freeze_time("2016-01-01 12:00:00.061258")
def test_should_not_block_api_call_if_over_day_limit_for_live_service(
def test_should_block_api_call_if_over_day_limit_for_live_service(
notify_db,
notify_db_session,
notify_api,
@@ -371,8 +371,7 @@ def test_should_not_block_api_call_if_over_day_limit_for_live_service(
data=json.dumps(data),
headers=[('Content-Type', 'application/json'), auth_header])
json.loads(response.get_data(as_text=True))
assert response.status_code == 201
assert response.status_code == 429
@freeze_time("2016-01-01 12:00:00.061258")
@@ -698,15 +697,13 @@ def test_should_delete_notification_and_return_error_if_sqs_fails(
save_model_api_key(api_key)
auth_header = create_jwt_token(secret=api_key.unsigned_secret, client_id=str(api_key.service_id))
with pytest.raises(Exception) as exc:
response = client.post(
path='/notifications/{}'.format(template_type),
data=json.dumps(data),
headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))])
response = client.post(
path='/notifications/{}'.format(template_type),
data=json.dumps(data),
headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))])
mocked.assert_called_once_with([fake_uuid], queue='send-{}'.format(template_type))
assert str(exc.value) == 'failed to talk to SQS'
assert response.status_code == 500
assert not notifications_dao.get_notification_by_id(fake_uuid)
assert not NotificationHistory.query.get(fake_uuid)

49
tests/app/notifications/test_process_notification.py
View File

@@ -3,8 +3,10 @@ import datetime
import pytest
from boto3.exceptions import Boto3Error
from sqlalchemy.exc import SQLAlchemyError
from freezegun import freeze_time
from app.models import Template, Notification, NotificationHistory
from app.notifications import SendNotificationToQueueError
from app.notifications.process_notifications import (create_content_for_notification,
persist_notification, send_notification_to_queue)
from app.v2.errors import BadRequestError
@@ -37,7 +39,10 @@ def test_create_content_for_notification_fails_with_additional_personalisation(s
assert e.value.message == 'Template personalisation not needed for template: Additional placeholder'
def test_persist_notification_creates_and_save_to_db(sample_template, sample_api_key):
@freeze_time("2016-01-01 11:09:00.061258")
def test_persist_notification_creates_and_save_to_db(sample_template, sample_api_key, mocker):
mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.incr')
assert Notification.query.count() == 0
assert NotificationHistory.query.count() == 0
notification = persist_notification(sample_template.id, sample_template.version, '+447111111111',
@@ -46,6 +51,7 @@ def test_persist_notification_creates_and_save_to_db(sample_template, sample_api
assert Notification.query.count() == 1
assert Notification.query.get(notification.id) is not None
assert NotificationHistory.query.count() == 1
mocked_redis.assert_called_once_with(str(sample_template.service_id) + "-2016-01-01-count")
def test_persist_notification_throws_exception_when_missing_template(sample_api_key):
@@ -64,9 +70,45 @@ def test_persist_notification_throws_exception_when_missing_template(sample_api_
assert NotificationHistory.query.count() == 0
def test_persist_notification_with_optionals(sample_job, sample_api_key):
def test_exception_thown_by_redis_store_get_should_not_be_fatal(sample_template, sample_api_key, mocker):
mocker.patch(
'app.notifications.process_notifications.redis_store.redis_store.incr',
side_effect=Exception("broken redis"))
notification = persist_notification(
sample_template.id,
sample_template.version,
'+447111111111',
sample_template.service.id,
{},
'sms',
sample_api_key.id,
sample_api_key.key_type)
assert Notification.query.count() == 1
assert Notification.query.get(notification.id) is not None
assert NotificationHistory.query.count() == 1
def test_cache_is_not_incremented_on_failure_to_persist_notification(sample_api_key, mocker):
mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.incr')
with pytest.raises(SQLAlchemyError):
persist_notification(template_id=None,
template_version=None,
recipient='+447111111111',
service_id=sample_api_key.service_id,
personalisation=None,
notification_type='sms',
api_key_id=sample_api_key.id,
key_type=sample_api_key.key_type)
mocked_redis.assert_not_called()
@freeze_time("2016-01-01 11:09:00.061258")
def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker):
assert Notification.query.count() == 0
assert NotificationHistory.query.count() == 0
mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.incr')
created_at = datetime.datetime(2016, 11, 11, 16, 8, 18)
persist_notification(template_id=sample_job.template.id,
template_version=sample_job.template.version,
@@ -85,6 +127,7 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key):
assert persisted_notification.job_id == sample_job.id
assert persisted_notification.job_row_number == 10
assert persisted_notification.created_at == created_at
mocked_redis.assert_called_once_with(str(sample_job.service_id) + "-2016-01-01-count")
assert persisted_notification.client_reference == "ref from client"
assert persisted_notification.reference is None
@@ -111,7 +154,7 @@ def test_send_notification_to_queue(notify_db, notify_db_session,
def test_send_notification_to_queue_throws_exception_deletes_notification(sample_notification, mocker):
mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async', side_effect=Boto3Error("EXPECTED"))
with pytest.raises(Boto3Error):
with pytest.raises(SendNotificationToQueueError):
send_notification_to_queue(sample_notification, False)
mocked.assert_called_once_with([(str(sample_notification.id))], queue='send-sms')

149
tests/app/notifications/test_validators.py
View File

@@ -1,35 +1,146 @@
import pytest
from app.notifications.validators import check_service_message_limit, check_template_is_for_notification_type, \
check_template_is_active, service_can_send_to_recipient, check_sms_content_char_count
from app.v2.errors import BadRequestError, TooManyRequestsError
from tests.app.conftest import (sample_notification as create_notification,
sample_service as create_service, sample_service_whitelist)
from freezegun import freeze_time
@pytest.mark.parametrize('key_type', ['test', 'team', 'normal'])
def test_check_service_message_limit_with_unrestricted_service_passes(key_type,
sample_service,
sample_notification):
assert check_service_message_limit(key_type, sample_service) is None
@pytest.mark.parametrize('key_type', ['test', 'team', 'normal'])
def test_check_service_message_limit_under_message_limit_passes(key_type,
sample_service,
sample_notification):
assert check_service_message_limit(key_type, sample_service) is None
import app
from app.notifications.validators import (
check_service_message_limit,
check_template_is_for_notification_type,
check_template_is_active,
service_can_send_to_recipient,
check_sms_content_char_count
)
from app.v2.errors import (
BadRequestError,
TooManyRequestsError
)
from tests.app.conftest import (
sample_notification as create_notification,
sample_service as create_service,
sample_service_whitelist
)
@pytest.mark.parametrize('key_type', ['team', 'normal'])
def test_check_service_message_limit_over_message_limit_fails(key_type, notify_db, notify_db_session):
def test_exception_thown_by_redis_store_get_should_not_be_fatal(
notify_db,
notify_db_session,
notify_api,
key_type,
mocker):
mocker.patch('app.notifications.validators.redis_store.redis_store.get', side_effect=Exception("broken redis"))
mocker.patch('app.notifications.validators.redis_store.set')
service = create_service(notify_db, notify_db_session, restricted=True, limit=4)
for x in range(5):
create_notification(notify_db, notify_db_session, service=service)
with pytest.raises(TooManyRequestsError) as e:
check_service_message_limit(key_type, service)
assert e.value.status_code == 429
assert e.value.message == 'Exceeded send limits (4) for today'
assert e.value.fields == []
app.notifications.validators.redis_store.set.assert_not_called()
@pytest.mark.parametrize('key_type', ['test', 'team', 'normal'])
def test_exception_thown_by_redis_store_set_should_not_be_fatal(
key_type,
sample_service,
mocker):
mocker.patch('app.notifications.validators.redis_store.redis_store.set', side_effect=Exception("broken redis"))
mocker.patch('app.notifications.validators.redis_store.get', return_value=None)
assert not check_service_message_limit(key_type, sample_service)
@pytest.mark.parametrize('key_type', ['test', 'team', 'normal'])
def test_check_service_message_limit_in_cache_with_unrestricted_service_passes(
key_type,
sample_service,
mocker):
mocker.patch('app.notifications.validators.redis_store.get', return_value=1)
mocker.patch('app.notifications.validators.redis_store.set')
mocker.patch('app.notifications.validators.services_dao')
assert not check_service_message_limit(key_type, sample_service)
app.notifications.validators.redis_store.set.assert_not_called()
assert not app.notifications.validators.services_dao.mock_calls
@pytest.mark.parametrize('key_type', ['test', 'team', 'normal'])
def test_check_service_message_limit_in_cache_under_message_limit_passes(
key_type,
sample_service,
mocker):
mocker.patch('app.notifications.validators.redis_store.get', return_value=1)
mocker.patch('app.notifications.validators.redis_store.set')
mocker.patch('app.notifications.validators.services_dao')
assert not check_service_message_limit(key_type, sample_service)
app.notifications.validators.redis_store.set.assert_not_called()
assert not app.notifications.validators.services_dao.mock_calls
def test_should_not_interact_with_cache_for_test_key(sample_service, mocker):
mocker.patch('app.notifications.validators.redis_store')
assert not check_service_message_limit('test', sample_service)
assert not app.notifications.validators.redis_store.mock_calls
@pytest.mark.parametrize('key_type', ['team', 'normal'])
def test_should_set_cache_value_as_value_from_database_if_cache_not_set(
key_type,
notify_db,
notify_db_session,
sample_service,
mocker
):
with freeze_time("2016-01-01 12:00:00.000000"):
for x in range(5):
create_notification(notify_db, notify_db_session, service=sample_service)
mocker.patch('app.notifications.validators.redis_store.get', return_value=None)
mocker.patch('app.notifications.validators.redis_store.set')
assert not check_service_message_limit(key_type, sample_service)
app.notifications.validators.redis_store.set.assert_called_with(
str(sample_service.id) + "-2016-01-01-count", 5, ex=3600
)
@pytest.mark.parametrize('key_type', ['team', 'normal'])
def test_check_service_message_limit_over_message_limit_fails(key_type, notify_db, notify_db_session, mocker):
with freeze_time("2016-01-01 12:00:00.000000"):
mocker.patch('app.redis_store.get', return_value=None)
mocker.patch('app.notifications.validators.redis_store.set')
service = create_service(notify_db, notify_db_session, restricted=True, limit=4)
for x in range(5):
create_notification(notify_db, notify_db_session, service=service)
with pytest.raises(TooManyRequestsError) as e:
check_service_message_limit(key_type, service)
assert e.value.status_code == 429
assert e.value.message == 'Exceeded send limits (4) for today'
assert e.value.fields == []
app.notifications.validators.redis_store.set.assert_called_with(
str(service.id) + "-2016-01-01-count", 5, ex=3600
)
@pytest.mark.parametrize('key_type', ['team', 'normal'])
def test_check_service_message_limit_in_cache_over_message_limit_fails(
notify_db,
notify_db_session,
key_type,
mocker):
with freeze_time("2016-01-01 12:00:00.000000"):
mocker.patch('app.redis_store.get', return_value=5)
mocker.patch('app.notifications.validators.redis_store.set')
mocker.patch('app.notifications.validators.services_dao')
service = create_service(notify_db, notify_db_session, restricted=True, limit=4)
with pytest.raises(TooManyRequestsError) as e:
check_service_message_limit(key_type, service)
assert e.value.status_code == 429
assert e.value.message == 'Exceeded send limits (4) for today'
assert e.value.fields == []
app.notifications.validators.redis_store.set.assert_not_called()
assert not app.notifications.validators.services_dao.mock_calls
@pytest.mark.parametrize('template_type, notification_type',