Merge branch 'master' into stronger-2fa-security

app/config.py

@@ -49,6 +49,7 @@ class Config(object):
     # URL of redis instance
     REDIS_URL = os.getenv('REDIS_URL')
     REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
+    EXPIRE_CACHE_IN_SECONDS = 600
 
     # Performance platform
     PERFORMANCE_PLATFORM_ENABLED = os.getenv('PERFORMANCE_PLATFORM_ENABLED') == '1'

app/dao/templates_dao.py

@@ -1,6 +1,7 @@
 import uuid
 
-from sqlalchemy import (asc, desc)
+from sqlalchemy import desc
+from sqlalchemy.sql.expression import bindparam
 
 from app import db
 from app.models import (Template, TemplateHistory)
@@ -56,3 +57,26 @@ def dao_get_template_versions(service_id, template_id):
     ).order_by(
         desc(TemplateHistory.version)
     ).all()
+
+
+def dao_get_templates_for_cache(cache):
+    if not cache or len(cache) == 0:
+        return []
+
+    # First create a subquery that is a union select of the cache values
+    # Then join templates to the subquery
+    cache_queries = [
+        db.session.query(bindparam("template_id" + str(i),
+                                   uuid.UUID(template_id.decode())).label('template_id'),
+                         bindparam("count" + str(i), int(count.decode())).label('count'))
+        for i, (template_id, count) in enumerate(cache)]
+    cache_subq = cache_queries[0].union(*cache_queries[1:]).subquery()
+    query = db.session.query(Template.id.label('template_id'),
+                             Template.template_type,
+                             Template.name,
+                             cache_subq.c.count.label('count')
+                             ).join(cache_subq,
+                                    Template.id == cache_subq.c.template_id
+                                    ).order_by(Template.name)
+
+    return query.all()

app/notifications/process_notifications.py

@@ -8,7 +8,7 @@ from notifications_utils.clients import redis
 from app.dao.notifications_dao import dao_create_notification, dao_delete_notifications_and_history_by_id
 from app.models import SMS_TYPE, Notification, KEY_TYPE_TEST, EMAIL_TYPE
 from app.v2.errors import BadRequestError, SendNotificationToQueueError
-from app.utils import get_template_instance
+from app.utils import get_template_instance, cache_key_for_service_template_counter
 
 
 def create_content_for_notification(template, personalisation):
@@ -62,7 +62,10 @@ def persist_notification(template_id,
     )
     if not simulated:
         dao_create_notification(notification)
-        redis_store.incr(redis.daily_limit_cache_key(service.id))
+        if redis_store.get(redis.daily_limit_cache_key(service.id)):
+            redis_store.incr(redis.daily_limit_cache_key(service.id))
+        if redis_store.get_all_from_hash(cache_key_for_service_template_counter(service.id)):
+            redis_store.increment_hash_value(cache_key_for_service_template_counter(service.id), template_id)
         current_app.logger.info(
             "{} {} created at {}".format(notification.notification_type, notification.id, notification.created_at)
         )

app/template_statistics/rest.py

@@ -1,14 +1,17 @@
 from flask import (
     Blueprint,
     jsonify,
-    request
+    request,
-)
+    current_app)
 
+from app import redis_store
 from app.dao.notifications_dao import (
     dao_get_template_usage,
     dao_get_last_template_usage)
+from app.dao.templates_dao import dao_get_templates_for_cache
 
-from app.schemas import notifications_filter_schema, NotificationWithTemplateSchema, notification_with_template_schema
+from app.schemas import notification_with_template_schema
+from app.utils import cache_key_for_service_template_counter
 
 template_statistics = Blueprint('template-statistics',
                                 __name__,
@@ -30,7 +33,11 @@ def get_template_statistics_for_service_by_day(service_id):
             raise InvalidRequest(message, status_code=400)
     else:
         limit_days = None
-    stats = dao_get_template_usage(service_id, limit_days=limit_days)
+
+    if limit_days == 7:
+        stats = get_template_statistics_for_7_days(limit_days, service_id)
+    else:
+        stats = dao_get_template_usage(service_id, limit_days=limit_days)
 
     def serialize(data):
         return {
@@ -52,3 +59,15 @@ def get_template_statistics_for_template_id(service_id, template_id):
         raise InvalidRequest(errors, status_code=404)
     data = notification_with_template_schema.dump(notification).data
     return jsonify(data=data)
+
+
+def get_template_statistics_for_7_days(limit_days, service_id):
+    cache_key = cache_key_for_service_template_counter(service_id)
+    template_stats_by_id = redis_store.get_all_from_hash(cache_key)
+    if not template_stats_by_id:
+        stats = dao_get_template_usage(service_id, limit_days=limit_days)
+        cache_values = dict([(x.template_id, x.count) for x in stats])
+        redis_store.set_hash_and_expire(cache_key, cache_values, current_app.config.get('EXPIRE_CACHE_IN_SECONDS', 600))
+    else:
+        stats = dao_get_templates_for_cache(template_stats_by_id.items())
+    return stats

app/utils.py

@@ -63,3 +63,7 @@ def get_london_month_from_utc_column(column):
         "month",
         func.timezone("Europe/London", func.timezone("UTC", column))
     )
+
+
+def cache_key_for_service_template_counter(service_id, limit_days=7):
+    return "{}-template-counter-limit-{}-days".format(service_id, limit_days)

appspec.yml

@@ -5,6 +5,10 @@ files:
   - destination: /home/notify-app/notifications-api
     source: /
 hooks:
+  BeforeInstall:
+    - location: scripts/aws_clear_instance.sh
+      runas: root
+      timeout: 1000
   AfterInstall:
     - location: scripts/aws_install_dependencies.sh
       runas: root

scripts/aws_clear_instance.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+echo "Removing application and dependencies"
+
+if [ -d "/home/notify-app/notifications-api" ]; then
+    # Remove and re-create the directory
+    rm -rf /home/notify-app/notifications-api
+    mkdir -p /home/notify-app/notifications-api
+fi
+

tests/app/dao/test_templates_dao.py

@@ -4,7 +4,8 @@ from app.dao.templates_dao import (
     dao_get_template_by_id_and_service_id,
     dao_get_all_templates_for_service,
     dao_update_template,
-    dao_get_template_versions)
+    dao_get_template_versions,
+    dao_get_templates_for_cache)
 from tests.app.conftest import sample_template as create_sample_template
 from app.models import Template, TemplateHistory
 import pytest
@@ -265,3 +266,54 @@ def test_get_template_versions(sample_template):
     from app.schemas import template_history_schema
     v = template_history_schema.load(versions, many=True)
     assert len(v) == 2
+
+
+def test_get_templates_by_ids_successful(notify_db, notify_db_session):
+    template_1 = create_sample_template(
+        notify_db,
+        notify_db_session,
+        template_name='Sample Template 1',
+        template_type="sms",
+        content="Template content"
+    )
+    template_2 = create_sample_template(
+        notify_db,
+        notify_db_session,
+        template_name='Sample Template 2',
+        template_type="sms",
+        content="Template content"
+    )
+    create_sample_template(
+        notify_db,
+        notify_db_session,
+        template_name='Sample Template 3',
+        template_type="email",
+        content="Template content"
+    )
+    sample_cache_dict = {str.encode(str(template_1.id)): str.encode('2'),
+                         str.encode(str(template_2.id)): str.encode('3')}
+    cache = [[k, v] for k, v in sample_cache_dict.items()]
+    templates = dao_get_templates_for_cache(cache)
+    assert len(templates) == 2
+    assert [(template_1.id, template_1.template_type, template_1.name, 2),
+            (template_2.id, template_2.template_type, template_2.name, 3)] == templates
+
+
+def test_get_templates_by_ids_successful_for_one_cache_item(notify_db, notify_db_session):
+    template_1 = create_sample_template(
+        notify_db,
+        notify_db_session,
+        template_name='Sample Template 1',
+        template_type="sms",
+        content="Template content"
+    )
+    sample_cache_dict = {str.encode(str(template_1.id)): str.encode('2')}
+    cache = [[k, v] for k, v in sample_cache_dict.items()]
+    templates = dao_get_templates_for_cache(cache)
+    assert len(templates) == 1
+    assert [(template_1.id, template_1.template_type, template_1.name, 2)] == templates
+
+
+def test_get_templates_by_ids_returns_empty_list():
+        assert dao_get_templates_for_cache({}) == []
+        assert dao_get_templates_for_cache(None) == []

tests/app/notifications/rest/test_send_notification.py

@@ -670,42 +670,41 @@ def test_should_persist_notification(notify_api, sample_template,
 @pytest.mark.parametrize('template_type',
                          ['sms', 'email'])
 def test_should_delete_notification_and_return_error_if_sqs_fails(
-        notify_api,
+        client,
         sample_email_template,
         sample_template,
         fake_uuid,
         mocker,
         template_type):
-    with notify_api.test_request_context(), notify_api.test_client() as client:
+    mocked = mocker.patch(
-        mocked = mocker.patch(
+        'app.celery.provider_tasks.deliver_{}.apply_async'.format(template_type),
-            'app.celery.provider_tasks.deliver_{}.apply_async'.format(template_type),
+        side_effect=Exception("failed to talk to SQS")
-            side_effect=Exception("failed to talk to SQS")
+    )
-        )
+    mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
-        m1 = mocker.patch('app.dao.notifications_dao.create_uuid', return_value=fake_uuid)
+    template = sample_template if template_type == 'sms' else sample_email_template
-        template = sample_template if template_type == 'sms' else sample_email_template
+    to = sample_template.service.created_by.mobile_number if template_type == 'sms' \
-        to = sample_template.service.created_by.mobile_number if template_type == 'sms' \
+        else sample_email_template.service.created_by.email_address
-            else sample_email_template.service.created_by.email_address
+    data = {
-        data = {
+        'to': to,
-            'to': to,
+        'template': template.id
-            'template': template.id
+    }
-        }
+    api_key = ApiKey(
-        api_key = ApiKey(
+        service=template.service,
-            service=template.service,
+        name='team_key',
-            name='team_key',
+        created_by=template.created_by,
-            created_by=template.created_by,
+        key_type=KEY_TYPE_TEAM)
-            key_type=KEY_TYPE_TEAM)
+    save_model_api_key(api_key)
-        save_model_api_key(api_key)
+    auth_header = create_jwt_token(secret=api_key.unsigned_secret, client_id=str(api_key.service_id))
-        auth_header = create_jwt_token(secret=api_key.unsigned_secret, client_id=str(api_key.service_id))
 
-        response = client.post(
+    response = client.post(
-            path='/notifications/{}'.format(template_type),
+        path='/notifications/{}'.format(template_type),
-            data=json.dumps(data),
+        data=json.dumps(data),
-            headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))])
+        headers=[('Content-Type', 'application/json'), ('Authorization', 'Bearer {}'.format(auth_header))])
 
-        mocked.assert_called_once_with([fake_uuid], queue='send-{}'.format(template_type))
+    mocked.assert_called_once_with([fake_uuid], queue='send-{}'.format(template_type))
-        assert response.status_code == 500
+    assert response.status_code == 500
-        assert not notifications_dao.get_notification_by_id(fake_uuid)
+    assert not notifications_dao.get_notification_by_id(fake_uuid)
-        assert not NotificationHistory.query.get(fake_uuid)
+    assert not NotificationHistory.query.get(fake_uuid)
 
 
 @pytest.mark.parametrize('to_email', [

tests/app/notifications/test_process_notification.py

@@ -13,6 +13,7 @@ from app.notifications.process_notifications import (create_content_for_notifica
                                                      persist_notification,
                                                      send_notification_to_queue,
                                                      simulated_recipient)
+from app.utils import cache_key_for_service_template_counter
 from app.v2.errors import BadRequestError
 
 
@@ -44,7 +45,7 @@ def test_create_content_for_notification_fails_with_additional_personalisation(s
 
 @freeze_time("2016-01-01 11:09:00.061258")
 def test_persist_notification_creates_and_save_to_db(sample_template, sample_api_key, sample_job, mocker):
-    mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.incr')
+    mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.get')
 
     assert Notification.query.count() == 0
     assert NotificationHistory.query.count() == 0
@@ -114,7 +115,8 @@ def test_exception_thown_by_redis_store_get_should_not_be_fatal(sample_template,
 
 
 def test_cache_is_not_incremented_on_failure_to_persist_notification(sample_api_key, mocker):
-    mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.incr')
+    mocked_redis = mocker.patch('app.redis_store.get')
+    mock_service_template_cache = mocker.patch('app.redis_store.get_all_from_hash')
     with pytest.raises(SQLAlchemyError):
         persist_notification(template_id=None,
                              template_version=None,
@@ -125,13 +127,16 @@ def test_cache_is_not_incremented_on_failure_to_persist_notification(sample_api_
                              api_key_id=sample_api_key.id,
                              key_type=sample_api_key.key_type)
     mocked_redis.assert_not_called()
+    mock_service_template_cache.assert_not_called()
 
 
 @freeze_time("2016-01-01 11:09:00.061258")
 def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker):
     assert Notification.query.count() == 0
     assert NotificationHistory.query.count() == 0
-    mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.incr')
+    mocked_redis = mocker.patch('app.notifications.process_notifications.redis_store.get')
+    mock_service_template_cache = mocker.patch(
+        'app.notifications.process_notifications.redis_store.get_all_from_hash')
     n_id = uuid.uuid4()
     created_at = datetime.datetime(2016, 11, 11, 16, 8, 18)
     persist_notification(template_id=sample_job.template.id,
@@ -154,10 +159,33 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key, mocker)
     assert persisted_notification.job_row_number == 10
     assert persisted_notification.created_at == created_at
     mocked_redis.assert_called_once_with(str(sample_job.service_id) + "-2016-01-01-count")
+    mock_service_template_cache.assert_called_once_with(cache_key_for_service_template_counter(sample_job.service_id))
     assert persisted_notification.client_reference == "ref from client"
     assert persisted_notification.reference is None
 
 
+@freeze_time("2016-01-01 11:09:00.061258")
+def test_persist_notification_increments_cache_if_key_exists(sample_template, sample_api_key, mocker):
+    mock_incr = mocker.patch('app.notifications.process_notifications.redis_store.incr')
+    mock_incr_hash_value = mocker.patch('app.notifications.process_notifications.redis_store.increment_hash_value')
+
+    persist_notification(sample_template.id, sample_template.version, '+447111111111',
+                         sample_template.service, {}, 'sms', sample_api_key.id,
+                         sample_api_key.key_type, reference="ref")
+    mock_incr.assert_not_called()
+    mock_incr_hash_value.assert_not_called()
+
+    mocker.patch('app.notifications.process_notifications.redis_store.get', return_value=1)
+    mocker.patch('app.notifications.process_notifications.redis_store.get_all_from_hash',
+                 return_value={sample_template.id, 1})
+    persist_notification(sample_template.id, sample_template.version, '+447111111122',
+                         sample_template.service, {}, 'sms', sample_api_key.id,
+                         sample_api_key.key_type, reference="ref2")
+    mock_incr.assert_called_once_with(str(sample_template.service_id) + "-2016-01-01-count", )
+    mock_incr_hash_value.assert_called_once_with(cache_key_for_service_template_counter(sample_template.service_id),
+                                                 sample_template.id)
+
+
 @pytest.mark.parametrize('research_mode, requested_queue, expected_queue, notification_type, key_type',
                          [(True, None, 'research-mode', 'sms', 'normal'),
                           (True, None, 'research-mode', 'email', 'normal'),

tests/app/template_statistics/test_rest.py

@@ -1,222 +1,227 @@
 from datetime import datetime, timedelta
 import json
 
+import pytest
 from freezegun import freeze_time
 
 from tests import create_authorization_header
-from tests.app.conftest import sample_template as create_sample_template, sample_template, sample_notification, \
+from tests.app.conftest import (sample_template as create_sample_template, sample_notification, sample_email_template)
-    sample_email_template
 
 
-def test_get_all_template_statistics_with_bad_arg_returns_400(notify_api, sample_service):
+def test_get_all_template_statistics_with_bad_arg_returns_400(client, sample_service):
-    with notify_api.test_request_context():
+    auth_header = create_authorization_header()
-        with notify_api.test_client() as client:
-            auth_header = create_authorization_header()
 
-            response = client.get(
+    response = client.get(
-                '/service/{}/template-statistics'.format(sample_service.id),
+        '/service/{}/template-statistics'.format(sample_service.id),
-                headers=[('Content-Type', 'application/json'), auth_header],
+        headers=[('Content-Type', 'application/json'), auth_header],
-                query_string={'limit_days': 'blurk'}
+        query_string={'limit_days': 'blurk'}
-            )
+    )
 
-            assert response.status_code == 400
+    assert response.status_code == 400
-            json_resp = json.loads(response.get_data(as_text=True))
+    json_resp = json.loads(response.get_data(as_text=True))
-            assert json_resp['result'] == 'error'
+    assert json_resp['result'] == 'error'
-            assert json_resp['message'] == {'limit_days': ['blurk is not an integer']}
+    assert json_resp['message'] == {'limit_days': ['blurk is not an integer']}
 
 
 @freeze_time('2016-08-18')
-def test_get_template_statistics_for_service(notify_db, notify_db_session, notify_api, sample_service):
+def test_get_template_statistics_for_service(notify_db, notify_db_session, client, mocker):
-    sms = sample_template(notify_db, notify_db_session, service=sample_service)
+    email, sms = set_up_notifications(notify_db, notify_db_session)
-    email = sample_email_template(notify_db, notify_db_session, service=sample_service)
-    today = datetime.now()
-    sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=sms)
-    sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=sms)
-    sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=email)
-    sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=email)
 
-    with notify_api.test_request_context():
+    mocked_redis = mocker.patch('app.redis_store.get_all_from_hash')
-        with notify_api.test_client() as client:
-            auth_header = create_authorization_header()
 
-            response = client.get(
+    auth_header = create_authorization_header()
-                '/service/{}/template-statistics'.format(sample_service.id),
-                headers=[('Content-Type', 'application/json'), auth_header]
-            )
 
-            assert response.status_code == 200
+    response = client.get(
-            json_resp = json.loads(response.get_data(as_text=True))
+        '/service/{}/template-statistics'.format(email.service_id),
-            assert len(json_resp['data']) == 2
+        headers=[('Content-Type', 'application/json'), auth_header]
-            assert json_resp['data'][0]['count'] == 2
+    )
-            assert json_resp['data'][0]['template_id'] == str(email.id)
+
-            assert json_resp['data'][0]['template_name'] == email.name
+    assert response.status_code == 200
-            assert json_resp['data'][0]['template_type'] == email.template_type
+    json_resp = json.loads(response.get_data(as_text=True))
-            assert json_resp['data'][1]['count'] == 2
+    assert len(json_resp['data']) == 2
-            assert json_resp['data'][1]['template_id'] == str(sms.id)
+    assert json_resp['data'][0]['count'] == 3
-            assert json_resp['data'][1]['template_name'] == sms.name
+    assert json_resp['data'][0]['template_id'] == str(email.id)
-            assert json_resp['data'][1]['template_type'] == sms.template_type
+    assert json_resp['data'][0]['template_name'] == email.name
+    assert json_resp['data'][0]['template_type'] == email.template_type
+    assert json_resp['data'][1]['count'] == 3
+    assert json_resp['data'][1]['template_id'] == str(sms.id)
+    assert json_resp['data'][1]['template_name'] == sms.name
+    assert json_resp['data'][1]['template_type'] == sms.template_type
+
+    mocked_redis.assert_not_called()
 
 
 @freeze_time('2016-08-18')
-def test_get_template_statistics_for_service_limited_by_day(notify_db, notify_db_session, notify_api, sample_service):
+def test_get_template_statistics_for_service_limited_1_day(notify_db, notify_db_session, client,
-    sms = sample_template(notify_db, notify_db_session, service=sample_service)
+                                                           mocker):
-    email = sample_email_template(notify_db, notify_db_session, service=sample_service)
+    email, sms = set_up_notifications(notify_db, notify_db_session)
+    mock_redis = mocker.patch('app.redis_store.get_all_from_hash')
+
+    auth_header = create_authorization_header()
+
+    response = client.get(
+        '/service/{}/template-statistics'.format(email.service_id),
+        headers=[('Content-Type', 'application/json'), auth_header],
+        query_string={'limit_days': 1}
+    )
+
+    assert response.status_code == 200
+    json_resp = json.loads(response.get_data(as_text=True))['data']
+    assert len(json_resp) == 2
+
+    assert json_resp[0]['count'] == 1
+    assert json_resp[0]['template_id'] == str(email.id)
+    assert json_resp[0]['template_name'] == email.name
+    assert json_resp[0]['template_type'] == email.template_type
+    assert json_resp[1]['count'] == 1
+    assert json_resp[1]['template_id'] == str(sms.id)
+    assert json_resp[1]['template_name'] == sms.name
+    assert json_resp[1]['template_type'] == sms.template_type
+
+    mock_redis.assert_not_called()
+
+
+@pytest.mark.parametrize("cache_values", [False, True])
+@freeze_time('2016-08-18')
+def test_get_template_statistics_for_service_limit_7_days(notify_db, notify_db_session, client,
+                                                          mocker,
+                                                          cache_values):
+    email, sms = set_up_notifications(notify_db, notify_db_session)
+    mock_cache_values = {str.encode(str(sms.id)): str.encode('2'),
+                         str.encode(str(email.id)): str.encode('2')} if cache_values else None
+    mocked_redis_get = mocker.patch('app.redis_store.get_all_from_hash', return_value=mock_cache_values)
+    mocked_redis_set = mocker.patch('app.redis_store.set_hash_and_expire')
+
+    auth_header = create_authorization_header()
+    response_for_a_week = client.get(
+        '/service/{}/template-statistics'.format(email.service_id),
+        headers=[('Content-Type', 'application/json'), auth_header],
+        query_string={'limit_days': 7}
+    )
+
+    assert response_for_a_week.status_code == 200
+    json_resp = json.loads(response_for_a_week.get_data(as_text=True))
+    assert len(json_resp['data']) == 2
+    assert json_resp['data'][0]['count'] == 2
+    assert json_resp['data'][0]['template_name'] == 'Email Template Name'
+    assert json_resp['data'][1]['count'] == 2
+    assert json_resp['data'][1]['template_name'] == 'Template Name'
+
+    mocked_redis_get.assert_called_once_with("{}-template-counter-limit-7-days".format(email.service_id))
+    if cache_values:
+        mocked_redis_set.assert_not_called()
+    else:
+        mocked_redis_set.assert_called_once_with("{}-template-counter-limit-7-days".format(email.service_id),
+                                                 {sms.id: 2, email.id: 2}, 600)
+
+
+@freeze_time('2016-08-18')
+def test_get_template_statistics_for_service_limit_30_days(notify_db, notify_db_session, client,
+                                                           mocker):
+    email, sms = set_up_notifications(notify_db, notify_db_session)
+    mock_redis = mocker.patch('app.redis_store.get_all_from_hash')
+
+    auth_header = create_authorization_header()
+
+    response_for_a_month = client.get(
+        '/service/{}/template-statistics'.format(email.service_id),
+        headers=[('Content-Type', 'application/json'), auth_header],
+        query_string={'limit_days': 30}
+    )
+
+    assert response_for_a_month.status_code == 200
+    json_resp = json.loads(response_for_a_month.get_data(as_text=True))
+    assert len(json_resp['data']) == 2
+    assert json_resp['data'][0]['count'] == 3
+    assert json_resp['data'][0]['template_name'] == 'Email Template Name'
+    assert json_resp['data'][1]['count'] == 3
+    assert json_resp['data'][1]['template_name'] == 'Template Name'
+
+    mock_redis.assert_not_called()
+
+
+@freeze_time('2016-08-18')
+def test_get_template_statistics_for_service_no_limit(notify_db, notify_db_session, client,
+                                                      mocker):
+    email, sms = set_up_notifications(notify_db, notify_db_session)
+    mock_redis = mocker.patch('app.redis_store.get_all_from_hash')
+    auth_header = create_authorization_header()
+    response_for_all = client.get(
+        '/service/{}/template-statistics'.format(email.service_id),
+        headers=[('Content-Type', 'application/json'), auth_header]
+    )
+    assert response_for_all.status_code == 200
+    json_resp = json.loads(response_for_all.get_data(as_text=True))
+    assert len(json_resp['data']) == 2
+    assert json_resp['data'][0]['count'] == 3
+    assert json_resp['data'][0]['template_name'] == 'Email Template Name'
+    assert json_resp['data'][1]['count'] == 3
+    assert json_resp['data'][1]['template_name'] == 'Template Name'
+
+    mock_redis.assert_not_called()
+
+
+def set_up_notifications(notify_db, notify_db_session):
+    sms = create_sample_template(notify_db, notify_db_session)
+    email = sample_email_template(notify_db, notify_db_session)
     today = datetime.now()
     a_week_ago = datetime.now() - timedelta(days=7)
     a_month_ago = datetime.now() - timedelta(days=30)
-    sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=sms)
+    sample_notification(notify_db, notify_db_session, created_at=today, template=sms)
-    sample_notification(notify_db, notify_db_session, created_at=today, service=sample_service, template=email)
+    sample_notification(notify_db, notify_db_session, created_at=today, template=email)
-    sample_notification(notify_db, notify_db_session, created_at=a_week_ago, service=sample_service, template=sms)
+    sample_notification(notify_db, notify_db_session, created_at=a_week_ago, template=sms)
-    sample_notification(notify_db, notify_db_session, created_at=a_week_ago, service=sample_service, template=email)
+    sample_notification(notify_db, notify_db_session, created_at=a_week_ago, template=email)
-    sample_notification(notify_db, notify_db_session, created_at=a_month_ago, service=sample_service, template=sms)
+    sample_notification(notify_db, notify_db_session, created_at=a_month_ago, template=sms)
-    sample_notification(notify_db, notify_db_session, created_at=a_month_ago, service=sample_service, template=email)
+    sample_notification(notify_db, notify_db_session, created_at=a_month_ago, template=email)
-
+    return email, sms
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            auth_header = create_authorization_header()
-
-            response = client.get(
-                '/service/{}/template-statistics'.format(sample_service.id),
-                headers=[('Content-Type', 'application/json'), auth_header],
-                query_string={'limit_days': 1}
-            )
-
-            assert response.status_code == 200
-            json_resp = json.loads(response.get_data(as_text=True))
-            assert len(json_resp['data']) == 2
-            assert json_resp['data'][0]['count'] == 1
-            assert json_resp['data'][0]['template_id'] == str(email.id)
-            assert json_resp['data'][0]['template_name'] == email.name
-            assert json_resp['data'][0]['template_type'] == email.template_type
-            assert json_resp['data'][1]['count'] == 1
-            assert json_resp['data'][1]['template_id'] == str(sms.id)
-            assert json_resp['data'][1]['template_name'] == sms.name
-            assert json_resp['data'][1]['template_type'] == sms.template_type
-
-            response_for_a_week = client.get(
-                '/service/{}/template-statistics'.format(sample_service.id),
-                headers=[('Content-Type', 'application/json'), auth_header],
-                query_string={'limit_days': 7}
-            )
-
-            assert response.status_code == 200
-            json_resp = json.loads(response_for_a_week.get_data(as_text=True))
-            assert len(json_resp['data']) == 2
-            assert json_resp['data'][0]['count'] == 2
-            assert json_resp['data'][0]['template_name'] == 'Email Template Name'
-            assert json_resp['data'][1]['count'] == 2
-            assert json_resp['data'][1]['template_name'] == 'Template Name'
-
-            response_for_a_month = client.get(
-                '/service/{}/template-statistics'.format(sample_service.id),
-                headers=[('Content-Type', 'application/json'), auth_header],
-                query_string={'limit_days': 30}
-            )
-
-            assert response_for_a_month.status_code == 200
-            json_resp = json.loads(response_for_a_month.get_data(as_text=True))
-            assert len(json_resp['data']) == 2
-            assert json_resp['data'][0]['count'] == 3
-            assert json_resp['data'][0]['template_name'] == 'Email Template Name'
-            assert json_resp['data'][1]['count'] == 3
-            assert json_resp['data'][1]['template_name'] == 'Template Name'
-
-            response_for_all = client.get(
-                '/service/{}/template-statistics'.format(sample_service.id),
-                headers=[('Content-Type', 'application/json'), auth_header]
-            )
-
-            assert response_for_all.status_code == 200
-            json_resp = json.loads(response_for_all.get_data(as_text=True))
-            assert len(json_resp['data']) == 2
-            assert json_resp['data'][0]['count'] == 3
-            assert json_resp['data'][0]['template_name'] == 'Email Template Name'
-            assert json_resp['data'][1]['count'] == 3
-            assert json_resp['data'][1]['template_name'] == 'Template Name'
 
 
 @freeze_time('2016-08-18')
-def test_returns_empty_list_if_no_templates_used(notify_api, sample_service):
+def test_returns_empty_list_if_no_templates_used(client, sample_service):
-    with notify_api.test_request_context():
+    auth_header = create_authorization_header()
-        with notify_api.test_client() as client:
-            auth_header = create_authorization_header()
 
-            response = client.get(
+    response = client.get(
-                '/service/{}/template-statistics'.format(sample_service.id),
+        '/service/{}/template-statistics'.format(sample_service.id),
-                headers=[('Content-Type', 'application/json'), auth_header]
+        headers=[('Content-Type', 'application/json'), auth_header]
-            )
+    )
 
-            assert response.status_code == 200
+    assert response.status_code == 200
-            json_resp = json.loads(response.get_data(as_text=True))
+    json_resp = json.loads(response.get_data(as_text=True))
-            assert len(json_resp['data']) == 0
+    assert len(json_resp['data']) == 0
 
 
 def test_get_template_statistics_by_id_returns_last_notification(
         notify_db,
         notify_db_session,
-        notify_api,
+        client):
-        sample_service):
+    sample_notification(notify_db, notify_db_session)
+    sample_notification(notify_db, notify_db_session)
+    notification_3 = sample_notification(notify_db, notify_db_session)
 
-    template = create_sample_template(
+    auth_header = create_authorization_header()
-        notify_db,
+
-        notify_db_session,
+    response = client.get(
-        template_name='Sample Template 1',
+        '/service/{}/template-statistics/{}'.format(notification_3.service_id, notification_3.template_id),
-        service=sample_service
+        headers=[('Content-Type', 'application/json'), auth_header],
     )
 
-    notification_1 = sample_notification(
+    assert response.status_code == 200
-        notify_db,
+    json_resp = json.loads(response.get_data(as_text=True))['data']
-        notify_db_session,
+    assert json_resp['id'] == str(notification_3.id)
-        service=sample_service,
-        template=template)
-    notification_2 = sample_notification(
-        notify_db,
-        notify_db_session,
-        service=sample_service,
-        template=template)
-    notification_3 = sample_notification(
-        notify_db,
-        notify_db_session,
-        service=sample_service,
-        template=template)
-
-    with notify_api.test_request_context():
-        with notify_api.test_client() as client:
-            auth_header = create_authorization_header()
-
-            response = client.get(
-                '/service/{}/template-statistics/{}'.format(sample_service.id, template.id),
-                headers=[('Content-Type', 'application/json'), auth_header],
-            )
-
-            assert response.status_code == 200
-            json_resp = json.loads(response.get_data(as_text=True))['data']
-            assert json_resp['id'] == str(notification_3.id)
 
 
 def test_get_template_statistics_for_template_returns_empty_if_no_statistics(
-        notify_db,
+        client,
-        notify_db_session,
+        sample_template,
-        notify_api,
-        sample_service
 ):
-    template = create_sample_template(
+    auth_header = create_authorization_header()
-        notify_db,
+
-        notify_db_session,
+    response = client.get(
-        template_name='Sample Template 1',
+        '/service/{}/template-statistics/{}'.format(sample_template.service_id, sample_template.id),
-        service=sample_service
+        headers=[('Content-Type', 'application/json'), auth_header],
     )
 
-    with notify_api.test_request_context():
+    assert response.status_code == 404
-        with notify_api.test_client() as client:
+    json_resp = json.loads(response.get_data(as_text=True))
-            auth_header = create_authorization_header()
+    assert json_resp['result'] == 'error'
-
+    assert json_resp['message']['template_id'] == ['No template found for id {}'.format(sample_template.id)]
-            response = client.get(
-                '/service/{}/template-statistics/{}'.format(sample_service.id, template.id),
-                headers=[('Content-Type', 'application/json'), auth_header],
-            )
-
-            assert response.status_code == 404
-            json_resp = json.loads(response.get_data(as_text=True))
-            assert json_resp['result'] == 'error'
-            assert json_resp['message']['template_id'] == ['No template found for id {}'.format(template.id)]