Merge branch 'master' of https://github.com/alphagov/notifications-api into vb-callback-receipt-1

app/config.py

@@ -388,6 +388,7 @@ class Staging(Config):
     FROM_NUMBER = 'stage'
     API_RATE_LIMIT_ENABLED = True
     CHECK_PROXY_HEADER = True
+    REDIS_ENABLED = False
 
     API_KEY_LIMITS = {
         KEY_TYPE_TEAM: {
@@ -416,7 +417,7 @@ class Live(Config):
     FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID = 'ba9e1789-a804-40b8-871f-cc60d4c1286f'
     PERFORMANCE_PLATFORM_ENABLED = True
     API_RATE_LIMIT_ENABLED = True
-    CHECK_PROXY_HEADER = False
+    CHECK_PROXY_HEADER = True
 
 
 class CloudFoundryConfig(Config):

app/models.py

@@ -178,6 +178,7 @@ INTERNATIONAL_SMS_TYPE = 'international_sms'
 INBOUND_SMS_TYPE = 'inbound_sms'
 SCHEDULE_NOTIFICATIONS = 'schedule_notifications'
 EMAIL_AUTH = 'email_auth'
+LETTERS_AS_PDF = 'letters_as_pdf'
 
 SERVICE_PERMISSION_TYPES = [
     EMAIL_TYPE,
@@ -187,6 +188,7 @@ SERVICE_PERMISSION_TYPES = [
     INBOUND_SMS_TYPE,
     SCHEDULE_NOTIFICATIONS,
     EMAIL_AUTH,
+    LETTERS_AS_PDF,
 ]
 
 

app/notifications/process_notifications.py

@@ -13,6 +13,7 @@ from notifications_utils.recipients import (
 from app import redis_store
 from app.celery import provider_tasks
 from app.config import QueueNames
+
 from app.models import (
     EMAIL_TYPE,
     KEY_TYPE_TEST,
@@ -26,6 +27,8 @@ from app.dao.notifications_dao import (
     dao_delete_notifications_and_history_by_id,
     dao_created_scheduled_notification
 )
+
+from app.statsd_decorators import statsd
 from app.v2.errors import BadRequestError
 from app.utils import get_template_instance, cache_key_for_service_template_counter, convert_bst_to_utc
 
@@ -43,6 +46,7 @@ def check_placeholders(template_object):
         raise BadRequestError(fields=[{'template': message}], message=message)
 
 
+@statsd(namespace="performance-testing")
 def persist_notification(
     *,
     template_id,
@@ -112,6 +116,7 @@ def persist_notification(
     return notification
 
 
+@statsd(namespace="performance-testing")
 def send_notification_to_queue(notification, research_mode, queue=None):
     if research_mode or notification.key_type == KEY_TYPE_TEST:
         queue = QueueNames.RESEARCH_MODE

app/notifications/validators.py

@@ -14,6 +14,7 @@ from app.models import (
     KEY_TYPE_TEST, KEY_TYPE_TEAM, SCHEDULE_NOTIFICATIONS
 )
 from app.service.utils import service_allowed_to_send_to
+from app.statsd_decorators import statsd
 from app.v2.errors import TooManyRequestsError, BadRequestError, RateLimitError
 from app import redis_store
 from app.notifications.process_notifications import create_content_for_notification
@@ -22,7 +23,7 @@ from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 
 
 def check_service_over_api_rate_limit(service, api_key):
-    if current_app.config['API_RATE_LIMIT_ENABLED']:
+    if current_app.config['API_RATE_LIMIT_ENABLED'] and current_app.config['REDIS_ENABLED']:
         cache_key = rate_limit_cache_key(service.id, api_key.key_type)
         rate_limit = current_app.config['API_KEY_LIMITS'][api_key.key_type]['limit']
         interval = current_app.config['API_KEY_LIMITS'][api_key.key_type]['interval']
@@ -32,7 +33,7 @@ def check_service_over_api_rate_limit(service, api_key):
 
 
 def check_service_over_daily_message_limit(key_type, service):
-    if key_type != KEY_TYPE_TEST:
+    if key_type != KEY_TYPE_TEST and current_app.config['REDIS_ENABLED']:
         cache_key = daily_limit_cache_key(service.id)
         service_stats = redis_store.get(cache_key)
         if not service_stats:
@@ -46,6 +47,7 @@ def check_service_over_daily_message_limit(key_type, service):
             raise TooManyRequestsError(service.message_limit)
 
 
+@statsd(namespace="performance-testing")
 def check_rate_limiting(service, api_key):
     check_service_over_api_rate_limit(service, api_key)
     check_service_over_daily_message_limit(api_key.key_type, service)
@@ -80,12 +82,14 @@ def service_has_permission(notify_type, permissions):
     return notify_type in [p.permission for p in permissions]
 
 
+@statsd(namespace="performance-testing")
 def check_service_has_permission(notify_type, permissions):
     if not service_has_permission(notify_type, permissions):
         raise BadRequestError(message="Cannot send {}".format(
             get_public_notify_type_text(notify_type, plural=True)))
 
 
+@statsd(namespace="performance-testing")
 def check_service_can_schedule_notification(permissions, scheduled_for):
     if scheduled_for:
         if not service_has_permission(SCHEDULE_NOTIFICATIONS, permissions):
@@ -117,6 +121,7 @@ def check_sms_content_char_count(content_count):
         raise BadRequestError(message=message)
 
 
+@statsd(namespace="performance-testing")
 def validate_template(template_id, personalisation, service, notification_type):
     try:
         template = templates_dao.dao_get_template_by_id_and_service_id(

app/v2/notifications/create_response.py

@@ -1,3 +1,5 @@
+from app.statsd_decorators import statsd
+
 
 def create_post_sms_response_from_notification(notification, content, from_number, url_root, scheduled_for):
     noti = __create_notification_response(notification, url_root, scheduled_for)
@@ -8,6 +10,7 @@ def create_post_sms_response_from_notification(notification, content, from_numbe
     return noti
 
 
+@statsd(namespace="performance-testing")
 def create_post_email_response_from_notification(notification, content, subject, email_from, url_root, scheduled_for):
     noti = __create_notification_response(notification, url_root, scheduled_for)
     noti['content'] = {

app/v2/notifications/post_notifications.py

@@ -34,6 +34,7 @@ from app.notifications.validators import (
     check_service_sms_sender_id
 )
 from app.schema_validation import validate
+from app.statsd_decorators import statsd
 from app.v2.errors import BadRequestError
 from app.v2.notifications import v2_notification_blueprint
 from app.v2.notifications.notification_schemas import (
@@ -49,6 +50,7 @@ from app.v2.notifications.create_response import (
 
 
 @v2_notification_blueprint.route('/<notification_type>', methods=['POST'])
+@statsd(namespace="performance-testing")
 def post_notification(notification_type):
     if notification_type == EMAIL_TYPE:
         form = validate(request.get_json(), post_email_request)
@@ -118,6 +120,7 @@ def post_notification(notification_type):
     return jsonify(resp), 201
 
 
+@statsd(namespace="performance-testing")
 def process_sms_or_email_notification(*, form, notification_type, api_key, template, service, reply_to_text=None):
     form_send_to = form['email_address'] if notification_type == EMAIL_TYPE else form['phone_number']
 
@@ -187,6 +190,7 @@ def process_letter_notification(*, letter_data, api_key, template):
     return notification
 
 
+@statsd(namespace="performance-testing")
 def get_reply_to_text(notification_type, form):
     reply_to = None
     if notification_type == EMAIL_TYPE:

migrations/versions/0148_add_letters_as_pdf_svc_perm.py

@@ -0,0 +1,24 @@
+"""empty message
+
+Revision ID: 0148_add_letters_as_pdf_svc_perm
+Revises: 0147_drop_mapping_tables
+Create Date: 2017-12-01 13:33:18.581320
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0148_add_letters_as_pdf_svc_perm'
+down_revision = '0147_drop_mapping_tables'
+
+from alembic import op
+
+
+def upgrade():
+    op.get_bind()
+    op.execute("insert into service_permission_types values('letters_as_pdf')")
+
+
+def downgrade():
+    op.get_bind()
+    op.execute("delete from service_permissions where permission = 'letters_as_pdf'")
+    op.execute("delete from service_permission_types where name = 'letters_as_pdf'")

requirements.txt

@@ -23,7 +23,7 @@ statsd==3.2.1
 notifications-python-client==4.6.0
 
 # PaaS
-awscli==1.14.1
+awscli==1.14.2
 awscli-cwlogs>=1.4,<1.5
 
 git+https://github.com/alphagov/notifications-utils.git@23.2.1#egg=notifications-utils==23.2.1

tests/app/notifications/rest/test_send_notification.py

@@ -16,7 +16,10 @@ from app.models import (
 from app.dao.templates_dao import dao_get_all_templates_for_service, dao_update_template
 from app.dao.services_dao import dao_update_service
 from app.dao.api_key_dao import save_model_api_key
-from app.v2.errors import RateLimitError
+from app.errors import InvalidRequest
+from app.models import Template
+from app.v2.errors import RateLimitError, TooManyRequestsError
+
 from tests import create_authorization_header
 from tests.app.conftest import (
     sample_notification as create_sample_notification,
@@ -28,9 +31,6 @@ from tests.app.conftest import (
     sample_service,
     sample_template_without_sms_permission,
     sample_template_without_email_permission)
-
-from app.models import Template
-from app.errors import InvalidRequest
 from tests.app.db import create_service, create_reply_to_email
 
 
@@ -414,6 +414,10 @@ def test_should_block_api_call_if_over_day_limit_for_live_service(
         mocker):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
+            mocker.patch(
+                'app.notifications.validators.check_service_over_daily_message_limit',
+                side_effect=TooManyRequestsError(1)
+            )
             mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
 
             service = create_sample_service(notify_db, notify_db_session, limit=1, restricted=False)
@@ -446,6 +450,10 @@ def test_should_block_api_call_if_over_day_limit_for_restricted_service(
     with notify_api.test_request_context():
         with notify_api.test_client() as client:
             mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
+            mocker.patch(
+                'app.notifications.validators.check_service_over_daily_message_limit',
+                side_effect=TooManyRequestsError(1)
+            )
 
             service = create_sample_service(notify_db, notify_db_session, limit=1, restricted=True)
             email_template = create_sample_email_template(notify_db, notify_db_session, service=service)

tests/app/notifications/test_validators.py

@@ -1,6 +1,7 @@
 import pytest
 from freezegun import freeze_time
 from flask import current_app
+
 import app
 from app.models import INTERNATIONAL_SMS_TYPE, SMS_TYPE, EMAIL_TYPE
 from app.notifications.validators import (
@@ -18,6 +19,8 @@ from app.v2.errors import (
     BadRequestError,
     TooManyRequestsError,
     RateLimitError)
+
+from tests.conftest import set_config
 from tests.app.conftest import (
     sample_notification as create_notification,
     sample_service as create_service,
@@ -26,6 +29,13 @@ from tests.app.conftest import (
 from tests.app.db import create_reply_to_email, create_service_sms_sender
 
 
+# all of these tests should have redis enabled (except where we specifically disable it)
+@pytest.fixture(scope='module', autouse=True)
+def enable_redis(notify_api):
+    with set_config(notify_api, 'REDIS_ENABLED', True):
+        yield
+
+
 @pytest.mark.parametrize('key_type', ['test', 'team', 'normal'])
 def test_check_service_message_limit_in_cache_with_unrestricted_service_is_allowed(
         key_type,
@@ -78,6 +88,15 @@ def test_should_set_cache_value_as_value_from_database_if_cache_not_set(
         )
 
 
+def test_should_not_access_database_if_redis_disabled(notify_api, sample_service, mocker):
+    with set_config(notify_api, 'REDIS_ENABLED', False):
+        db_mock = mocker.patch('app.notifications.validators.services_dao')
+
+        check_service_over_daily_message_limit('normal', sample_service)
+
+        assert db_mock.method_calls == []
+
+
 @pytest.mark.parametrize('key_type', ['team', 'normal'])
 def test_check_service_message_limit_over_message_limit_fails(key_type, notify_db, notify_db_session, mocker):
     with freeze_time("2016-01-01 12:00:00.000000"):

tests/app/service/test_rest.py

@@ -2163,18 +2163,17 @@ def test_search_for_notification_by_to_field_returns_content(
     assert notifications[0]['template']['content'] == 'Hello (( Name))\nYour thing is due soon'
 
 
-def test_send_one_off_notification(admin_request, mocker):
+def test_send_one_off_notification(sample_service, admin_request, mocker):
-    service = create_service()
+    template = create_template(service=sample_service)
-    template = create_template(service=service)
     mocker.patch('app.service.send_notification.send_notification_to_queue')
 
     response = admin_request.post(
         'service.create_one_off_notification',
-        service_id=service.id,
+        service_id=sample_service.id,
         _data={
             'template_id': str(template.id),
             'to': '07700900001',
-            'created_by': str(service.created_by_id)
+            'created_by': str(sample_service.created_by_id)
         },
         _expected_status=201
     )

tests/app/service/test_send_one_off_notification.py

@@ -151,9 +151,13 @@ def test_send_one_off_notification_raises_if_cant_send_to_recipient(notify_db_se
     assert 'service is in trial mode' in e.value.message
 
 
-def test_send_one_off_notification_raises_if_over_limit(notify_db_session):
+def test_send_one_off_notification_raises_if_over_limit(notify_db_session, mocker):
     service = create_service(message_limit=0)
     template = create_template(service=service)
+    mocker.patch(
+        'app.service.send_notification.check_service_over_daily_message_limit',
+        side_effect=TooManyRequestsError(1)
+    )
 
     post_data = {
         'template_id': str(template.id),