diff --git a/app/config.py b/app/config.py index 3199fbe93..1ba975619 100644 --- a/app/config.py +++ b/app/config.py @@ -388,6 +388,7 @@ class Staging(Config): FROM_NUMBER = 'stage' API_RATE_LIMIT_ENABLED = True CHECK_PROXY_HEADER = True + REDIS_ENABLED = False API_KEY_LIMITS = { KEY_TYPE_TEAM: { @@ -416,7 +417,7 @@ class Live(Config): FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID = 'ba9e1789-a804-40b8-871f-cc60d4c1286f' PERFORMANCE_PLATFORM_ENABLED = True API_RATE_LIMIT_ENABLED = True - CHECK_PROXY_HEADER = False + CHECK_PROXY_HEADER = True class CloudFoundryConfig(Config): diff --git a/app/models.py b/app/models.py index fb3a2610d..9d81073c7 100644 --- a/app/models.py +++ b/app/models.py @@ -178,6 +178,7 @@ INTERNATIONAL_SMS_TYPE = 'international_sms' INBOUND_SMS_TYPE = 'inbound_sms' SCHEDULE_NOTIFICATIONS = 'schedule_notifications' EMAIL_AUTH = 'email_auth' +LETTERS_AS_PDF = 'letters_as_pdf' SERVICE_PERMISSION_TYPES = [ EMAIL_TYPE, @@ -187,6 +188,7 @@ SERVICE_PERMISSION_TYPES = [ INBOUND_SMS_TYPE, SCHEDULE_NOTIFICATIONS, EMAIL_AUTH, + LETTERS_AS_PDF, ] diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index df46b0f02..1277d9921 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -13,6 +13,7 @@ from notifications_utils.recipients import ( from app import redis_store from app.celery import provider_tasks from app.config import QueueNames + from app.models import ( EMAIL_TYPE, KEY_TYPE_TEST, @@ -26,6 +27,8 @@ from app.dao.notifications_dao import ( dao_delete_notifications_and_history_by_id, dao_created_scheduled_notification ) + +from app.statsd_decorators import statsd from app.v2.errors import BadRequestError from app.utils import get_template_instance, cache_key_for_service_template_counter, convert_bst_to_utc @@ -43,6 +46,7 @@ def check_placeholders(template_object): raise BadRequestError(fields=[{'template': message}], message=message) +@statsd(namespace="performance-testing") def persist_notification( *, template_id, @@ -112,6 +116,7 @@ def persist_notification( return notification +@statsd(namespace="performance-testing") def send_notification_to_queue(notification, research_mode, queue=None): if research_mode or notification.key_type == KEY_TYPE_TEST: queue = QueueNames.RESEARCH_MODE diff --git a/app/notifications/validators.py b/app/notifications/validators.py index 344f05ebe..176241f1e 100644 --- a/app/notifications/validators.py +++ b/app/notifications/validators.py @@ -14,6 +14,7 @@ from app.models import ( KEY_TYPE_TEST, KEY_TYPE_TEAM, SCHEDULE_NOTIFICATIONS ) from app.service.utils import service_allowed_to_send_to +from app.statsd_decorators import statsd from app.v2.errors import TooManyRequestsError, BadRequestError, RateLimitError from app import redis_store from app.notifications.process_notifications import create_content_for_notification @@ -22,7 +23,7 @@ from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id def check_service_over_api_rate_limit(service, api_key): - if current_app.config['API_RATE_LIMIT_ENABLED']: + if current_app.config['API_RATE_LIMIT_ENABLED'] and current_app.config['REDIS_ENABLED']: cache_key = rate_limit_cache_key(service.id, api_key.key_type) rate_limit = current_app.config['API_KEY_LIMITS'][api_key.key_type]['limit'] interval = current_app.config['API_KEY_LIMITS'][api_key.key_type]['interval'] @@ -32,7 +33,7 @@ def check_service_over_api_rate_limit(service, api_key): def check_service_over_daily_message_limit(key_type, service): - if key_type != KEY_TYPE_TEST: + if key_type != KEY_TYPE_TEST and current_app.config['REDIS_ENABLED']: cache_key = daily_limit_cache_key(service.id) service_stats = redis_store.get(cache_key) if not service_stats: @@ -46,6 +47,7 @@ def check_service_over_daily_message_limit(key_type, service): raise TooManyRequestsError(service.message_limit) +@statsd(namespace="performance-testing") def check_rate_limiting(service, api_key): check_service_over_api_rate_limit(service, api_key) check_service_over_daily_message_limit(api_key.key_type, service) @@ -80,12 +82,14 @@ def service_has_permission(notify_type, permissions): return notify_type in [p.permission for p in permissions] +@statsd(namespace="performance-testing") def check_service_has_permission(notify_type, permissions): if not service_has_permission(notify_type, permissions): raise BadRequestError(message="Cannot send {}".format( get_public_notify_type_text(notify_type, plural=True))) +@statsd(namespace="performance-testing") def check_service_can_schedule_notification(permissions, scheduled_for): if scheduled_for: if not service_has_permission(SCHEDULE_NOTIFICATIONS, permissions): @@ -117,6 +121,7 @@ def check_sms_content_char_count(content_count): raise BadRequestError(message=message) +@statsd(namespace="performance-testing") def validate_template(template_id, personalisation, service, notification_type): try: template = templates_dao.dao_get_template_by_id_and_service_id( diff --git a/app/v2/notifications/create_response.py b/app/v2/notifications/create_response.py index 7eecfb1b9..a2e44001c 100644 --- a/app/v2/notifications/create_response.py +++ b/app/v2/notifications/create_response.py @@ -1,3 +1,5 @@ +from app.statsd_decorators import statsd + def create_post_sms_response_from_notification(notification, content, from_number, url_root, scheduled_for): noti = __create_notification_response(notification, url_root, scheduled_for) @@ -8,6 +10,7 @@ def create_post_sms_response_from_notification(notification, content, from_numbe return noti +@statsd(namespace="performance-testing") def create_post_email_response_from_notification(notification, content, subject, email_from, url_root, scheduled_for): noti = __create_notification_response(notification, url_root, scheduled_for) noti['content'] = { diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py index be35f30b4..08847f2cb 100644 --- a/app/v2/notifications/post_notifications.py +++ b/app/v2/notifications/post_notifications.py @@ -34,6 +34,7 @@ from app.notifications.validators import ( check_service_sms_sender_id ) from app.schema_validation import validate +from app.statsd_decorators import statsd from app.v2.errors import BadRequestError from app.v2.notifications import v2_notification_blueprint from app.v2.notifications.notification_schemas import ( @@ -49,6 +50,7 @@ from app.v2.notifications.create_response import ( @v2_notification_blueprint.route('/', methods=['POST']) +@statsd(namespace="performance-testing") def post_notification(notification_type): if notification_type == EMAIL_TYPE: form = validate(request.get_json(), post_email_request) @@ -118,6 +120,7 @@ def post_notification(notification_type): return jsonify(resp), 201 +@statsd(namespace="performance-testing") def process_sms_or_email_notification(*, form, notification_type, api_key, template, service, reply_to_text=None): form_send_to = form['email_address'] if notification_type == EMAIL_TYPE else form['phone_number'] @@ -187,6 +190,7 @@ def process_letter_notification(*, letter_data, api_key, template): return notification +@statsd(namespace="performance-testing") def get_reply_to_text(notification_type, form): reply_to = None if notification_type == EMAIL_TYPE: diff --git a/migrations/versions/0148_add_letters_as_pdf_svc_perm.py b/migrations/versions/0148_add_letters_as_pdf_svc_perm.py new file mode 100644 index 000000000..94f56a2af --- /dev/null +++ b/migrations/versions/0148_add_letters_as_pdf_svc_perm.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: 0148_add_letters_as_pdf_svc_perm +Revises: 0147_drop_mapping_tables +Create Date: 2017-12-01 13:33:18.581320 + +""" + +# revision identifiers, used by Alembic. +revision = '0148_add_letters_as_pdf_svc_perm' +down_revision = '0147_drop_mapping_tables' + +from alembic import op + + +def upgrade(): + op.get_bind() + op.execute("insert into service_permission_types values('letters_as_pdf')") + + +def downgrade(): + op.get_bind() + op.execute("delete from service_permissions where permission = 'letters_as_pdf'") + op.execute("delete from service_permission_types where name = 'letters_as_pdf'") diff --git a/requirements.txt b/requirements.txt index 4cdb13bdb..d9699e50a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ statsd==3.2.1 notifications-python-client==4.6.0 # PaaS -awscli==1.14.1 +awscli==1.14.2 awscli-cwlogs>=1.4,<1.5 git+https://github.com/alphagov/notifications-utils.git@23.2.1#egg=notifications-utils==23.2.1 diff --git a/tests/app/notifications/rest/test_send_notification.py b/tests/app/notifications/rest/test_send_notification.py index 4b57c4128..480455e75 100644 --- a/tests/app/notifications/rest/test_send_notification.py +++ b/tests/app/notifications/rest/test_send_notification.py @@ -16,7 +16,10 @@ from app.models import ( from app.dao.templates_dao import dao_get_all_templates_for_service, dao_update_template from app.dao.services_dao import dao_update_service from app.dao.api_key_dao import save_model_api_key -from app.v2.errors import RateLimitError +from app.errors import InvalidRequest +from app.models import Template +from app.v2.errors import RateLimitError, TooManyRequestsError + from tests import create_authorization_header from tests.app.conftest import ( sample_notification as create_sample_notification, @@ -28,9 +31,6 @@ from tests.app.conftest import ( sample_service, sample_template_without_sms_permission, sample_template_without_email_permission) - -from app.models import Template -from app.errors import InvalidRequest from tests.app.db import create_service, create_reply_to_email @@ -414,6 +414,10 @@ def test_should_block_api_call_if_over_day_limit_for_live_service( mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: + mocker.patch( + 'app.notifications.validators.check_service_over_daily_message_limit', + side_effect=TooManyRequestsError(1) + ) mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') service = create_sample_service(notify_db, notify_db_session, limit=1, restricted=False) @@ -446,6 +450,10 @@ def test_should_block_api_call_if_over_day_limit_for_restricted_service( with notify_api.test_request_context(): with notify_api.test_client() as client: mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') + mocker.patch( + 'app.notifications.validators.check_service_over_daily_message_limit', + side_effect=TooManyRequestsError(1) + ) service = create_sample_service(notify_db, notify_db_session, limit=1, restricted=True) email_template = create_sample_email_template(notify_db, notify_db_session, service=service) diff --git a/tests/app/notifications/test_validators.py b/tests/app/notifications/test_validators.py index cb841e431..1c9fbb579 100644 --- a/tests/app/notifications/test_validators.py +++ b/tests/app/notifications/test_validators.py @@ -1,6 +1,7 @@ import pytest from freezegun import freeze_time from flask import current_app + import app from app.models import INTERNATIONAL_SMS_TYPE, SMS_TYPE, EMAIL_TYPE from app.notifications.validators import ( @@ -18,6 +19,8 @@ from app.v2.errors import ( BadRequestError, TooManyRequestsError, RateLimitError) + +from tests.conftest import set_config from tests.app.conftest import ( sample_notification as create_notification, sample_service as create_service, @@ -26,6 +29,13 @@ from tests.app.conftest import ( from tests.app.db import create_reply_to_email, create_service_sms_sender +# all of these tests should have redis enabled (except where we specifically disable it) +@pytest.fixture(scope='module', autouse=True) +def enable_redis(notify_api): + with set_config(notify_api, 'REDIS_ENABLED', True): + yield + + @pytest.mark.parametrize('key_type', ['test', 'team', 'normal']) def test_check_service_message_limit_in_cache_with_unrestricted_service_is_allowed( key_type, @@ -78,6 +88,15 @@ def test_should_set_cache_value_as_value_from_database_if_cache_not_set( ) +def test_should_not_access_database_if_redis_disabled(notify_api, sample_service, mocker): + with set_config(notify_api, 'REDIS_ENABLED', False): + db_mock = mocker.patch('app.notifications.validators.services_dao') + + check_service_over_daily_message_limit('normal', sample_service) + + assert db_mock.method_calls == [] + + @pytest.mark.parametrize('key_type', ['team', 'normal']) def test_check_service_message_limit_over_message_limit_fails(key_type, notify_db, notify_db_session, mocker): with freeze_time("2016-01-01 12:00:00.000000"): diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 73679001f..2c09939b0 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -2163,18 +2163,17 @@ def test_search_for_notification_by_to_field_returns_content( assert notifications[0]['template']['content'] == 'Hello (( Name))\nYour thing is due soon' -def test_send_one_off_notification(admin_request, mocker): - service = create_service() - template = create_template(service=service) +def test_send_one_off_notification(sample_service, admin_request, mocker): + template = create_template(service=sample_service) mocker.patch('app.service.send_notification.send_notification_to_queue') response = admin_request.post( 'service.create_one_off_notification', - service_id=service.id, + service_id=sample_service.id, _data={ 'template_id': str(template.id), 'to': '07700900001', - 'created_by': str(service.created_by_id) + 'created_by': str(sample_service.created_by_id) }, _expected_status=201 ) diff --git a/tests/app/service/test_send_one_off_notification.py b/tests/app/service/test_send_one_off_notification.py index 14c49aa56..6a9a681ff 100644 --- a/tests/app/service/test_send_one_off_notification.py +++ b/tests/app/service/test_send_one_off_notification.py @@ -151,9 +151,13 @@ def test_send_one_off_notification_raises_if_cant_send_to_recipient(notify_db_se assert 'service is in trial mode' in e.value.message -def test_send_one_off_notification_raises_if_over_limit(notify_db_session): +def test_send_one_off_notification_raises_if_over_limit(notify_db_session, mocker): service = create_service(message_limit=0) template = create_template(service=service) + mocker.patch( + 'app.service.send_notification.check_service_over_daily_message_limit', + side_effect=TooManyRequestsError(1) + ) post_data = { 'template_id': str(template.id),