darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-01-29 22:11:52 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #917 from alphagov/rate-limit-api-calls

Browse Source 
Rate limit api calls
This commit is contained in:
minglis
2017-05-02 15:07:54 +01:00
committed by GitHub
parent 93a325eaae b0baa1532a
commit 2dcebc0b52
8 changed files with 283 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
app/config.py
View File

@@ -2,12 +2,13 @@ from datetime import timedelta
from celery.schedules import crontab
from kombu import Exchange, Queue
import os
from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST
if os.environ.get('VCAP_SERVICES'):
# on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
# standard environment variables from it
from app.cloudfoundry_config import extract_cloudfoundry_config
extract_cloudfoundry_config()
@@ -176,6 +177,21 @@ class Config(object):
DVLA_UPLOAD_BUCKET_NAME = "{}-dvla-file-per-job".format(os.getenv('NOTIFY_ENVIRONMENT'))
API_KEY_LIMITS = {
KEY_TYPE_TEAM: {
"limit": 3000,
"interval": 60
},
KEY_TYPE_NORMAL: {
"limit": 3000,
"interval": 60
},
KEY_TYPE_TEST: {
"limit": 3000,
"interval": 60
}
}
######################
# Config overrides ###
@@ -198,6 +214,7 @@ class Development(Config):
Queue('research-mode', Exchange('default'), routing_key='research-mode')
]
API_HOST_NAME = "http://localhost:6011"
API_RATE_LIMIT_ENABLED = True
class Test(Config):
@@ -220,14 +237,31 @@ class Test(Config):
Queue('research-mode', Exchange('default'), routing_key='research-mode')
]
REDIS_ENABLED = True
API_RATE_LIMIT_ENABLED = True
API_HOST_NAME = "http://localhost:6011"
API_KEY_LIMITS = {
KEY_TYPE_TEAM: {
"limit": 1,
"interval": 2
},
KEY_TYPE_NORMAL: {
"limit": 10,
"interval": 20
},
KEY_TYPE_TEST: {
"limit": 100,
"interval": 200
}
}
class Preview(Config):
NOTIFY_EMAIL_DOMAIN = 'notify.works'
NOTIFY_ENVIRONMENT = 'preview'
CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
FROM_NUMBER = 'preview'
API_RATE_LIMIT_ENABLED = True
class Staging(Config):
@@ -236,6 +270,7 @@ class Staging(Config):
CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload'
STATSD_ENABLED = True
FROM_NUMBER = 'stage'
API_RATE_LIMIT_ENABLED = True
class Live(Config):
@@ -247,6 +282,7 @@ class Live(Config):
FUNCTIONAL_TEST_PROVIDER_SERVICE_ID = '6c1d81bb-dae2-4ee9-80b0-89a4aae9f649'
FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID = 'ba9e1789-a804-40b8-871f-cc60d4c1286f'
PERFORMANCE_PLATFORM_ENABLED = True
API_RATE_LIMIT_ENABLED = False
class CloudFoundryConfig(Config):

6
app/notifications/rest.py
View File

@@ -16,9 +16,9 @@ from app.models import SMS_TYPE
from app.notifications.process_notifications import (persist_notification,
send_notification_to_queue,
simulated_recipient)
from app.notifications.validators import (check_service_message_limit,
from app.notifications.validators import (check_service_over_daily_message_limit,
check_template_is_for_notification_type,
check_template_is_active)
check_template_is_active, check_rate_limiting)
from app.schemas import (
email_notification_schema,
sms_template_notification_schema,
@@ -105,7 +105,7 @@ def send_notification(notification_type):
if errors:
raise InvalidRequest(errors, status_code=400)
check_service_message_limit(api_user.key_type, service)
check_rate_limiting(service, api_user)
template = templates_dao.dao_get_template_by_id_and_service_id(
template_id=notification_form['template'],

27
app/notifications/validators.py
View File

@@ -8,12 +8,22 @@ from notifications_utils.recipients import (
from app.dao import services_dao
from app.models import KEY_TYPE_TEST, KEY_TYPE_TEAM, SMS_TYPE
from app.service.utils import service_allowed_to_send_to
from app.v2.errors import TooManyRequestsError, BadRequestError
from app.v2.errors import TooManyRequestsError, BadRequestError, RateLimitError
from app import redis_store
from notifications_utils.clients import redis
def check_service_message_limit(key_type, service):
def check_service_over_api_rate_limit(service, api_key):
if current_app.config['API_RATE_LIMIT_ENABLED']:
cache_key = redis.rate_limit_cache_key(service.id, api_key.key_type)
rate_limit = current_app.config['API_KEY_LIMITS'][api_key.key_type]['limit']
interval = current_app.config['API_KEY_LIMITS'][api_key.key_type]['interval']
if redis_store.exceeded_rate_limit(cache_key, rate_limit, interval):
current_app.logger.error("service {} has been rate limited for throughput".format(service.id))
raise RateLimitError(rate_limit, interval, api_key.key_type)
def check_service_over_daily_message_limit(key_type, service):
if key_type != KEY_TYPE_TEST:
cache_key = redis.daily_limit_cache_key(service.id)
service_stats = redis_store.get(cache_key)
@@ -21,9 +31,18 @@ def check_service_message_limit(key_type, service):
service_stats = services_dao.fetch_todays_total_message_count(service.id)
redis_store.set(cache_key, service_stats, ex=3600)
if int(service_stats) >= service.message_limit:
current_app.logger.error(
"service {} has been rate limited for daily use sent {} limit {}".format(
service.id, int(service_stats), service.message_limit)
)
raise TooManyRequestsError(service.message_limit)
def check_rate_limiting(service, api_key):
check_service_over_api_rate_limit(service, api_key)
check_service_over_daily_message_limit(api_key.key_type, service)
def check_template_is_for_notification_type(notification_type, template_type):
if notification_type != template_type:
message = "{0} template is not suitable for {1} notification".format(template_type,
@@ -68,8 +87,6 @@ def validate_and_format_recipient(send_to, key_type, service, notification_type)
def check_sms_content_char_count(content_count):
char_count_limit = current_app.config.get('SMS_CHAR_COUNT_LIMIT')
if (
content_count > char_count_limit
):
if content_count > char_count_limit:
message = 'Content for template has a character count greater than the limit of {}'.format(char_count_limit)
raise BadRequestError(message=message)

13
app/v2/errors.py
View File

@@ -16,6 +16,19 @@ class TooManyRequestsError(InvalidRequest):
self.message = self.message_template.format(sending_limit)
class RateLimitError(InvalidRequest):
status_code = 429
message_template = 'Exceeded rate limit for key type {} of {} requests per {} seconds'
def __init__(self, sending_limit, interval, key_type):
# normal keys are spoken of as "live" in the documentation
# so using this in the error messaging
if key_type == 'normal':
key_type = 'live'
self.message = self.message_template.format(key_type.upper(), sending_limit, interval)
class BadRequestError(InvalidRequest):
status_code = 400
message = "An error occurred"

34
app/v2/notifications/post_notifications.py
View File

@@ -4,21 +4,25 @@ from sqlalchemy.orm.exc import NoResultFound
from app import api_user
from app.dao import services_dao, templates_dao
from app.models import SMS_TYPE, EMAIL_TYPE, PRIORITY
from app.notifications.process_notifications import (create_content_for_notification,
persist_notification,
send_notification_to_queue,
simulated_recipient)
from app.notifications.validators import (check_service_message_limit,
check_template_is_for_notification_type,
check_template_is_active,
check_sms_content_char_count,
validate_and_format_recipient)
from app.notifications.process_notifications import (
create_content_for_notification,
persist_notification,
send_notification_to_queue,
simulated_recipient)
from app.notifications.validators import (
check_template_is_for_notification_type,
check_template_is_active,
check_sms_content_char_count,
validate_and_format_recipient,
check_rate_limiting)
from app.schema_validation import validate
from app.v2.errors import BadRequestError
from app.v2.notifications import v2_notification_blueprint
from app.v2.notifications.notification_schemas import (post_sms_request,
create_post_sms_response_from_notification, post_email_request,
create_post_email_response_from_notification)
from app.v2.notifications.notification_schemas import (
post_sms_request,
create_post_sms_response_from_notification,
post_email_request,
create_post_email_response_from_notification)
@v2_notification_blueprint.route('/<notification_type>', methods=['POST'])
@@ -29,7 +33,9 @@ def post_notification(notification_type):
form = validate(request.get_json(), post_sms_request)
service = services_dao.dao_fetch_service_by_id(api_user.service_id)
check_service_message_limit(api_user.key_type, service)
check_rate_limiting(service, api_user)
form_send_to = form['phone_number'] if notification_type == SMS_TYPE else form['email_address']
send_to = validate_and_format_recipient(send_to=form_send_to,
key_type=api_user.key_type,
@@ -40,6 +46,7 @@ def post_notification(notification_type):
# Do not persist or send notification to the queue if it is a simulated recipient
simulated = simulated_recipient(send_to, notification_type)
notification = persist_notification(template_id=template.id,
template_version=template.version,
recipient=form_send_to,
@@ -50,6 +57,7 @@ def post_notification(notification_type):
key_type=api_user.key_type,
client_reference=form.get('reference', None),
simulated=simulated)
if not simulated:
queue_name = 'priority' if template.process_type == PRIORITY else None
send_notification_to_queue(notification=notification, research_mode=service.research_mode, queue=queue_name)