add a validation check for a total-message daily limit

app/config.py

@@ -270,6 +270,8 @@ class Config(object):
 
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 
+    DAILY_MESSAGE_LIMIT = 5000
+
     HIGH_VOLUME_SERVICE = json.loads(getenv('HIGH_VOLUME_SERVICE', '[]'))
 
     TEMPLATE_PREVIEW_API_HOST = getenv('TEMPLATE_PREVIEW_API_HOST', 'http://localhost:6013')

app/notifications/process_notifications.py

@@ -142,6 +142,7 @@ def persist_notification(
         if key_type != KEY_TYPE_TEST and current_app.config['REDIS_ENABLED']:
             current_app.logger.info('Redis enabled, querying cache key for service id: {}'.format(service.id))
             cache_key = redis.daily_limit_cache_key(service.id)
+            total_key = "{}-{}".format(datetime.utcnow().strftime("%Y-%m-%d"), "total")
             current_app.logger.info('Redis daily limit cache key: {}'.format(cache_key))
             if redis_store.get(cache_key) is None:
                 current_app.logger.info('Redis daily limit cache key does not exist')
@@ -155,6 +156,14 @@ def persist_notification(
                 current_app.logger.info('Redis daily limit cache key does exist')
                 redis_store.incr(cache_key)
                 current_app.logger.info('Redis daily limit cache key has been incremented')
+            if redis_store.get(total_key) is None:
+                current_app.logger.info('Redis daily total cache key does not exist')
+                redis_store.set(total_key, 1, ex=86400)
+                current_app.logger.info('Set redis daily total cache key to 1')
+            else:
+                current_app.logger.info('Redis total limit cache key does exist')
+                redis_store.incr(total_key)
+                current_app.logger.info('Redis total limit cache key has been incremented')
         current_app.logger.info(
             "{} {} created at {}".format(notification_type, notification_id, notification_created_at)
         )

app/notifications/validators.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 from flask import current_app
 from gds_metrics.metrics import Histogram
 from notifications_utils import SMS_CHAR_COUNT_LIMIT
@@ -29,7 +31,12 @@ from app.notifications.process_notifications import (
 from app.serialised_models import SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
 from app.utils import get_public_notify_type_text
-from app.v2.errors import BadRequestError, RateLimitError, TooManyRequestsError
+from app.v2.errors import (
+    BadRequestError,
+    RateLimitError,
+    TooManyRequestsError,
+    TotalRequestsError,
+)
 
 REDIS_EXCEEDED_RATE_LIMIT_DURATION_SECONDS = Histogram(
     'redis_exceeded_rate_limit_duration_seconds',
@@ -68,8 +75,31 @@ def check_service_over_daily_message_limit(key_type, service):
     return int(service_stats)
 
 
+def check_application_over_daily_message_total(key_type, service):
+    if key_type == KEY_TYPE_TEST or not current_app.config['REDIS_ENABLED']:
+        return 0
+
+    # cache_key = daily_total_cache_key()
+    cache_key = "{}-{}".format(datetime.utcnow().strftime("%Y-%m-%d"), "total")
+    daily_message_limit = current_app.config['DAILY_MESSAGE_LIMIT']
+    total_stats = redis_store.get(cache_key)
+    if total_stats is None:
+        # first message of the day, set the cache to 0 and the expiry to 24 hours
+        total_stats = 0
+        redis_store.set(cache_key, total_stats, ex=86400)
+        return total_stats
+    if int(total_stats) >= daily_message_limit:
+        current_app.logger.info(
+            "while sending for service {}, daily message limit of {} reached".format(
+                service.id, daily_message_limit)
+        )
+        raise TotalRequestsError(daily_message_limit)
+    return int(total_stats)
+
+
 def check_rate_limiting(service, api_key):
     check_service_over_api_rate_limit(service, api_key)
+    check_application_over_daily_message_total(api_key.key_type, service)
     check_service_over_daily_message_limit(api_key.key_type, service)
 
 

app/v2/errors.py

@@ -18,6 +18,14 @@ class TooManyRequestsError(InvalidRequest):
         self.message = self.message_template.format(sending_limit)
 
 
+class TotalRequestsError(InvalidRequest):
+    status_code = 429
+    message_template = 'Exceeded total application limits ({}) for today'
+
+    def __init__(self, sending_limit):
+        self.message = self.message_template.format(sending_limit)
+
+
 class RateLimitError(InvalidRequest):
     status_code = 429
     message_template = 'Exceeded rate limit for key type {} of {} requests per {} seconds'

tests/app/notifications/test_process_notification.py

@@ -1,6 +1,7 @@
 import datetime
 import uuid
 from collections import namedtuple
+from unittest.mock import call
 
 import pytest
 from boto3.exceptions import Boto3Error
@@ -217,7 +218,11 @@ def test_persist_notification_increments_cache_for_trial_or_live_service(
             key_type=api_key.key_type,
             reference="ref2")
 
-        mock_incr.assert_called_once_with(str(service.id) + "-2016-01-01-count", )
+        assert mock_incr.call_count == 2
+        mock_incr.assert_has_calls([
+            call(str(service.id) + "-2016-01-01-count", ),
+            call("2016-01-01-total", )
+        ])
 
 
 @pytest.mark.parametrize('restricted_service', [True, False])
@@ -242,7 +247,11 @@ def test_persist_notification_sets_daily_limit_cache_if_one_does_not_exists(
             key_type=api_key.key_type,
             reference="ref2")
 
-        mock_set.assert_called_once_with(str(service.id) + "-2016-01-01-count", 1, ex=86400)
+        assert mock_set.call_count == 2
+        mock_set.assert_has_calls([
+            call(str(service.id) + "-2016-01-01-count", 1, ex=86400),
+            call("2016-01-01-total", 1, ex=86400)
+        ])
 
 
 @pytest.mark.parametrize((

tests/app/notifications/test_validators.py

@@ -12,6 +12,7 @@ from app.notifications.process_notifications import (
     create_content_for_notification,
 )
 from app.notifications.validators import (
+    check_application_over_daily_message_total,
     check_if_service_can_send_files_by_email,
     check_is_message_too_long,
     check_notification_content_is_not_empty,
@@ -33,7 +34,12 @@ from app.serialised_models import (
     SerialisedTemplate,
 )
 from app.utils import get_template_instance
-from app.v2.errors import BadRequestError, RateLimitError, TooManyRequestsError
+from app.v2.errors import (
+    BadRequestError,
+    RateLimitError,
+    TooManyRequestsError,
+    TotalRequestsError,
+)
 from tests.app.db import (
     create_api_key,
     create_reply_to_email,
@@ -113,6 +119,18 @@ def test_check_service_message_limit_over_message_limit_fails(key_type, mocker,
     assert e.value.fields == []
 
 
+@pytest.mark.parametrize('key_type', ['team', 'normal'])
+def test_check_service_message_limit_over_total_limit_fails(key_type, mocker, notify_db_session):
+    service = create_service()
+    mocker.patch('app.redis_store.get', return_value="5001")
+
+    with pytest.raises(TotalRequestsError) as e:
+        check_application_over_daily_message_total(key_type, service)
+    assert e.value.status_code == 429
+    assert e.value.message == 'Exceeded total application limits (5000) for today'
+    assert e.value.fields == []
+
+
 @pytest.mark.parametrize('template_type, notification_type',
                          [(EMAIL_TYPE, EMAIL_TYPE),
                           (SMS_TYPE, SMS_TYPE)])