notify-300 set total message limit of 250k

app/celery/tasks.py

@@ -30,7 +30,10 @@ from app.models import (
     SMS_TYPE,
 )
 from app.notifications.process_notifications import persist_notification
-from app.notifications.validators import check_service_over_daily_message_limit
+from app.notifications.validators import (
+    check_service_over_daily_message_limit,
+    check_service_over_total_message_limit,
+)
 from app.serialised_models import SerialisedService, SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
 from app.utils import DATETIME_FORMAT
@@ -59,7 +62,10 @@ def process_job(job_id, sender_id=None):
             "Job {} has been cancelled, service {} is inactive".format(job_id, service.id))
         return
 
-    if __sending_limits_for_job_exceeded(service, job, job_id):
+    if __daily_sending_limits_for_job_exceeded(service, job, job_id):
+        return
+
+    if __total_sending_limits_for_job_exceeded(service, job, job_id):
         return
 
     recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job)
@@ -134,10 +140,10 @@ def process_row(row, template, job, service, sender_id=None):
     return notification_id
 
 
-def __sending_limits_for_job_exceeded(service, job, job_id):
+def __daily_sending_limits_for_job_exceeded(service, job, job_id):
     try:
-        total_sent = check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
-        if total_sent + job.notification_count > service.message_limit:
+        total_daily_sent = check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
+        if total_daily_sent + job.notification_count > service.message_limit:
             raise TooManyRequestsError(service.message_limit)
         else:
             return False
@@ -146,7 +152,25 @@ def __sending_limits_for_job_exceeded(service, job, job_id):
         job.processing_finished = datetime.utcnow()
         dao_update_job(job)
         current_app.logger.info(
-            "Job {} size {} error. Sending limits {} exceeded".format(
+            "Job {} size {} error. Daily ending limits {} exceeded".format(
+                job_id, job.notification_count, service.message_limit)
+        )
+        return True
+
+
+def __total_sending_limits_for_job_exceeded(service, job, job_id):
+    try:
+        total_sent = check_service_over_total_message_limit(KEY_TYPE_NORMAL, service)
+        if total_sent + job.notification_count > service.total_message_limit:
+            raise TooManyRequestsError(service.total_message_limit)
+        else:
+            return False
+    except TooManyRequestsError:
+        job.job_status = 'total sending limits exceeded'
+        job.processing_finished = datetime.utcnow()
+        dao_update_job(job)
+        current_app.logger.info(
+            "Job {} size {} error. Total sending limits {} exceeded".format(
                 job_id, job.notification_count, service.message_limit)
         )
         return True

app/config.py

@@ -267,6 +267,7 @@ class Config(object):
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 
     DAILY_MESSAGE_LIMIT = 5000
+    TOTAL_MESSAGE_LIMIT = 7
 
     HIGH_VOLUME_SERVICE = json.loads(getenv('HIGH_VOLUME_SERVICE', '[]'))
 

app/models.py

@@ -435,6 +435,8 @@ class Service(db.Model, Versioned):
         onupdate=datetime.datetime.utcnow)
     active = db.Column(db.Boolean, index=False, unique=False, nullable=False, default=True)
     message_limit = db.Column(db.BigInteger, index=False, unique=False, nullable=False)
+    # TODO nullable if we are adding it late?
+    total_message_limit = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=250000)
     restricted = db.Column(db.Boolean, index=False, unique=False, nullable=False)
     research_mode = db.Column(db.Boolean, index=False, unique=False, nullable=False, default=False)
     email_from = db.Column(db.Text, index=False, unique=True, nullable=False)

app/notifications/validators.py

@@ -5,6 +5,7 @@ from notifications_utils.clients.redis import (
     daily_limit_cache_key,
     daily_total_cache_key,
     rate_limit_cache_key,
+    total_limit_cache_key,
 )
 from notifications_utils.recipients import (
     get_international_phone_info,
@@ -74,6 +75,26 @@ def check_service_over_daily_message_limit(key_type, service):
     return int(service_stats)
 
 
+def check_service_over_total_message_limit(key_type, service):
+    if key_type == KEY_TYPE_TEST or not current_app.config['REDIS_ENABLED']:
+        return 0
+
+    cache_key = total_limit_cache_key(service.id)
+    service_stats = redis_store.get(cache_key)
+    if service_stats is None:
+        # first message of the day, set the cache to 0 and the expiry to 24 hours
+        service_stats = 0
+        redis_store.set(cache_key, service_stats, ex=86400)
+        return service_stats
+    if int(service_stats) >= service.total_message_limit:
+        current_app.logger.info(
+            "service {} has been rate limited for total use sent {} limit {}".format(
+                service.id, int(service_stats), service.total_message_limit)
+        )
+        raise TooManyRequestsError(service.total_message_limit)
+    return int(service_stats)
+
+
 def check_application_over_daily_message_total(key_type, service):
     if key_type == KEY_TYPE_TEST or not current_app.config['REDIS_ENABLED']:
         return 0

app/schemas.py

@@ -317,6 +317,7 @@ class DetailedServiceSchema(BaseSchema):
             'inbound_sms',
             'jobs',
             'message_limit',
+            'total_message_limit',
             'permissions',
             'rate_limit',
             'reply_to_email_addresses',
@@ -697,6 +698,7 @@ class ServiceHistorySchema(ma.Schema):
     updated_at = FlexibleDateTime()
     active = fields.Boolean()
     message_limit = fields.Integer()
+    total_message_limit = fields.Integer()
     restricted = fields.Boolean()
     email_from = fields.String()
     created_by_id = fields.UUID()

app/serialised_models.py

@@ -79,6 +79,7 @@ class SerialisedService(SerialisedModel):
         'contact_link',
         'email_from',
         'message_limit',
+        'total_message_limit',
         'permissions',
         'rate_limit',
         'research_mode',

app/service/send_notification.py

@@ -13,6 +13,7 @@ from app.notifications.process_notifications import (
 )
 from app.notifications.validators import (
     check_service_over_daily_message_limit,
+    check_service_over_total_message_limit,
     validate_and_format_recipient,
     validate_template,
 )
@@ -47,6 +48,8 @@ def send_one_off_notification(service_id, post_data):
 
     check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
 
+    check_service_over_total_message_limit(KEY_TYPE_NORMAL, service)
+
     validate_and_format_recipient(
         send_to=post_data['to'],
         key_type=KEY_TYPE_NORMAL,