Merge pull request #1541 from GSA/notify-api-1540

change total message limit to 100000

app/celery/provider_tasks.py

@@ -14,6 +14,7 @@ from app.dao.notifications_dao import update_notification_status_by_id
 from app.delivery import send_to_providers
 from app.enums import NotificationStatus
 from app.exceptions import NotificationTechnicalFailureException
+from notifications_utils.clients.redis import total_limit_cache_key
 
 
 @notify_celery.task(
@@ -41,6 +42,9 @@ def deliver_sms(self, notification_id):
         # Code branches off to send_to_providers.py
         send_to_providers.send_sms_to_provider(notification)
 
+        cache_key = total_limit_cache_key(notification.service_id)
+        redis_store.incr(cache_key)
+
     except Exception as e:
         update_notification_status_by_id(
             notification_id,

app/celery/tasks.py

@@ -159,7 +159,14 @@ def process_row(row, template, job, service, sender_id=None):
     return notification_id
 
 
+# TODO
+# Originally this was checking a daily limit
+# It is now checking an overall limit (annual?) for the free tier
+# Is there any limit for the paid tier?
+# Assuming the limit is annual, is it calendar year, fiscal year, MOU year?
+# Do we need a command to run to clear the redis value, or should it happen automatically?
 def __total_sending_limits_for_job_exceeded(service, job, job_id):
+    print(hilite("ENTER __total_sending_limits_for_job_exceeded"))
     try:
         total_sent = check_service_over_total_message_limit(KeyType.NORMAL, service)
         if total_sent + job.notification_count > service.total_message_limit:
@@ -172,7 +179,7 @@ def __total_sending_limits_for_job_exceeded(service, job, job_id):
         dao_update_job(job)
         current_app.logger.exception(
             "Job {} size {} error. Total sending limits {} exceeded".format(
-                job_id, job.notification_count, service.message_limit
+                job_id, job.notification_count, service.total_message_limit
             ),
         )
         return True

app/config.py

@@ -339,7 +339,7 @@ class Config(object):
 
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 
-    TOTAL_MESSAGE_LIMIT = 250000
+    TOTAL_MESSAGE_LIMIT = 100000
 
     DAILY_MESSAGE_LIMIT = notifications_utils.DAILY_MESSAGE_LIMIT
 

app/delivery/send_to_providers.py

@@ -26,6 +26,7 @@ from app.enums import BrandType, KeyType, NotificationStatus, NotificationType
 from app.exceptions import NotificationTechnicalFailureException
 from app.serialised_models import SerialisedService, SerialisedTemplate
 from app.utils import hilite, utc_now
+from notifications_utils.clients.redis import total_limit_cache_key
 from notifications_utils.template import (
     HTMLEmailTemplate,
     PlainTextEmailTemplate,
@@ -119,7 +120,7 @@ def send_sms_to_provider(notification):
                 db.session.close()  # no commit needed as no changes to objects have been made above
 
                 message_id = provider.send_sms(**send_sms_kwargs)
-                current_app.logger.info(f"got message_id {message_id}")
+
                 update_notification_message_id(notification.id, message_id)
             except Exception as e:
                 n = notification
@@ -132,10 +133,14 @@ def send_sms_to_provider(notification):
             else:
                 # Here we map the job_id and row number to the aws message_id
                 n = notification
-                msg = f"Send to aws for job_id {n.job_id} row_number {n.job_row_number} message_id {message_id}"
+                msg = f"Send to AWS!!! for job_id {n.job_id} row_number {n.job_row_number} message_id {message_id}"
                 current_app.logger.info(hilite(msg))
                 notification.billable_units = template.fragment_count
                 update_notification_to_sending(notification, provider)
+
+                cache_key = total_limit_cache_key(service.id)
+                redis_store.incr(cache_key)
+
     return message_id
 
 

app/notifications/rest.py

@@ -12,7 +12,6 @@ from app.notifications.process_notifications import (
 )
 from app.notifications.validators import (
     check_if_service_can_send_to_number,
-    check_rate_limiting,
     service_has_permission,
     validate_template,
 )
@@ -125,8 +124,6 @@ def send_notification(notification_type):
         else email_notification_schema
     ).load(request.get_json())
 
-    check_rate_limiting(authenticated_service, api_user)
-
     template, template_with_content = validate_template(
         template_id=notification_form["template"],
         personalisation=notification_form.get("personalisation", {}),

app/notifications/validators.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
 from flask import current_app
 from sqlalchemy.orm.exc import NoResultFound
 
@@ -6,17 +9,14 @@ from app.dao.notifications_dao import dao_get_notification_count_for_service
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
 from app.enums import KeyType, NotificationType, ServicePermissionType, TemplateType
-from app.errors import BadRequestError, RateLimitError, TotalRequestsError
+from app.errors import BadRequestError, TotalRequestsError
 from app.models import ServicePermission
 from app.notifications.process_notifications import create_content_for_notification
 from app.serialised_models import SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
 from app.utils import get_public_notify_type_text
 from notifications_utils import SMS_CHAR_COUNT_LIMIT
-from notifications_utils.clients.redis import (
-    rate_limit_cache_key,
-    total_limit_cache_key,
-)
+from notifications_utils.clients.redis import total_limit_cache_key
 from notifications_utils.recipients import (
     get_international_phone_info,
     validate_and_format_email_address,
@@ -24,31 +24,27 @@ from notifications_utils.recipients import (
 )
 
 
-def check_service_over_api_rate_limit(service, api_key):
-    if (
-        current_app.config["API_RATE_LIMIT_ENABLED"]
-        and current_app.config["REDIS_ENABLED"]
-    ):
-        cache_key = rate_limit_cache_key(service.id, api_key.key_type)
-        rate_limit = service.rate_limit
-        interval = 60
-        if redis_store.exceeded_rate_limit(cache_key, rate_limit, interval):
-            current_app.logger.info(
-                "service {} has been rate limited for throughput".format(service.id)
-            )
-            raise RateLimitError(rate_limit, interval, api_key.key_type)
-
-
 def check_service_over_total_message_limit(key_type, service):
     if key_type == KeyType.TEST or not current_app.config["REDIS_ENABLED"]:
         return 0
 
     cache_key = total_limit_cache_key(service.id)
     service_stats = redis_store.get(cache_key)
+
+    # TODO
+    # For now we are using calendar year
+    # Switch to using service agreement dates when the Agreement model is ready
+    # If the service stat has never been set before, compute the remaining seconds for 2025
+    # and set it (all services) to expire on 12/31/2025.
     if service_stats is None:
-        # first message of the day, set the cache to 0 and the expiry to 24 hours
+        now_et = datetime.now(ZoneInfo("America/New_York"))
+        target_time = datetime(
+            2025, 12, 31, 23, 59, 59, tzinfo=ZoneInfo("America/New_York")
+        )
+        time_difference = target_time - now_et
+        seconds_difference = int(time_difference.total_seconds())
         service_stats = 0
-        redis_store.set(cache_key, service_stats, ex=86400)
+        redis_store.set(cache_key, service_stats, ex=seconds_difference)
         return service_stats
     if int(service_stats) >= service.total_message_limit:
         current_app.logger.warning(
@@ -57,6 +53,7 @@ def check_service_over_total_message_limit(key_type, service):
             )
         )
         raise TotalRequestsError(service.total_message_limit)
+
     return int(service_stats)
 
 
@@ -77,11 +74,6 @@ def check_application_over_retention_limit(key_type, service):
     return int(total_stats)
 
 
-def check_rate_limiting(service, api_key):
-    check_service_over_api_rate_limit(service, api_key)
-    check_application_over_retention_limit(api_key.key_type, service)
-
-
 def check_template_is_for_notification_type(notification_type, template_type):
     if notification_type != template_type:
         message = "{0} template is not suitable for {1} notification".format(

app/service/rest.py

@@ -7,7 +7,7 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
 from werkzeug.datastructures import MultiDict
 
-from app import db
+from app import db, redis_store
 from app.aws.s3 import get_personalisation_from_s3, get_phone_number_from_s3
 from app.config import QueueNames
 from app.dao import fact_notification_status_dao, notifications_dao
@@ -109,6 +109,7 @@ from app.service.service_senders_schema import (
 from app.service.utils import get_guest_list_objects
 from app.user.users_schema import post_set_permissions_schema
 from app.utils import get_prev_next_pagination_links, utc_now
+from notifications_utils.clients.redis import total_limit_cache_key
 
 service_blueprint = Blueprint("service", __name__)
 
@@ -1135,6 +1136,28 @@ def modify_service_data_retention(service_id, data_retention_id):
     return "", 204
 
 
+@service_blueprint.route("/get-service-message-ratio")
+def get_service_message_ratio():
+    service_id = request.args.get("service_id")
+
+    my_service = dao_fetch_service_by_id(service_id)
+
+    cache_key = total_limit_cache_key(service_id)
+    messages_sent = redis_store.get(cache_key)
+    if messages_sent is None:
+        messages_sent = 0
+        current_app.logger.warning(
+            f"Messages sent was not being tracked for service {service_id}"
+        )
+    else:
+        messages_sent = int(messages_sent)
+
+    return {
+        "messages_sent": messages_sent,
+        "total_message_limit": my_service.total_message_limit,
+    }, 200
+
+
 @service_blueprint.route("/monthly-data-by-service")
 def get_monthly_notification_data_by_service():
     start_date = request.args.get("start_date")

app/v2/notifications/post_notifications.py

@@ -18,7 +18,6 @@ from app.notifications.process_notifications import (
 from app.notifications.validators import (
     check_if_service_can_send_files_by_email,
     check_is_message_too_long,
-    check_rate_limiting,
     check_service_email_reply_to_id,
     check_service_has_permission,
     check_service_sms_sender_id,
@@ -54,8 +53,6 @@ def post_notification(notification_type):
 
     check_service_has_permission(notification_type, authenticated_service.permissions)
 
-    check_rate_limiting(authenticated_service, api_user)
-
     template, template_with_content = validate_template(
         form["template_id"],
         form.get("personalisation", {}),