diff --git a/app/celery/tasks.py b/app/celery/tasks.py index cbbec1d19..31aee0dd6 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -30,9 +30,11 @@ from app.models import ( SMS_TYPE, ) from app.notifications.process_notifications import persist_notification +from app.notifications.validators import check_service_over_total_message_limit from app.serialised_models import SerialisedService, SerialisedTemplate from app.service.utils import service_allowed_to_send_to from app.utils import DATETIME_FORMAT +from app.v2.errors import TotalRequestsError @notify_celery.task(name="process-job") @@ -64,6 +66,9 @@ def process_job(job_id, sender_id=None): ) return + if __total_sending_limits_for_job_exceeded(service, job, job_id): + return + recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id( job ) @@ -145,6 +150,25 @@ def process_row(row, template, job, service, sender_id=None): return notification_id +def __total_sending_limits_for_job_exceeded(service, job, job_id): + try: + total_sent = check_service_over_total_message_limit(KEY_TYPE_NORMAL, service) + if total_sent + job.notification_count > service.total_message_limit: + raise TotalRequestsError(service.total_message_limit) + else: + return False + except TotalRequestsError: + job.job_status = "sending limits exceeded" + job.processing_finished = datetime.utcnow() + dao_update_job(job) + current_app.logger.error( + "Job {} size {} error. Total sending limits {} exceeded".format( + job_id, job.notification_count, service.message_limit + ) + ) + return True + + @notify_celery.task(bind=True, name="save-sms", max_retries=5, default_retry_delay=300) def save_sms(self, service_id, notification_id, encrypted_notification, sender_id=None): notification = encryption.decrypt(encrypted_notification) diff --git a/app/commands.py b/app/commands.py index 88ddde0e3..f25bf6770 100644 --- a/app/commands.py +++ b/app/commands.py @@ -792,3 +792,27 @@ def update_templates(): data = json.load(f) for d in data: _update_template(d["id"], d["name"], d["type"], d["content"], d["subject"]) + + +@notify_command(name="create-new-service") +@click.option("-n", "--name", required=True, prompt=True) +@click.option("-l", "--message_limit", required=False, default=40000) +@click.option("-r", "--restricted", required=False, default=False) +@click.option("-e", "--email_from", required=True) +@click.option("-c", "--created_by_id", required=True) +def create_new_service(name, message_limit, restricted, email_from, created_by_id): + data = { + "name": name, + "message_limit": message_limit, + "restricted": restricted, + "email_from": email_from, + "created_by_id": created_by_id, + } + + service = Service(**data) + try: + db.session.add(service) + db.session.commit() + except IntegrityError: + print("duplicate service", service.name) + db.session.rollback() diff --git a/app/config.py b/app/config.py index be3b964c7..d90b8b54c 100644 --- a/app/config.py +++ b/app/config.py @@ -282,6 +282,8 @@ class Config(object): FREE_SMS_TIER_FRAGMENT_COUNT = 250000 + TOTAL_MESSAGE_LIMIT = 250000 + DAILY_MESSAGE_LIMIT = notifications_utils.DAILY_MESSAGE_LIMIT HIGH_VOLUME_SERVICE = json.loads(getenv("HIGH_VOLUME_SERVICE", "[]")) diff --git a/app/models.py b/app/models.py index ca7cfe951..f4649ce7c 100644 --- a/app/models.py +++ b/app/models.py @@ -487,6 +487,9 @@ class Service(db.Model, Versioned): db.Boolean, index=False, unique=False, nullable=False, default=True ) message_limit = db.Column(db.BigInteger, index=False, unique=False, nullable=False) + total_message_limit = db.Column( + db.BigInteger, index=False, unique=False, nullable=False + ) restricted = db.Column(db.Boolean, index=False, unique=False, nullable=False) email_from = db.Column(db.Text, index=False, unique=True, nullable=False) created_by_id = db.Column( diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index 66a7d8409..0aff166f3 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -81,7 +81,7 @@ def persist_notification( reply_to_text=None, billable_units=None, document_download_count=None, - updated_at=None + updated_at=None, ): current_app.logger.info("Persisting notification") @@ -150,7 +150,7 @@ def persist_notification( current_app.logger.info("Redis total limit cache key does exist") redis_store.incr(total_key) current_app.logger.info( - "Redis total limit cache key has been incremented" + f"Redis total limit cache key has been incremented to {redis_store.get(total_key)}" ) current_app.logger.info( "{} {} created at {}".format( diff --git a/app/notifications/validators.py b/app/notifications/validators.py index 4d3e5efdd..e6c164080 100644 --- a/app/notifications/validators.py +++ b/app/notifications/validators.py @@ -3,6 +3,7 @@ from notifications_utils import SMS_CHAR_COUNT_LIMIT from notifications_utils.clients.redis import ( daily_total_cache_key, rate_limit_cache_key, + total_limit_cache_key, ) from notifications_utils.recipients import ( get_international_phone_info, @@ -44,6 +45,27 @@ def check_service_over_api_rate_limit(service, api_key): raise RateLimitError(rate_limit, interval, api_key.key_type) +def check_service_over_total_message_limit(key_type, service): + if key_type == KEY_TYPE_TEST or not current_app.config["REDIS_ENABLED"]: + return 0 + + cache_key = total_limit_cache_key(service.id) + service_stats = redis_store.get(cache_key) + if service_stats is None: + # first message of the day, set the cache to 0 and the expiry to 24 hours + service_stats = 0 + redis_store.set(cache_key, service_stats, ex=86400) + return service_stats + if int(service_stats) >= service.total_message_limit: + current_app.logger.warning( + "service {} has been rate limited for total use sent {} limit {}".format( + service.id, int(service_stats), service.total_message_limit + ) + ) + raise TotalRequestsError(service.total_message_limit) + return int(service_stats) + + def check_application_over_retention_limit(key_type, service): if key_type == KEY_TYPE_TEST or not current_app.config["REDIS_ENABLED"]: return 0 diff --git a/app/schemas.py b/app/schemas.py index 4a0b79b37..3c60d7a07 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -327,6 +327,7 @@ class DetailedServiceSchema(BaseSchema): "inbound_sms", "jobs", "message_limit", + "total_message_limit", "permissions", "rate_limit", "reply_to_email_addresses", @@ -735,6 +736,7 @@ class ServiceHistorySchema(ma.Schema): updated_at = FlexibleDateTime() active = fields.Boolean() message_limit = fields.Integer() + total_message_limit = fields.Integer() restricted = fields.Boolean() email_from = fields.String() created_by_id = fields.UUID() diff --git a/app/serialised_models.py b/app/serialised_models.py index da386c0a0..d9a227ccd 100644 --- a/app/serialised_models.py +++ b/app/serialised_models.py @@ -79,6 +79,7 @@ class SerialisedService(SerialisedModel): "contact_link", "email_from", "message_limit", + "total_message_limit", "permissions", "rate_limit", "restricted", diff --git a/app/service/send_notification.py b/app/service/send_notification.py index f70d800a0..f01056fee 100644 --- a/app/service/send_notification.py +++ b/app/service/send_notification.py @@ -12,6 +12,7 @@ from app.notifications.process_notifications import ( send_notification_to_queue, ) from app.notifications.validators import ( + check_service_over_total_message_limit, validate_and_format_recipient, validate_template, ) @@ -44,6 +45,8 @@ def send_one_off_notification(service_id, post_data): validate_template(template.id, personalisation, service, template.template_type) + check_service_over_total_message_limit(KEY_TYPE_NORMAL, service) + validate_and_format_recipient( send_to=post_data["to"], key_type=KEY_TYPE_NORMAL, diff --git a/migrations/versions/0400_add_total_message_limit.py b/migrations/versions/0400_add_total_message_limit.py new file mode 100644 index 000000000..b4410c36b --- /dev/null +++ b/migrations/versions/0400_add_total_message_limit.py @@ -0,0 +1,23 @@ +""" + +Revision ID: 0400_add_total_message_limit +Revises: 0399_remove_research_mode +Create Date: 2023-04-24 11:35:22.873930 + +""" +from alembic import op +import sqlalchemy as sa + + +revision = "0400_add_total_message_limit" +down_revision = "0399_remove_research_mode" + + +def upgrade(): + op.add_column("services", sa.Column("total_message_limit", sa.Integer)) + op.add_column("services_history", sa.Column("total_message_limit", sa.Integer)) + + +def downgrade(): + op.drop_column("services", "total_message_limit") + op.drop_column("services_history", "total_message_limit") diff --git a/pyproject.toml b/pyproject.toml index d33f82b16..2bc1853a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ python-dotenv = "==1.0.0" radon = "==6.0.1" sqlalchemy = "==1.4.40" werkzeug = "~=2.3" +notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils.git"} vulture = "==2.8" packaging = "==23.1" notifications-utils = {git = "https://github.com/GSA/notifications-utils.git", develop = true, rev = "main"} diff --git a/tests/app/conftest.py b/tests/app/conftest.py index d58ec33dc..55828ed35 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -211,6 +211,7 @@ def sample_service(sample_user): data = { "name": service_name, "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "email_from": email_from, "created_by": sample_user, diff --git a/tests/app/db.py b/tests/app/db.py index d78916bfe..63d7720bc 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -107,6 +107,7 @@ def create_service( email_from=None, prefix_sms=True, message_limit=1000, + total_message_limit=250000, organization_type="federal", check_if_service_exists=False, go_live_user=None, @@ -123,6 +124,7 @@ def create_service( service = Service( name=service_name, message_limit=message_limit, + total_message_limit=total_message_limit, restricted=restricted, email_from=email_from if email_from diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 69c06bc01..9cf0829ea 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -248,6 +248,7 @@ def test_get_service_by_id(admin_request, sample_service): "id", "inbound_api", "message_limit", + "total_message_limit", "name", "notes", "organization", @@ -372,6 +373,7 @@ def test_create_service( "name": "created service", "user_id": str(sample_user.id), "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "email_from": "created.service", @@ -441,6 +443,7 @@ def test_create_service_with_domain_sets_organization( "name": "created service", "user_id": str(sample_user.id), "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "email_from": "created.service", @@ -465,6 +468,7 @@ def test_create_service_should_create_annual_billing_for_service( "name": "created service", "user_id": str(sample_user.id), "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "email_from": "created.service", @@ -488,6 +492,7 @@ def test_create_service_should_raise_exception_and_not_create_service_if_annual_ "name": "created service", "user_id": str(sample_user.id), "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "email_from": "created.service", @@ -518,6 +523,7 @@ def test_create_service_inherits_branding_from_organization( "name": "created service", "user_id": str(sample_user.id), "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "email_from": "created.service", @@ -536,6 +542,7 @@ def test_should_not_create_service_with_missing_user_id_field(notify_api, fake_u "email_from": "service", "name": "created service", "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "created_by": str(fake_uuid), @@ -556,6 +563,7 @@ def test_should_error_if_created_by_missing(notify_api, sample_user): "email_from": "service", "name": "created service", "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "user_id": str(sample_user.id), @@ -581,6 +589,7 @@ def test_should_not_create_service_with_missing_if_user_id_is_not_in_database( "user_id": fake_uuid, "name": "created service", "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "created_by": str(fake_uuid), @@ -623,6 +632,7 @@ def test_should_not_create_service_with_duplicate_name( "name": sample_service.name, "user_id": str(sample_service.users[0].id), "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "email_from": "sample.service2", @@ -650,6 +660,7 @@ def test_create_service_should_throw_duplicate_key_constraint_for_existing_email "name": service_name, "user_id": str(first_service.users[0].id), "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "email_from": "first.service", @@ -1137,6 +1148,7 @@ def test_default_permissions_are_added_for_user_service( "name": "created service", "user_id": str(sample_user.id), "message_limit": 1000, + "total_message_limit": 250000, "restricted": False, "active": False, "email_from": "created.service", diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index 12274d5ea..adc91273d 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -5,6 +5,7 @@ import pytest from app.commands import ( _update_template, + create_new_service, create_test_user, fix_billable_units, insert_inbound_numbers_from_file, @@ -24,6 +25,7 @@ from app.models import ( Job, Notification, Organization, + Service, Template, User, ) @@ -324,3 +326,39 @@ def test_update_template(notify_db_session, email_2fa_code_template): t = Template.query.all() assert t[0].name == "Example text message template!" + + +def test_create_service_command(notify_db_session, notify_api): + notify_api.test_cli_runner().invoke( + create_test_user, + [ + "--email", + "somebody@fake.gov", + "--mobile_number", + "202-555-5555", + "--password", + "correct horse battery staple", + "--name", + "Fake Personson", + ], + ) + + user = User.query.first() + + service_count = Service.query.count() + + # run the command + result = notify_api.test_cli_runner().invoke( + create_new_service, + ["-e", "somebody@fake.gov", "-n", "Fake Service", "-c", user.id], + ) + print(result) + + # there should be one more service + assert Service.query.count() == service_count + 1 + + # that service should be the one we added + service = Service.query.filter_by(name="Fake Service").first() + assert service.email_from == "somebody@fake.gov" + assert service.restricted is False + assert service.message_limit == 40000