From 16e93011efe7d1b893251686076388a8217cb3d1 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 29 Sep 2025 12:02:13 -0700 Subject: [PATCH 1/3] add log statement when our send quota is used up --- app/clients/cloudwatch/aws_cloudwatch.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/clients/cloudwatch/aws_cloudwatch.py b/app/clients/cloudwatch/aws_cloudwatch.py index 0a6d3d7be..0f5dc6c1e 100644 --- a/app/clients/cloudwatch/aws_cloudwatch.py +++ b/app/clients/cloudwatch/aws_cloudwatch.py @@ -7,6 +7,7 @@ from flask import current_app from app.clients import AWS_CLIENT_CONFIG, Client from app.cloudfoundry_config import cloud_config +from app.utils import hilite class AwsCloudwatchClient(Client): @@ -179,6 +180,13 @@ class AwsCloudwatchClient(Client): failed_event_set = self._get_receipts(log_group_name, start, end) current_app.logger.info((f"Failed message count: {len(failed_event_set)}")) + for failure in failed_event_set: + failure = json.loads(failure) + if "No quota left for account" == failure["delivery.providerResponse"]: + current_app.logger.warning( + hilite("**********NO QUOTA LEFT TO SEND MESSAGES!!!**********") + ) + return delivered_event_set, failed_event_set def _get_receipts(self, log_group_name, start, end): From c7f6a01048ed3fdf220839b7374caae3d9c75430 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 29 Sep 2025 12:06:26 -0700 Subject: [PATCH 2/3] raise exception if we see No Quota Left --- app/clients/cloudwatch/aws_cloudwatch.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/clients/cloudwatch/aws_cloudwatch.py b/app/clients/cloudwatch/aws_cloudwatch.py index 0f5dc6c1e..ec199d0fe 100644 --- a/app/clients/cloudwatch/aws_cloudwatch.py +++ b/app/clients/cloudwatch/aws_cloudwatch.py @@ -179,13 +179,16 @@ class AwsCloudwatchClient(Client): ) failed_event_set = self._get_receipts(log_group_name, start, end) current_app.logger.info((f"Failed message count: {len(failed_event_set)}")) - + raise_exception = False for failure in failed_event_set: failure = json.loads(failure) if "No quota left for account" == failure["delivery.providerResponse"]: current_app.logger.warning( hilite("**********NO QUOTA LEFT TO SEND MESSAGES!!!**********") ) + raise_exception = True + if raise_exception: + raise Exception("No Quota Left") return delivered_event_set, failed_event_set From a95269f9a1bf31a3db52f6e7d80fb1fc910d13f2 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Mon, 29 Sep 2025 13:03:42 -0700 Subject: [PATCH 3/3] handle delivery receipts when it's a test --- app/clients/cloudwatch/aws_cloudwatch.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/app/clients/cloudwatch/aws_cloudwatch.py b/app/clients/cloudwatch/aws_cloudwatch.py index ec199d0fe..df4945657 100644 --- a/app/clients/cloudwatch/aws_cloudwatch.py +++ b/app/clients/cloudwatch/aws_cloudwatch.py @@ -181,12 +181,15 @@ class AwsCloudwatchClient(Client): current_app.logger.info((f"Failed message count: {len(failed_event_set)}")) raise_exception = False for failure in failed_event_set: - failure = json.loads(failure) - if "No quota left for account" == failure["delivery.providerResponse"]: - current_app.logger.warning( - hilite("**********NO QUOTA LEFT TO SEND MESSAGES!!!**********") - ) - raise_exception = True + try: + failure = json.loads(failure) + if "No quota left for account" == failure["delivery.providerResponse"]: + current_app.logger.warning( + hilite("**********NO QUOTA LEFT TO SEND MESSAGES!!!**********") + ) + raise_exception = True + except Exception: + current_app.logger.exception("Malformed delivery receipt") if raise_exception: raise Exception("No Quota Left")