mirror of
https://github.com/GSA/notifications-api.git
synced 2026-01-31 06:52:06 -05:00
Log warning for SES send rate throttling rather than exception
We have hit throttling limits from SES approximately once a week during a spike of traffic from GOV.UK. The rate limiting usually only lasts a couple of minutes but generates enough exceptions to cause a p1 but with no potential action for the responder. Therefore we downgrade the warning for this case to a warning and assume traffic will level back out such that the problem resolves itself. Note, we will still get exceptions if we go over our daily limit, rather than our per minute sending limit, which does require immediate action by someone responding. If we were to continually go over our per second sending rate for a long continous period of time, then there is a chance we may not be aware but given the risk of this happening is low I think it's an acceptable risk for the moment.
This commit is contained in:
@@ -5,6 +5,7 @@ from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
from app import notify_celery
|
||||
from app.config import QueueNames
|
||||
from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
|
||||
from app.dao import notifications_dao
|
||||
from app.dao.notifications_dao import update_notification_status_by_id
|
||||
from app.delivery import send_to_providers
|
||||
@@ -49,11 +50,17 @@ def deliver_email(self, notification_id):
|
||||
except InvalidEmailError as e:
|
||||
current_app.logger.exception(e)
|
||||
update_notification_status_by_id(notification_id, 'technical-failure')
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
try:
|
||||
current_app.logger.exception(
|
||||
"RETRY: Email notification {} failed".format(notification_id)
|
||||
)
|
||||
if isinstance(e, AwsSesClientThrottlingSendRateException):
|
||||
current_app.logger.warning(
|
||||
f"RETRY: Email notification {notification_id} was rate limited by SES"
|
||||
)
|
||||
else:
|
||||
current_app.logger.exception(
|
||||
f"RETRY: Email notification {notification_id} failed"
|
||||
)
|
||||
|
||||
self.retry(queue=QueueNames.RETRY)
|
||||
except self.MaxRetriesExceededError:
|
||||
message = "RETRY FAILED: Max retries reached. " \
|
||||
|
||||
Reference in New Issue
Block a user