mirror of
https://github.com/GSA/notifications-api.git
synced 2025-12-12 08:12:27 -05:00
59ab5da5d3
- Thows a NoResultFound sqlalchemy exception - Which causes a retry. This means we give it a few goes (5, max 5 hours) for the notification to appear. - Should never happen, only if we get some task overlaps that are unusual that leads to tasks executed in an overlapping nature.
125 lines
4.8 KiB
Python
125 lines
4.8 KiB
Python
from flask import current_app
|
|
|
|
from app import notify_celery
|
|
from app.dao import notifications_dao
|
|
from app.dao.notifications_dao import update_notification_status_by_id
|
|
from app.statsd_decorators import statsd
|
|
|
|
from app.delivery import send_to_providers
|
|
from sqlalchemy.orm.exc import NoResultFound
|
|
|
|
def retry_iteration_to_delay(retry=0):
|
|
"""
|
|
:param retry times we have performed a retry
|
|
Given current retry calculate some delay before retrying
|
|
0: 10 seconds
|
|
1: 60 seconds (1 minutes)
|
|
2: 300 seconds (5 minutes)
|
|
3: 3600 seconds (60 minutes)
|
|
4: 14400 seconds (4 hours)
|
|
:param retry (zero indexed):
|
|
:return length to retry in seconds, default 10 seconds
|
|
"""
|
|
|
|
delays = {
|
|
0: 10,
|
|
1: 60,
|
|
2: 300,
|
|
3: 3600,
|
|
4: 14400
|
|
}
|
|
|
|
return delays.get(retry, 10)
|
|
|
|
|
|
@notify_celery.task(bind=True, name="deliver_sms", max_retries=5, default_retry_delay=5)
|
|
@statsd(namespace="tasks")
|
|
def deliver_sms(self, notification_id):
|
|
try:
|
|
notification = notifications_dao.get_notification_by_id(notification_id)
|
|
if not notification:
|
|
raise NoResultFound()
|
|
send_to_providers.send_sms_to_provider(notification)
|
|
except Exception as e:
|
|
try:
|
|
current_app.logger.error(
|
|
"RETRY: SMS notification {} failed".format(notification_id)
|
|
)
|
|
current_app.logger.exception(e)
|
|
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
|
|
except self.MaxRetriesExceededError:
|
|
current_app.logger.error(
|
|
"RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification_id),
|
|
e
|
|
)
|
|
update_notification_status_by_id(notification_id, 'technical-failure')
|
|
|
|
|
|
@notify_celery.task(bind=True, name="deliver_email", max_retries=5, default_retry_delay=5)
|
|
@statsd(namespace="tasks")
|
|
def deliver_email(self, notification_id):
|
|
try:
|
|
notification = notifications_dao.get_notification_by_id(notification_id)
|
|
if not notification:
|
|
raise NoResultFound()
|
|
send_to_providers.send_email_to_provider(notification)
|
|
except Exception as e:
|
|
try:
|
|
current_app.logger.error(
|
|
"RETRY: Email notification {} failed".format(notification_id)
|
|
)
|
|
current_app.logger.exception(e)
|
|
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
|
|
except self.MaxRetriesExceededError:
|
|
current_app.logger.error(
|
|
"RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification_id),
|
|
e
|
|
)
|
|
update_notification_status_by_id(notification_id, 'technical-failure')
|
|
|
|
|
|
@notify_celery.task(bind=True, name="send-sms-to-provider", max_retries=5, default_retry_delay=5)
|
|
@statsd(namespace="tasks")
|
|
def send_sms_to_provider(self, service_id, notification_id):
|
|
try:
|
|
notification = notifications_dao.get_notification_by_id(notification_id)
|
|
if not notification:
|
|
raise NoResultFound()
|
|
send_to_providers.send_sms_to_provider(notification)
|
|
except Exception as e:
|
|
try:
|
|
current_app.logger.error(
|
|
"RETRY: SMS notification {} failed".format(notification_id)
|
|
)
|
|
current_app.logger.exception(e)
|
|
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
|
|
except self.MaxRetriesExceededError:
|
|
current_app.logger.error(
|
|
"RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification_id),
|
|
e
|
|
)
|
|
update_notification_status_by_id(notification_id, 'technical-failure')
|
|
|
|
|
|
@notify_celery.task(bind=True, name="send-email-to-provider", max_retries=5, default_retry_delay=5)
|
|
@statsd(namespace="tasks")
|
|
def send_email_to_provider(self, service_id, notification_id):
|
|
try:
|
|
notification = notifications_dao.get_notification_by_id(notification_id)
|
|
if not notification:
|
|
raise NoResultFound()
|
|
send_to_providers.send_email_to_provider(notification)
|
|
except Exception as e:
|
|
try:
|
|
current_app.logger.error(
|
|
"RETRY: Email notification {} failed".format(notification_id)
|
|
)
|
|
current_app.logger.exception(e)
|
|
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
|
|
except self.MaxRetriesExceededError:
|
|
current_app.logger.error(
|
|
"RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification_id),
|
|
e
|
|
)
|
|
update_notification_status_by_id(notification_id, 'technical-failure')
|