darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-09 05:13:12 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #596 from alphagov/extend-timeout-on-db-retries

Browse Source 
Improve logging and extend time periods associated with retrying
This commit is contained in:
minglis
2016-08-17 09:24:49 +01:00
committed by GitHub
parent 721929ca75 67a4ee7d51
commit 7fab5ada38
2 changed files with 28 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
app/celery/provider_tasks.py
View File

@@ -92,11 +92,15 @@ def send_sms_to_provider(self, service_id, notification_id):
except SmsClientException as e: except SmsClientException as e:
try: try:
current_app.logger.error( current_app.logger.error(
"SMS notification {} failed".format(notification_id) "RETRY: SMS notification {} failed".format(notification_id)
) )
current_app.logger.exception(e) current_app.logger.exception(e)
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries)) self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
except self.MaxRetriesExceededError: except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification.id),
e
)
update_notification_status_by_id(notification.id, 'technical-failure', 'failure') update_notification_status_by_id(notification.id, 'technical-failure', 'failure')
current_app.logger.info( current_app.logger.info(
@@ -173,11 +177,15 @@ def send_email_to_provider(self, service_id, notification_id):
except EmailClientException as e: except EmailClientException as e:
try: try:
current_app.logger.error( current_app.logger.error(
"Email notification {} failed".format(notification_id) "RETRY: Email notification {} failed".format(notification_id)
) )
current_app.logger.exception(e) current_app.logger.exception(e)
self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries)) self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries))
except self.MaxRetriesExceededError: except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task send_email_to_provider failed for notification {}".format(notification.id),
e
)
update_notification_status_by_id(notification.id, 'technical-failure', 'failure') update_notification_status_by_id(notification.id, 'technical-failure', 'failure')
current_app.logger.info( current_app.logger.info(

24
app/celery/tasks.py
View File

@@ -126,7 +126,7 @@ def remove_job(job_id):
current_app.logger.info("Job {} has been removed from s3.".format(job_id)) current_app.logger.info("Job {} has been removed from s3.".format(job_id))
@notify_celery.task(bind=True, name="send-sms", max_retries=5, default_retry_delay=5) @notify_celery.task(bind=True, name="send-sms", max_retries=5, default_retry_delay=300)
@statsd(namespace="tasks") @statsd(namespace="tasks")
def send_sms(self, def send_sms(self,
service_id, service_id,
@@ -154,11 +154,17 @@ def send_sms(self,
) )
except SQLAlchemyError as e: except SQLAlchemyError as e:
current_app.logger.exception(e) current_app.logger.exception("RETRY: send_sms notification {}".format(notification_id), e)
raise self.retry(queue="retry", exc=e) try:
raise self.retry(queue="retry", exc=e)
except self.MaxRetriesExceededError:
current_app.logger.exception(
"RETRY FAILED: task send_sms failed for notification {}".format(notification.id),
e
)
@notify_celery.task(bind=True, name="send-email", max_retries=5, default_retry_delay=5) @notify_celery.task(bind=True, name="send-email", max_retries=5, default_retry_delay=300)
@statsd(namespace="tasks") @statsd(namespace="tasks")
def send_email(self, service_id, def send_email(self, service_id,
notification_id, notification_id,
@@ -180,8 +186,14 @@ def send_email(self, service_id,
current_app.logger.info("Email {} created at {}".format(notification_id, created_at)) current_app.logger.info("Email {} created at {}".format(notification_id, created_at))
except SQLAlchemyError as e: except SQLAlchemyError as e:
current_app.logger.exception(e) current_app.logger.exception("RETRY: send_email notification {}".format(notification_id), e)
raise self.retry(queue="retry", exc=e) try:
raise self.retry(queue="retry", exc=e)
except self.MaxRetriesExceededError:
current_app.logger.error(
"RETRY FAILED: task send_email failed for notification {}".format(notification.id),
e
)
def _save_notification(created_at, notification, notification_id, service_id, notification_type, api_key_id, key_type): def _save_notification(created_at, notification, notification_id, service_id, notification_type, api_key_id, key_type):