import os from datetime import datetime, timedelta from flask import current_app from sqlalchemy.orm.exc import NoResultFound from app import aws_cloudwatch_client, notify_celery from app.clients.email import EmailClientNonRetryableException from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException from app.clients.sms import SmsClientResponseException from app.config import QueueNames from app.dao import notifications_dao from app.dao.notifications_dao import ( sanitize_successful_notification_by_id, update_notification_status_by_id, ) from app.delivery import send_to_providers from app.exceptions import NotificationTechnicalFailureException from app.models import ( NOTIFICATION_DELIVERED, NOTIFICATION_FAILED, NOTIFICATION_TECHNICAL_FAILURE, NOTIFICATION_TEMPORARY_FAILURE, ) # This is the amount of time to wait after sending an sms message before we check the aws logs and look for delivery # receipts DELIVERY_RECEIPT_DELAY_IN_SECONDS = 120 @notify_celery.task( bind=True, name="check_sms_delivery_receipt", max_retries=48, default_retry_delay=300, ) def check_sms_delivery_receipt(self, message_id, notification_id, sent_at): """ This is called after deliver_sms to check the status of the message. This uses the same number of retries and the same delay period as deliver_sms. In addition, this fires five minutes after deliver_sms initially. So the idea is that most messages will succeed and show up in the logs quickly. Other message will resolve successfully after a retry or to. A few will fail but it will take up to 4 hours to know for sure. The call to check_sms will raise an exception if neither a success nor a failure appears in the cloudwatch logs, so this should keep retrying until the log appears, or until we run out of retries. """ # TODO the localstack cloudwatch doesn't currently have our log groups. Possibly create them with awslocal? if aws_cloudwatch_client.is_localstack(): status = "success" provider_response = "this is a fake successful localstack sms message" else: try: status, provider_response = aws_cloudwatch_client.check_sms( message_id, notification_id, sent_at ) except NotificationTechnicalFailureException as ntfe: provider_response = "Unable to find carrier response -- still looking" status = "pending" update_notification_status_by_id( notification_id, status, provider_response=provider_response ) raise self.retry(exc=ntfe) if status == "success": status = NOTIFICATION_DELIVERED elif status == "failure": status = NOTIFICATION_FAILED # if status is not success or failure the client raised an exception and this method will retry if status == NOTIFICATION_DELIVERED: sanitize_successful_notification_by_id( notification_id, provider_response=provider_response ) current_app.logger.info( f"Sanitized notification {notification_id} that was successfully delivered" ) else: update_notification_status_by_id( notification_id, status, provider_response=provider_response ) current_app.logger.info( f"Updated notification {notification_id} with response '{provider_response}'" ) @notify_celery.task( bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300 ) def deliver_sms(self, notification_id): try: current_app.logger.info( "Start sending SMS for notification id: {}".format(notification_id) ) notification = notifications_dao.get_notification_by_id(notification_id) ansi_green = "\033[32m" ansi_reset = "\033[0m" if not notification: raise NoResultFound() if ( os.getenv("NOTIFY_ENVIRONMENT") == "development" and "authentication code" in notification.content ): current_app.logger.warning( ansi_green + f"AUTHENTICATION CODE: {notification.content}" + ansi_reset ) message_id = send_to_providers.send_sms_to_provider(notification) # We have to put it in UTC. For other timezones, the delay # will be ignored and it will fire immediately (although this probably only affects developer testing) my_eta = datetime.utcnow() + timedelta( seconds=DELIVERY_RECEIPT_DELAY_IN_SECONDS ) check_sms_delivery_receipt.apply_async( [message_id, notification_id, notification.created_at], eta=my_eta, queue=QueueNames.CHECK_SMS, ) except Exception as e: update_notification_status_by_id( notification_id, NOTIFICATION_TEMPORARY_FAILURE ) if isinstance(e, SmsClientResponseException): current_app.logger.warning( "SMS notification delivery for id: {} failed".format(notification_id), exc_info=True, ) else: current_app.logger.exception( "SMS notification delivery for id: {} failed".format(notification_id) ) try: if self.request.retries == 0: self.retry(queue=QueueNames.RETRY, countdown=0) else: self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = ( "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. " "Notification has been updated to technical-failure".format( notification_id ) ) update_notification_status_by_id( notification_id, NOTIFICATION_TECHNICAL_FAILURE ) raise NotificationTechnicalFailureException(message) @notify_celery.task( bind=True, name="deliver_email", max_retries=48, default_retry_delay=300 ) def deliver_email(self, notification_id): try: current_app.logger.info( "Start sending email for notification id: {}".format(notification_id) ) notification = notifications_dao.get_notification_by_id(notification_id) if not notification: raise NoResultFound() send_to_providers.send_email_to_provider(notification) except EmailClientNonRetryableException as e: current_app.logger.exception( f"Email notification {notification_id} failed: {e}" ) update_notification_status_by_id(notification_id, "technical-failure") except Exception as e: try: if isinstance(e, AwsSesClientThrottlingSendRateException): current_app.logger.warning( f"RETRY: Email notification {notification_id} was rate limited by SES" ) else: current_app.logger.exception( f"RETRY: Email notification {notification_id} failed" ) self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = ( "RETRY FAILED: Max retries reached. " "The task send_email_to_provider failed for notification {}. " "Notification has been updated to technical-failure".format( notification_id ) ) update_notification_status_by_id( notification_id, NOTIFICATION_TECHNICAL_FAILURE ) raise NotificationTechnicalFailureException(message)