Files
notifications-api/app/celery/service_callback_tasks.py
David McDonald 224d9bf35a Log when we don't retry a callback
We don't retry any callbacks when it receives a 4xx status. We should
probably be aware of this happening and at the moment there is nothing
in our logs to easily identify whether the request failed and is being
retried or if it failed and is not being retried. This will enable us to
search our logs easily and figure out how much it's happening.

It's quite likely that we should in the future allow callbacks to retry
if they get a 429 http response (rate limiting) but we should do this in
a smart way (exponential backoff) and so this is a first step to being
aware of how big a problem it is in case we want to do something about
it.
2020-11-17 11:26:32 +00:00

147 lines
5.3 KiB
Python

import json
from flask import current_app
from notifications_utils.statsd_decorators import statsd
from requests import (
HTTPError,
request,
RequestException
)
from app import (
notify_celery,
encryption
)
from app.config import QueueNames
@notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
@statsd(namespace="tasks")
def send_delivery_status_to_service(
self, notification_id, encrypted_status_update
):
status_update = encryption.decrypt(encrypted_status_update)
data = {
"id": str(notification_id),
"reference": status_update['notification_client_reference'],
"to": status_update['notification_to'],
"status": status_update['notification_status'],
"created_at": status_update['notification_created_at'],
"completed_at": status_update['notification_updated_at'],
"sent_at": status_update['notification_sent_at'],
"notification_type": status_update['notification_type']
}
_send_data_to_service_callback_api(
self,
data,
status_update['service_callback_api_url'],
status_update['service_callback_api_bearer_token'],
'send_delivery_status_to_service'
)
@notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
@statsd(namespace="tasks")
def send_complaint_to_service(self, complaint_data):
complaint = encryption.decrypt(complaint_data)
data = {
"notification_id": complaint['notification_id'],
"complaint_id": complaint['complaint_id'],
"reference": complaint['reference'],
"to": complaint['to'],
"complaint_date": complaint['complaint_date']
}
_send_data_to_service_callback_api(
self,
data,
complaint['service_callback_api_url'],
complaint['service_callback_api_bearer_token'],
'send_complaint_to_service'
)
def _send_data_to_service_callback_api(self, data, service_callback_url, token, function_name):
notification_id = (data["notification_id"] if "notification_id" in data else data["id"])
try:
response = request(
method="POST",
url=service_callback_url,
data=json.dumps(data),
headers={
'Content-Type': 'application/json',
'Authorization': 'Bearer {}'.format(token)
},
timeout=60
)
current_app.logger.info('{} sending {} to {}, response {}'.format(
function_name,
notification_id,
service_callback_url,
response.status_code
))
response.raise_for_status()
except RequestException as e:
current_app.logger.warning(
"{} request failed for notification_id: {} and url: {}. exception: {}".format(
function_name,
notification_id,
service_callback_url,
e
)
)
if not isinstance(e, HTTPError) or e.response.status_code >= 500:
try:
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
current_app.logger.warning(
"Retry: {} has retried the max num of times for callback url {} and notification_id: {}".format(
function_name,
service_callback_url,
notification_id
)
)
else:
current_app.logger.warning(
"{} callback is not being retried for notification_id: {} and url: {}. exception: {}".format(
function_name,
notification_id,
service_callback_url,
e
)
)
def create_delivery_status_callback_data(notification, service_callback_api):
from app import DATETIME_FORMAT, encryption
data = {
"notification_id": str(notification.id),
"notification_client_reference": notification.client_reference,
"notification_to": notification.to,
"notification_status": notification.status,
"notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
"notification_updated_at":
notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None,
"notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT) if notification.sent_at else None,
"notification_type": notification.notification_type,
"service_callback_api_url": service_callback_api.url,
"service_callback_api_bearer_token": service_callback_api.bearer_token,
}
return encryption.encrypt(data)
def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
from app import DATETIME_FORMAT, encryption
data = {
"complaint_id": str(complaint.id),
"notification_id": str(notification.id),
"reference": notification.client_reference,
"to": recipient,
"complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT),
"service_callback_api_url": service_callback_api.url,
"service_callback_api_bearer_token": service_callback_api.bearer_token,
}
return encryption.encrypt(data)