darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-20 14:29:25 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3383 from alphagov/email-sms-created-alert-180344153

Browse Source 
Add new log / alert for 'created' email / SMS
This commit is contained in:
Ben Thorner
2021-12-13 12:56:05 +00:00
committed by GitHub
parent c25585fd60 a8edfeb941
commit 2fb432adaf
11 changed files with 96 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
app/celery/nightly_tasks.py
View File

@@ -10,10 +10,6 @@ from sqlalchemy.exc import SQLAlchemyError
from app import notify_celery, zendesk_client
from app.aws import s3
from app.celery.service_callback_tasks import (
create_delivery_status_callback_data,
send_delivery_status_to_service,
)
from app.config import QueueNames
from app.cronitor import cronitor
from app.dao.fact_processing_time_dao import insert_update_processing_time
@@ -27,10 +23,6 @@ from app.dao.notifications_dao import (
dao_timeout_notifications,
delete_notifications_older_than_retention_by_type,
)
from app.dao.service_callback_api_dao import (
get_service_delivery_status_callback_api_for_service,
)
from app.exceptions import NotificationTechnicalFailureException
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
@@ -40,6 +32,9 @@ from app.models import (
FactProcessingTime,
Notification,
)
from app.notifications.notifications_ses_callback import (
check_and_queue_callback_task,
)
from app.utils import get_london_midnight_in_utc
@@ -123,25 +118,14 @@ def timeout_notifications():
# dao_timeout_notifications to return up to 100K notifications, so this task
# will operate on up to 500K - normally we only get around 20K.
for _ in range(0, 5):
technical_failure_notifications, temporary_failure_notifications = \
notifications = \
dao_timeout_notifications(current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD'))
notifications = technical_failure_notifications + temporary_failure_notifications
for notification in notifications:
# queue callback task only if the service_callback_api exists
service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) # noqa: E501
if service_callback_api:
encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
queue=QueueNames.CALLBACKS)
check_and_queue_callback_task(notification)
current_app.logger.info(
"Timeout period reached for {} notifications, status has been updated.".format(len(notifications)))
if technical_failure_notifications:
message = "{} notifications have been updated to technical-failure because they " \
"have timed out and are still in created.Notification ids: {}".format(
len(technical_failure_notifications), [str(x.id) for x in technical_failure_notifications])
raise NotificationTechnicalFailureException(message)
if len(notifications) < 100000:
return

4
app/celery/process_ses_receipts_tasks.py
View File

@@ -11,8 +11,8 @@ from app.config import QueueNames
from app.dao import notifications_dao
from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING
from app.notifications.notifications_ses_callback import (
_check_and_queue_callback_task,
_check_and_queue_complaint_callback_task,
check_and_queue_callback_task,
determine_notification_bounce_type,
handle_complaint,
)
@@ -76,7 +76,7 @@ def process_ses_results(self, response):
notification.sent_at
)
_check_and_queue_callback_task(notification)
check_and_queue_callback_task(notification)
return True

18
app/celery/process_sms_client_response_tasks.py
View File

@@ -5,20 +5,15 @@ from flask import current_app
from notifications_utils.template import SMSMessageTemplate
from app import notify_celery, statsd_client
from app.celery.service_callback_tasks import (
create_delivery_status_callback_data,
send_delivery_status_to_service,
)
from app.clients import ClientException
from app.clients.sms.firetext import get_firetext_responses
from app.clients.sms.mmg import get_mmg_responses
from app.config import QueueNames
from app.dao import notifications_dao
from app.dao.service_callback_api_dao import (
get_service_delivery_status_callback_api_for_service,
)
from app.dao.templates_dao import dao_get_template_by_id
from app.models import NOTIFICATION_PENDING
from app.notifications.notifications_ses_callback import (
check_and_queue_callback_task,
)
sms_response_mapper = {
'MMG': get_mmg_responses,
@@ -94,9 +89,4 @@ def _process_for_status(notification_status, client_name, provider_reference, de
notifications_dao.dao_update_notification(notification)
if notification_status != NOTIFICATION_PENDING:
service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
# queue callback task only if the service_callback_api exists
if service_callback_api:
encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
queue=QueueNames.CALLBACKS)
check_and_queue_callback_task(notification)

45
app/commands.py
View File

@@ -26,10 +26,6 @@ from app.celery.letters_pdf_tasks import (
from app.celery.reporting_tasks import (
create_nightly_notification_status_for_day,
)
from app.celery.service_callback_tasks import (
create_delivery_status_callback_data,
send_delivery_status_to_service,
)
from app.celery.tasks import process_row, record_daily_sorted_counts
from app.config import QueueNames
from app.dao.annual_billing_dao import (
@@ -52,9 +48,6 @@ from app.dao.permissions_dao import permission_dao
from app.dao.provider_rates_dao import (
create_provider_rates as dao_create_provider_rates,
)
from app.dao.service_callback_api_dao import (
get_service_delivery_status_callback_api_for_service,
)
from app.dao.services_dao import (
dao_fetch_all_services_by_user,
dao_fetch_all_services_created_by_user,
@@ -284,44 +277,6 @@ def recreate_pdf_for_precompiled_or_uploaded_letter(notification_id):
resanitise_pdf.apply_async([str(notification_id)], queue=QueueNames.LETTERS)
@notify_command(name='replay-service-callbacks')
@click.option('-f', '--file_name', required=True,
help="""Full path of the file to upload, file is a contains client references of
notifications that need the status to be sent to the service.""")
@click.option('-s', '--service_id', required=True,
help="""The service that the callbacks are for""")
def replay_service_callbacks(file_name, service_id):
print("Start send service callbacks for service: ", service_id)
callback_api = get_service_delivery_status_callback_api_for_service(service_id=service_id)
if not callback_api:
print("Callback api was not found for service: {}".format(service_id))
return
errors = []
notifications = []
file = open(file_name)
for ref in file:
try:
notification = Notification.query.filter_by(client_reference=ref.strip()).one()
notifications.append(notification)
except NoResultFound:
errors.append("Reference: {} was not found in notifications.".format(ref))
for e in errors:
print(e)
if errors:
raise Exception("Some notifications for the given references were not found")
for n in notifications:
encrypted_status_update = create_delivery_status_callback_data(n, callback_api)
send_delivery_status_to_service.apply_async([str(n.id), encrypted_status_update],
queue=QueueNames.CALLBACKS)
print("Replay service status for service: {}. Sent {} notification status updates to the queue".format(
service_id, len(notifications)))
def setup_commands(application):
application.cli.add_command(command_group)

47
app/dao/notifications_dao.py
View File

@@ -1,4 +1,3 @@
import functools
from datetime import datetime, timedelta
from itertools import groupby
from operator import attrgetter
@@ -40,7 +39,6 @@ from app.models import (
NOTIFICATION_SENDING,
NOTIFICATION_SENT,
NOTIFICATION_STATUS_TYPES_COMPLETED,
NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_TEMPORARY_FAILURE,
SMS_TYPE,
FactNotificationStatus,
@@ -491,7 +489,21 @@ def dao_delete_notifications_by_id(notification_id):
).delete(synchronize_session='fetch')
def _timeout_notifications(current_statuses, new_status, timeout_start, updated_at):
def dao_timeout_notifications(timeout_period_in_seconds):
"""
Timeout SMS and email notifications by the following rules:
the notification was sent to the provider but there was not a delivery receipt
sending -> temporary-failure
pending -> temporary-failure
Letter notifications are not timed out
"""
timeout_start = datetime.utcnow() - timedelta(seconds=timeout_period_in_seconds)
updated_at = datetime.utcnow()
current_statuses = [NOTIFICATION_SENDING, NOTIFICATION_PENDING]
new_status = NOTIFICATION_TEMPORARY_FAILURE
# TEMPORARY: limit the notifications to 100K as otherwise we
# see an issues where the task vanishes after it starts executing
# - we believe this is a OOM error but there are no logs. From
@@ -511,36 +523,9 @@ def _timeout_notifications(current_statuses, new_status, timeout_start, updated_
{'status': new_status, 'updated_at': updated_at},
synchronize_session=False
)
return notifications
def dao_timeout_notifications(timeout_period_in_seconds):
"""
Timeout SMS and email notifications by the following rules:
we never sent the notification to the provider for some reason
created -> technical-failure
the notification was sent to the provider but there was not a delivery receipt
sending -> temporary-failure
pending -> temporary-failure
Letter notifications are not timed out
"""
timeout_start = datetime.utcnow() - timedelta(seconds=timeout_period_in_seconds)
updated_at = datetime.utcnow()
timeout = functools.partial(_timeout_notifications, timeout_start=timeout_start, updated_at=updated_at)
# Notifications still in created status are marked with a technical-failure:
technical_failure_notifications = timeout([NOTIFICATION_CREATED], NOTIFICATION_TECHNICAL_FAILURE)
# Notifications still in sending or pending status are marked with a temporary-failure:
temporary_failure_notifications = timeout([NOTIFICATION_SENDING, NOTIFICATION_PENDING],
NOTIFICATION_TEMPORARY_FAILURE)
db.session.commit()
return technical_failure_notifications, temporary_failure_notifications
return notifications
def is_delivery_slow_for_providers(

2
app/notifications/notifications_ses_callback.py
View File

@@ -68,7 +68,7 @@ def remove_emails_from_complaint(complaint_dict):
return complaint_dict['mail'].pop('destination')
def _check_and_queue_callback_task(notification):
def check_and_queue_callback_task(notification):
# queue callback task only if the service_callback_api exists
service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
if service_callback_api: