diff --git a/app/celery/provider_tasks.py b/app/celery/provider_tasks.py index 8efcc4d92..2e59541a9 100644 --- a/app/celery/provider_tasks.py +++ b/app/celery/provider_tasks.py @@ -25,6 +25,7 @@ def deliver_sms(self, notification_id): notification = notifications_dao.get_notification_by_id(notification_id) if not notification: raise NoResultFound() + current_app.logger.info("Start sending SMS for notification id: {}".format(notification_id)) send_to_providers.send_sms_to_provider(notification) except Exception as e: try: @@ -46,6 +47,7 @@ def deliver_email(self, notification_id): notification = notifications_dao.get_notification_by_id(notification_id) if not notification: raise NoResultFound() + current_app.logger.info("Start sending email for notification id: {}".format(notification_id)) send_to_providers.send_email_to_provider(notification) except InvalidEmailError as e: current_app.logger.exception(e) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index d863dda19..a979c418c 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -39,6 +39,7 @@ from app.dao.notifications_dao import ( dao_get_count_of_letters_to_process_for_date, dao_get_scheduled_notifications, set_scheduled_notification_to_processed, + notifications_not_yet_sent ) from app.dao.provider_details_dao import ( get_current_provider, @@ -53,7 +54,9 @@ from app.models import ( LETTER_TYPE, JOB_STATUS_IN_PROGRESS, JOB_STATUS_READY_TO_SEND, - JOB_STATUS_ERROR + JOB_STATUS_ERROR, + SMS_TYPE, + EMAIL_TYPE ) from app.notifications.process_notifications import send_notification_to_queue from app.celery.tasks import ( @@ -535,3 +538,22 @@ def letter_raise_alert_if_no_ack_file_for_zip(): current_app.logger.info( "letter ack contains zip that is not for today: {}".format(ack_content_set - zip_file_set) ) + + +@notify_celery.task(name='replay-created-notifications') +@statsd(namespace="tasks") +def replay_created_notifications(): + # if the notification has not be send after 4 hours + 15 minutes, then try to resend. + resend_created_notifications_older_than = (60 * 60 * 4) + (60 * 15) + for notification_type in (EMAIL_TYPE, SMS_TYPE): + notifications_to_resend = notifications_not_yet_sent( + resend_created_notifications_older_than, + notification_type + ) + + current_app.logger.info("Sending {} {} notifications " + "to the delivery queue because the notification " + "status was created.".format(len(notifications_to_resend), notification_type)) + + for n in notifications_to_resend: + send_notification_to_queue(notification=n, research_mode=n.service.research_mode) diff --git a/app/config.py b/app/config.py index 6b89cf331..315845a5d 100644 --- a/app/config.py +++ b/app/config.py @@ -266,6 +266,11 @@ class Config(object): 'task': 'daily-stats-template-usage-by-month', 'schedule': crontab(hour=0, minute=5), 'options': {'queue': QueueNames.PERIODIC} + }, + 'replay-created-notifications': { + 'task': 'replay-created-notifications', + 'schedule': crontab(minute='0, 15, 30, 45'), + 'options': {'queue': QueueNames.PERIODIC} } } CELERY_QUEUES = [] diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 2e0a77011..05ca4667c 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -600,3 +600,14 @@ def dao_get_count_of_letters_to_process_for_date(date_to_process=None): ).count() return count_of_letters_to_process_for_date + + +def notifications_not_yet_sent(should_be_sending_after_seconds, notification_type): + older_than_date = datetime.utcnow() - timedelta(seconds=should_be_sending_after_seconds) + + notifications = Notification.query.filter( + Notification.created_at <= older_than_date, + Notification.notification_type == notification_type, + Notification.status == NOTIFICATION_CREATED + ).all() + return notifications diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 008a4bcf7..ad431b3cd 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -34,7 +34,8 @@ from app.celery.scheduled_tasks import ( switch_current_sms_provider_on_slow_delivery, timeout_notifications, daily_stats_template_usage_by_month, - letter_raise_alert_if_no_ack_file_for_zip + letter_raise_alert_if_no_ack_file_for_zip, + replay_created_notifications ) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.config import QueueNames, TaskNames @@ -1191,3 +1192,33 @@ def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db assert mock_file_list.call_count == 2 assert mock_get_file.call_count == 0 + + +def test_replay_created_notifications(notify_db_session, sample_service, mocker): + email_delivery_queue = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') + sms_delivery_queue = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async') + + sms_template = create_template(service=sample_service, template_type='sms') + email_template = create_template(service=sample_service, template_type='email') + older_than = (60 * 60 * 4) + (60 * 15) # 4 hours 15 minutes + # notifications expected to be resent + old_sms = create_notification(template=sms_template, created_at=datetime.utcnow() - timedelta(seconds=older_than), + status='created') + old_email = create_notification(template=email_template, + created_at=datetime.utcnow() - timedelta(seconds=older_than), + status='created') + # notifications that are not to be resent + create_notification(template=sms_template, created_at=datetime.utcnow() - timedelta(seconds=older_than), + status='sending') + create_notification(template=email_template, created_at=datetime.utcnow() - timedelta(seconds=older_than), + status='delivered') + create_notification(template=sms_template, created_at=datetime.utcnow(), + status='created') + create_notification(template=email_template, created_at=datetime.utcnow(), + status='created') + + replay_created_notifications() + email_delivery_queue.assert_called_once_with([str(old_email.id)], + queue='send-email-tasks') + sms_delivery_queue.assert_called_once_with([str(old_sms.id)], + queue="send-sms-tasks") diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index 1d6d63755..e8a688339 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -33,6 +33,7 @@ from app.dao.notifications_dao import ( dao_get_notification_by_reference, dao_get_notifications_by_references, dao_get_notification_history_by_reference, + notifications_not_yet_sent ) from app.dao.services_dao import dao_update_service from app.models import ( @@ -2216,3 +2217,40 @@ def test_dao_get_count_of_letters_to_process_for_date_ignores_test_keys(sample_l count_for_date = dao_get_count_of_letters_to_process_for_date() assert count_for_date == 2 + + +@pytest.mark.parametrize("notification_type", + ["letter", "email", "sms"] + ) +def test_notifications_not_yet_sent(sample_service, notification_type): + older_than = 4 # number of seconds the notification can not be older than + template = create_template(service=sample_service, template_type=notification_type) + old_notification = create_notification(template=template, + created_at=datetime.utcnow() - timedelta(seconds=older_than), + status='created') + create_notification(template=template, + created_at=datetime.utcnow() - timedelta(seconds=older_than), + status='sending') + create_notification(template=template, created_at=datetime.utcnow(), status='created') + + results = notifications_not_yet_sent(older_than, notification_type) + assert len(results) == 1 + assert results[0] == old_notification + + +@pytest.mark.parametrize("notification_type", + ["letter", "email", "sms"] + ) +def test_notifications_not_yet_sent_return_no_rows(sample_service, notification_type): + older_than = 5 # number of seconds the notification can not be older than + template = create_template(service=sample_service, template_type=notification_type) + create_notification(template=template, + created_at=datetime.utcnow(), + status='created') + create_notification(template=template, + created_at=datetime.utcnow(), + status='sending') + create_notification(template=template, created_at=datetime.utcnow(), status='delivered') + + results = notifications_not_yet_sent(older_than, notification_type) + assert len(results) == 0