diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 5edc35b0b..d9639e86a 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -14,6 +14,7 @@ from app.celery.tasks import ( get_recipient_csv_and_template_and_sender_id, process_row ) +from app.celery.letters_pdf_tasks import create_letters_pdf from app.config import QueueNames, TaskNames from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago @@ -30,6 +31,7 @@ from app.dao.notifications_dao import ( notifications_not_yet_sent, dao_precompiled_letters_still_pending_virus_check, dao_old_letters_with_created_status, + letters_missing_from_sending_bucket ) from app.dao.provider_details_dao import ( get_current_provider, @@ -173,8 +175,8 @@ def check_job_status(): @notify_celery.task(name='replay-created-notifications') @statsd(namespace="tasks") def replay_created_notifications(): - # if the notification has not be send after 4 hours + 15 minutes, then try to resend. - resend_created_notifications_older_than = (60 * 60 * 4) + (60 * 15) + # if the notification has not be send after 1 hour, then try to resend. + resend_created_notifications_older_than = (60 * 60) for notification_type in (EMAIL_TYPE, SMS_TYPE): notifications_to_resend = notifications_not_yet_sent( resend_created_notifications_older_than, @@ -189,6 +191,20 @@ def replay_created_notifications(): for n in notifications_to_resend: send_notification_to_queue(notification=n, research_mode=n.service.research_mode) + # if the letter has not be send after an hour, then create a zendesk ticket + letters = letters_missing_from_sending_bucket(resend_created_notifications_older_than) + + if len(letters) > 0: + msg = "{} letters were created over an hour ago, " \ + "but do not have an updated_at timestamp or billable units. " \ + "\n Creating app.celery.letters_pdf_tasks.create_letters tasks to upload letter to S3 " \ + "and update notifications for the following notification ids: " \ + "\n {}".format(len(letters), [x.id for x in letters]) + + current_app.logger.info(msg) + for letter in letters: + create_letters_pdf.apply_async([letter.id], queue=QueueNames.LETTERS) + @notify_celery.task(name='check-precompiled-letter-state') @statsd(namespace="tasks") diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 8b8c20cb8..025b355e5 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -32,6 +32,7 @@ from app.models import ( Notification, NotificationHistory, ScheduledNotification, + KEY_TYPE_NORMAL, KEY_TYPE_TEST, LETTER_TYPE, NOTIFICATION_CREATED, @@ -46,7 +47,7 @@ from app.models import ( SMS_TYPE, EMAIL_TYPE, ServiceDataRetention, - Service + Service, ) from app.utils import get_london_midnight_in_utc from app.utils import midnight_n_days_ago, escape_special_characters @@ -698,15 +699,32 @@ def dao_old_letters_with_created_status(): last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0) notifications = Notification.query.filter( - Notification.updated_at < convert_bst_to_utc(last_processing_deadline), + Notification.created_at < convert_bst_to_utc(last_processing_deadline), Notification.notification_type == LETTER_TYPE, Notification.status == NOTIFICATION_CREATED ).order_by( - Notification.updated_at + Notification.created_at ).all() return notifications +def letters_missing_from_sending_bucket(seconds_to_subtract): + older_than_date = datetime.utcnow() - timedelta(seconds=seconds_to_subtract) + # We expect letters to have a `created` status, updated_at timestamp and billable units greater than zero. + notifications = Notification.query.filter( + Notification.billable_units == 0, + Notification.updated_at == None, # noqa + Notification.status == NOTIFICATION_CREATED, + Notification.created_at <= older_than_date, + Notification.notification_type == LETTER_TYPE, + Notification.key_type == KEY_TYPE_NORMAL + ).order_by( + Notification.created_at + ).all() + + return notifications + + def dao_precompiled_letters_still_pending_virus_check(): ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400) diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 7c4f1cc09..9450fc48a 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -285,7 +285,7 @@ def test_replay_created_notifications(notify_db_session, sample_service, mocker) sms_template = create_template(service=sample_service, template_type='sms') email_template = create_template(service=sample_service, template_type='email') - older_than = (60 * 60 * 4) + (60 * 15) # 4 hours 15 minutes + older_than = (60 * 60) + (60 * 15) # 1 hour 15 minutes # notifications expected to be resent old_sms = create_notification(template=sms_template, created_at=datetime.utcnow() - timedelta(seconds=older_than), status='created') @@ -309,6 +309,28 @@ def test_replay_created_notifications(notify_db_session, sample_service, mocker) queue="send-sms-tasks") +def test_replay_created_notifications_create_letters_pdf_tasks_for_letters_not_ready_to_send( + sample_letter_template, mocker +): + mock_task = mocker.patch('app.celery.scheduled_tasks.create_letters_pdf.apply_async') + create_notification(template=sample_letter_template, billable_units=0, + created_at=datetime.utcnow() - timedelta(hours=4)) + + create_notification(template=sample_letter_template, billable_units=0, + created_at=datetime.utcnow() - timedelta(minutes=20)) + notification_1 = create_notification(template=sample_letter_template, billable_units=0, + created_at=datetime.utcnow() - timedelta(hours=1, minutes=20)) + notification_2 = create_notification(template=sample_letter_template, billable_units=0, + created_at=datetime.utcnow() - timedelta(hours=5)) + + replay_created_notifications() + + calls = [call([notification_1.id], queue=QueueNames.LETTERS), + call([notification_2.id], queue=QueueNames.LETTERS), + ] + mock_task.assert_has_calls(calls, any_order=True) + + def test_check_job_status_task_does_not_raise_error(sample_template): create_job( template=sample_template, @@ -363,12 +385,12 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template) mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception') mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket') - noti_1 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 1, 12, 0)) - noti_2 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 16, 29)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 16, 30)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 17, 29)) - create_notification(template=sample_letter_template, status='delivered', updated_at=datetime(2019, 5, 28, 10, 0)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 30, 10, 0)) + noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0)) + noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 30)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 17, 29)) + create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0)) check_templated_letter_state() @@ -386,14 +408,14 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template) @freeze_time("2019-01-30 14:00:00") def test_check_templated_letter_state_during_utc(mocker, sample_letter_template): mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception') - mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket') + mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket') - noti_1 = create_notification(template=sample_letter_template, updated_at=datetime(2018, 12, 1, 12, 0)) - noti_2 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 17, 29)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 17, 30)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 18, 29)) - create_notification(template=sample_letter_template, status='delivered', updated_at=datetime(2019, 1, 29, 10, 0)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 30, 10, 0)) + noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0)) + noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 30)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 18, 29)) + create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0)) check_templated_letter_state()