diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 383f80462..bc1391710 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -30,6 +30,7 @@ from app.dao.notifications_dao import ( notifications_not_yet_sent, dao_precompiled_letters_still_pending_virus_check, dao_old_letters_with_created_status, + letters_missing_from_sending_bucket ) from app.dao.provider_details_dao import ( get_current_provider, @@ -189,6 +190,25 @@ def replay_created_notifications(): for n in notifications_to_resend: send_notification_to_queue(notification=n, research_mode=n.service.research_mode) + # if the letter has not be send after 4 hours + 15 minutes, then create a zendesk ticket + letters = letters_missing_from_sending_bucket(resend_created_notifications_older_than) + + if len(letters) > 0: + msg = "{} letters were created four hours and 15 minutes ago, " \ + "but do not have an updated_at timestamp or billable units. " \ + "It is likely you need to run the " \ + "app.celery.letters_pdf_tasks.create_letters_pdf task again with " \ + "the notification id.\n {}".format(len(letters), + [x.id for x in letters]) + current_app.logger.info(msg) + if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']: + zendesk_client.create_ticket( + subject="[{}] Letters still in created status might be missing from S3".format( + current_app.config['NOTIFY_ENVIRONMENT']), + message=msg, + ticket_type=zendesk_client.TYPE_INCIDENT + ) + @notify_celery.task(name='check-precompiled-letter-state') @statsd(namespace="tasks") diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 8b8c20cb8..1a179481a 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -32,6 +32,7 @@ from app.models import ( Notification, NotificationHistory, ScheduledNotification, + KEY_TYPE_NORMAL, KEY_TYPE_TEST, LETTER_TYPE, NOTIFICATION_CREATED, @@ -46,7 +47,7 @@ from app.models import ( SMS_TYPE, EMAIL_TYPE, ServiceDataRetention, - Service + Service, ) from app.utils import get_london_midnight_in_utc from app.utils import midnight_n_days_ago, escape_special_characters @@ -698,15 +699,32 @@ def dao_old_letters_with_created_status(): last_processing_deadline = yesterday_bst.replace(hour=17, minute=30, second=0, microsecond=0) notifications = Notification.query.filter( - Notification.updated_at < convert_bst_to_utc(last_processing_deadline), + Notification.created_at < convert_bst_to_utc(last_processing_deadline), Notification.notification_type == LETTER_TYPE, Notification.status == NOTIFICATION_CREATED ).order_by( - Notification.updated_at + Notification.created_at ).all() return notifications +def letters_missing_from_sending_bucket(seconds_to_subtract): + older_than_date = datetime.utcnow() - timedelta(seconds=seconds_to_subtract) + + notifications = Notification.query.filter( + Notification.updated_at == None, # noqa + Notification.status == NOTIFICATION_CREATED, + Notification.billable_units == 0, + Notification.created_at <= older_than_date, + Notification.notification_type == LETTER_TYPE, + Notification.key_type == KEY_TYPE_NORMAL + ).order_by( + Notification.created_at + ).all() + + return notifications + + def dao_precompiled_letters_still_pending_virus_check(): ninety_minutes_ago = datetime.utcnow() - timedelta(seconds=5400) diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index f8d0f0a1d..29388eb6a 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -309,6 +309,34 @@ def test_replay_created_notifications(notify_db_session, sample_service, mocker) queue="send-sms-tasks") +def test_replay_created_notifications_create_zendesk_ticket_for_letters_not_ready_to_send( + sample_letter_template, mocker +): + mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket') + create_notification(template=sample_letter_template, billable_units=0, + created_at=datetime.utcnow() - timedelta(hours=4)) + + notification_1 = create_notification(template=sample_letter_template, billable_units=0, + created_at=datetime.utcnow() - timedelta(hours=4, minutes=20)) + notification_2 = create_notification(template=sample_letter_template, billable_units=0, + created_at=datetime.utcnow() - timedelta(hours=5)) + + replay_created_notifications() + + message = "{} letters were created four hours and 15 minutes ago, " \ + "but do not have an updated_at timestamp or billable units. " \ + "It is likely you need to run the " \ + "app.celery.letters_pdf_tasks.create_letters_pdf task again with " \ + "the notification id.\n {}".format(2, + [notification_2.id, notification_1.id]) + + mock_create_ticket.assert_called_with( + message=message, + subject='[test] Letters still in created status might be missing from S3', + ticket_type='incident' + ) + + def test_check_job_status_task_does_not_raise_error(sample_template): create_job( template=sample_template, @@ -363,12 +391,12 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template) mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception') mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket') - noti_1 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 1, 12, 0)) - noti_2 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 16, 29)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 16, 30)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 29, 17, 29)) - create_notification(template=sample_letter_template, status='delivered', updated_at=datetime(2019, 5, 28, 10, 0)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 5, 30, 10, 0)) + noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0)) + noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 30)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 17, 29)) + create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0)) check_templated_letter_state() @@ -386,14 +414,14 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template) @freeze_time("2019-01-30 14:00:00") def test_check_templated_letter_state_during_utc(mocker, sample_letter_template): mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception') - mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket') + mock_create_ticket = mocker.patch('app.celery.scheduled_tasks.zendesk_client.create_ticket') - noti_1 = create_notification(template=sample_letter_template, updated_at=datetime(2018, 12, 1, 12, 0)) - noti_2 = create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 17, 29)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 17, 30)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 29, 18, 29)) - create_notification(template=sample_letter_template, status='delivered', updated_at=datetime(2019, 1, 29, 10, 0)) - create_notification(template=sample_letter_template, updated_at=datetime(2019, 1, 30, 10, 0)) + noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0)) + noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 30)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 18, 29)) + create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0)) + create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0)) check_templated_letter_state()