Create a Zendesk ticket for letters in the wrong state

This creates a Zendesk ticket if either the
`check_precompiled_letter_state` or `check_templated_letter_state` tasks
fail.

app/celery/scheduled_tasks.py

@@ -8,7 +8,7 @@ from notifications_utils.statsd_decorators import statsd
 from sqlalchemy import and_
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import notify_celery
+from app import notify_celery, zendesk_client
 from app.celery.tasks import process_job
 from app.config import QueueNames, TaskNames
 from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
@@ -186,25 +186,39 @@ def replay_created_notifications():
 @statsd(namespace="tasks")
 def check_precompiled_letter_state():
     letters = dao_precompiled_letters_still_pending_virus_check()
-    letter_ids = [str(letter.id) for letter in letters]
 
     if len(letters) > 0:
-        current_app.logger.exception(
+        letter_ids = [str(letter.id) for letter in letters]
-            '{} precompiled letters have been pending-virus-check for over 90 minutes. Notifications: {}'.format(
+
-                len(letters),
+        msg = "{} precompiled letters have been pending-virus-check for over 90 minutes. " \
-                letter_ids)
+              "Notifications: {}".format(len(letters), letter_ids)
-        )
+
+        current_app.logger.exception(msg)
+
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still pending virus check".format(current_app.config['NOTIFY_ENVIRONMENT']),
+                message=msg,
+                ticket_type=zendesk_client.TYPE_INCIDENT
+            )
 
 
 @notify_celery.task(name='check-templated-letter-state')
 @statsd(namespace="tasks")
 def check_templated_letter_state():
     letters = dao_old_letters_with_created_status()
-    letter_ids = [str(letter.id) for letter in letters]
 
     if len(letters) > 0:
-        current_app.logger.exception(
+        letter_ids = [str(letter.id) for letter in letters]
-            "{} letters were created before 17.30 yesterday and still have 'created' state. Notifications: {}".format(
+
-                len(letters),
+        msg = "{} letters were created before 17.30 yesterday and still have 'created' status. " \
-                letter_ids)
+              "Notifications: {}".format(len(letters), letter_ids)
-        )
+
+        current_app.logger.exception(msg)
+
+        if current_app.config['NOTIFY_ENVIRONMENT'] in ['live', 'production', 'test']:
+            zendesk_client.create_ticket(
+                subject="[{}] Letters still in 'created' status".format(current_app.config['NOTIFY_ENVIRONMENT']),
+                message=msg,
+                ticket_type=zendesk_client.TYPE_INCIDENT
+            )

tests/app/celery/test_scheduled_tasks.py

@@ -62,7 +62,7 @@ def prepare_current_provider(restore_provider_details):
     db.session.commit()
 
 
-def test_should_call_delete_codes_on_delete_verify_codes_task(notify_api, mocker):
+def test_should_call_delete_codes_on_delete_verify_codes_task(notify_db_session, mocker):
     mocker.patch('app.celery.scheduled_tasks.delete_codes_older_created_more_than_a_day_ago')
     delete_verify_codes()
     assert scheduled_tasks.delete_codes_older_created_more_than_a_day_ago.call_count == 1
@@ -343,6 +343,7 @@ def test_check_job_status_task_does_not_raise_error(sample_template):
 @freeze_time("2019-05-30 14:00:00")
 def test_check_precompiled_letter_state(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
+    mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
     create_notification(template=sample_letter_template,
                         status=NOTIFICATION_PENDING_VIRUS_CHECK,
@@ -359,16 +360,21 @@ def test_check_precompiled_letter_state(mocker, sample_letter_template):
 
     check_precompiled_letter_state()
 
-    mock_logger.assert_called_once_with(
+    message = "2 precompiled letters have been pending-virus-check for over 90 minutes. " \
-        "2 precompiled letters have been pending-virus-check for over 90 minutes. Notifications: ['{}', '{}']".format(
+              "Notifications: ['{}', '{}']".format(noti_2.id, noti_1.id)
-            noti_2.id,
+
-            noti_1.id)
+    mock_logger.assert_called_once_with(message)
+    mock_create_ticket.assert_called_with(
+        message=message,
+        subject='[test] Letters still pending virus check',
+        ticket_type='incident'
     )
 
 
 @freeze_time("2019-05-30 14:00:00")
 def test_check_templated_letter_state_during_bst(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
+    mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
     noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0))
     noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29))
@@ -379,15 +385,21 @@ def test_check_templated_letter_state_during_bst(mocker, sample_letter_template)
 
     check_templated_letter_state()
 
-    mock_logger.assert_called_once_with(
+    message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
-        "2 letters were created before 17.30 yesterday and still have 'created' state. "
+              "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
-        "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
+
+    mock_logger.assert_called_once_with(message)
+    mock_create_ticket.assert_called_with(
+        message=message,
+        subject="[test] Letters still in 'created' status",
+        ticket_type='incident'
     )
 
 
 @freeze_time("2019-01-30 14:00:00")
 def test_check_templated_letter_state_during_utc(mocker, sample_letter_template):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
+    mock_create_ticket = mocker.patch('app.celery.nightly_tasks.zendesk_client.create_ticket')
 
     noti_1 = create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0))
     noti_2 = create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29))
@@ -398,7 +410,12 @@ def test_check_templated_letter_state_during_utc(mocker, sample_letter_template)
 
     check_templated_letter_state()
 
-    mock_logger.assert_called_once_with(
+    message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
-        "2 letters were created before 17.30 yesterday and still have 'created' state. "
+              "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
-        "Notifications: ['{}', '{}']".format(noti_1.id, noti_2.id)
+
+    mock_logger.assert_called_once_with(message)
+    mock_create_ticket.assert_called_with(
+        message=message,
+        subject="[test] Letters still in 'created' status",
+        ticket_type='incident'
     )