Added a new scheduled task that runs every minute, may want to run it every 3 minutes.

The tasks checks that a job is not still in progress 30 minutes after the job started processing.

app/celery/scheduled_tasks.py

@@ -4,6 +4,7 @@ from datetime import (
 )
 
 from flask import current_app
+from sqlalchemy import or_, and_
 from sqlalchemy.exc import SQLAlchemyError
 from notifications_utils.s3 import s3upload
 
@@ -17,8 +18,8 @@ from app.dao.invited_user_dao import delete_invitations_created_more_than_two_da
 from app.dao.jobs_dao import (
     dao_get_letter_job_ids_by_status,
     dao_set_scheduled_jobs_to_pending,
-    dao_get_jobs_older_than_limited_by
-)
+    dao_get_jobs_older_than_limited_by,
+    dao_get_job_by_id)
 from app.dao.monthly_billing_dao import (
     get_service_ids_that_need_billing_populated,
     create_or_update_monthly_billing
@@ -37,12 +38,14 @@ from app.dao.provider_details_dao import (
     dao_toggle_sms_provider
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
-from app.models import LETTER_TYPE, JOB_STATUS_READY_TO_SEND
+from app.models import LETTER_TYPE, JOB_STATUS_READY_TO_SEND, JOB_STATUS_SENT_TO_DVLA, JOB_STATUS_FINISHED, Job, \
+    EMAIL_TYPE, SMS_TYPE, JOB_STATUS_IN_PROGRESS
 from app.notifications.process_notifications import send_notification_to_queue
 from app.statsd_decorators import statsd
 from app.celery.tasks import process_job, create_dvla_file_contents_for_notifications
 from app.config import QueueNames, TaskNames
 from app.utils import convert_utc_to_bst
+from app.v2.errors import JobIncompleteError
 
 
 @notify_celery.task(name="remove_csv_files")
@@ -354,3 +357,30 @@ def run_letter_api_notifications():
                 QueueNames.PROCESS_FTP
             )
         )
+
+
+@notify_celery.task(name='check-job-status')
+@statsd(namespace="tasks")
+def check_job_status():
+    """
+    every x minutes do this check
+    select
+    from jobs
+    where job_status == 'in progress'
+    and template_type in ('sms', 'email')
+    and scheduled_at or created_at is older that 30 minutes.
+    if any results then
+        raise error
+        process the rows in the csv thatgit c are missing (in another task) just do the check here.
+    """
+    thirty_minutes_ago = datetime.utcnow() - timedelta(minutes=30)
+    thirty_five_minutes_ago = datetime.utcnow() - timedelta(minutes=35)
+
+    jobs_not_complete_after_30_minutes = Job.query.filter(
+        Job.job_status == JOB_STATUS_IN_PROGRESS,
+        and_(thirty_five_minutes_ago < Job.processing_started, Job.processing_started < thirty_minutes_ago)
+    ).all()
+
+    job_ids = [str(x.id) for x in jobs_not_complete_after_30_minutes]
+    if job_ids:
+        raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids))