Add celery task to check job finished

- celery task called after countdown of 60 minutes from start of job processing

app/celery/tasks.py

@@ -1,5 +1,5 @@
 import json
-from datetime import (datetime)
+from datetime import datetime
 from collections import namedtuple
 
 from flask import current_app
@@ -54,6 +54,7 @@ from app.models import (
 from app.notifications.process_notifications import persist_notification
 from app.service.utils import service_allowed_to_send_to
 from app.statsd_decorators import statsd
+from app.v2.errors import JobIncompleteError
 from notifications_utils.s3 import s3upload
 
 
@@ -88,6 +89,9 @@ def process_job(job_id):
 
     current_app.logger.info("Starting job {} processing {} notifications".format(job_id, job.notification_count))
 
+    if template.template_type != LETTER_TYPE:
+        check_job_status.apply_async([str(job.id)], queue=QueueNames.JOBS)
+
     for row_number, recipient, personalisation in RecipientCSV(
             s3.get_job_from_s3(str(service.id), str(job_id)),
             template_type=template.template_type,
@@ -95,21 +99,14 @@ def process_job(job_id):
     ).enumerated_recipients_and_personalisation:
         process_row(row_number, recipient, personalisation, template, job, service)
 
-    notification_total_in_db = dao_get_total_notifications_for_job_id(job.id)
-
-    if job.notification_count != notification_total_in_db:
-        current_app.logger.error("Job {} is missing {} notifications".format(
-            job.id, notification_total_in_db - notification_total_in_db))
-        job.job_status = JOB_STATUS_ERROR
-    else:
-        if template.template_type == LETTER_TYPE:
-            if service.research_mode:
-                update_job_to_sent_to_dvla.apply_async([str(job.id)], queue=QueueNames.RESEARCH_MODE)
-            else:
-                build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)
-                current_app.logger.info("send job {} to build-dvla-file in the {} queue".format(job_id, QueueNames.JOBS))
+    if template.template_type == LETTER_TYPE:
+        if service.research_mode:
+            update_job_to_sent_to_dvla.apply_async([str(job.id)], queue=QueueNames.RESEARCH_MODE)
         else:
-            job.job_status = JOB_STATUS_FINISHED
+            build_dvla_file.apply_async([str(job.id)], queue=QueueNames.JOBS)
+            current_app.logger.info("send job {} to build-dvla-file in the {} queue".format(job_id, QueueNames.JOBS))
+    else:
+        job.job_status = JOB_STATUS_FINISHED
 
     finished = datetime.utcnow()
     job.processing_started = start
@@ -333,6 +330,14 @@ def update_dvla_job_to_error(self, job_id):
     current_app.logger.info("Updated {} job to {}".format(job_id, JOB_STATUS_ERROR))
 
 
+@notify_celery.task(bind=True, name='check-job-status', countdown=3600)
+@statsd(namespace="tasks")
+def check_job_status(self, job_id):
+    job = dao_get_job_by_id(job_id)
+    if job.job_status != JOB_STATUS_FINISHED:
+        raise JobIncompleteError("Job {} did not complete".format(job_id))
+
+
 @notify_celery.task(bind=True, name='update-letter-notifications-to-sent')
 @statsd(namespace="tasks")
 def update_letter_notifications_to_sent_to_dvla(self, notification_references):

app/config.py

@@ -19,6 +19,7 @@ if os.environ.get('VCAP_SERVICES'):
 
 
 class QueueNames(object):
+    CHECK_JOBS = 'check-job-status-tasks'
     PERIODIC = 'periodic-tasks'
     PRIORITY = 'priority-tasks'
     DATABASE = 'database-tasks'
@@ -34,6 +35,7 @@ class QueueNames(object):
     @staticmethod
     def all_queues():
         return [
+            QueueNames.CHECK_JOBS,
             QueueNames.PRIORITY,
             QueueNames.PERIODIC,
             QueueNames.DATABASE,

app/v2/errors.py

@@ -10,10 +10,11 @@ from app.errors import InvalidRequest
 class JobIncompleteError(Exception):
     def __init__(self, message):
         self.message = message
+        self.status_code = 500
 
     def to_dict_v2(self):
         return {
-            'status_code': 500,
+            'status_code': self.status_code,
             "errors": [
                 {
                     "error": 'JobIncompleteError',