Make a job model for individual jobs

This follows the pattern of what we’ve done with services, users and
events.

It gives us a better interface to the data we get back from the API than
dealing with the raw JSON directly.

Now is a good time to do this because we’re going to be making a bunch
of changes to the jobs pages, and those changes will be easier to code
and understand with a sesnsible model behind them.

app/main/views/jobs.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 
-from datetime import datetime
-
 from flask import (
     Response,
     abort,
@@ -15,11 +13,6 @@ from flask import (
 )
 from flask_login import current_user
 from notifications_python_client.errors import HTTPError
-from notifications_utils.letter_timings import (
-    CANCELLABLE_JOB_LETTER_STATUSES,
-    get_letter_timings,
-    letter_can_be_cancelled,
-)
 from notifications_utils.template import Template, WithSubjectTemplate
 
 from app import (
@@ -32,6 +25,7 @@ from app import (
 )
 from app.main import main
 from app.main.forms import SearchNotificationsForm
+from app.models.job import Job
 from app.statistics_utils import add_rate_to_job
 from app.utils import (
     generate_next_dict,
@@ -84,61 +78,45 @@ def view_jobs(service_id):
 @main.route("/services/<uuid:service_id>/jobs/<uuid:job_id>")
 @user_has_permissions()
 def view_job(service_id, job_id):
-    job = job_api_client.get_job(service_id, job_id)['data']
+    job = Job.from_id(job_id, service_id=current_service.id)
-    if job['job_status'] == 'cancelled':
+    if job.cancelled:
         abort(404)
 
     filter_args = parse_filter_args(request.args)
     filter_args['status'] = set_status_filters(filter_args)
 
-    total_notifications = job.get('notification_count', 0)
+    total_notifications = job.notification_count
-    processed_notifications = job.get('notifications_delivered', 0) + job.get('notifications_failed', 0)
+    processed_notifications = job.notifications_processed
-
-    template = service_api_client.get_service_template(
-        service_id=service_id,
-        template_id=job['template'],
-        version=job['template_version']
-    )['data']
 
     just_sent_message = 'Your {} been sent. Printing starts {} at 5:30pm.'.format(
-        'letter has' if job['notification_count'] == 1 else 'letters have',
+        'letter has' if job.notification_count == 1 else 'letters have',
         printing_today_or_tomorrow()
     )
-    partials = get_job_partials(job, template)
-    can_cancel_letter_job = partials["can_letter_job_be_cancelled"]
 
     return render_template(
         'views/jobs/job.html',
         finished=(total_notifications == processed_notifications),
-        uploaded_file_name=job['original_file_name'],
+        job=job,
-        template_id=job['template'],
-        job_id=job_id,
         status=request.args.get('status', ''),
         updates_url=url_for(
             ".view_job_updates",
             service_id=service_id,
-            job_id=job['id'],
+            job_id=job.id,
             status=request.args.get('status', ''),
         ),
-        partials=partials,
+        partials=get_job_partials(job),
         just_sent=bool(
             request.args.get('just_sent') == 'yes'
-            and template['template_type'] == 'letter'
+            and job.template_type == 'letter'
         ),
         just_sent_message=just_sent_message,
-        can_cancel_letter_job=can_cancel_letter_job,
     )
 
 
 @main.route("/services/<uuid:service_id>/jobs/<uuid:job_id>.csv")
 @user_has_permissions('view_activity')
 def view_job_csv(service_id, job_id):
-    job = job_api_client.get_job(service_id, job_id)['data']
+    job = Job.from_id(job_id, service_id=service_id)
-    template = service_api_client.get_service_template(
-        service_id=service_id,
-        template_id=job['template'],
-        version=job['template_version']
-    )['data']
     filter_args = parse_filter_args(request.args)
     filter_args['status'] = set_status_filters(filter_args)
 
@@ -151,14 +129,14 @@ def view_job_csv(service_id, job_id):
                 page=request.args.get('page', 1),
                 page_size=5000,
                 format_for_csv=True,
-                template_type=template['template_type'],
+                template_type=job.template_type,
             )
         ),
         mimetype='text/csv',
         headers={
             'Content-Disposition': 'inline; filename="{} - {}.csv"'.format(
-                template['name'],
+                job.template['name'],
-                format_datetime_short(job['created_at'])
+                format_datetime_short(job.created_at)
             )
         }
     )
@@ -167,7 +145,7 @@ def view_job_csv(service_id, job_id):
 @main.route("/services/<uuid:service_id>/jobs/<uuid:job_id>", methods=['POST'])
 @user_has_permissions('send_messages')
 def cancel_job(service_id, job_id):
-    job_api_client.cancel_job(service_id, job_id)
+    Job.from_id(job_id, service_id=service_id).cancel()
     return redirect(url_for('main.service_dashboard', service_id=service_id))
 
 
@@ -175,20 +153,18 @@ def cancel_job(service_id, job_id):
 @user_has_permissions()
 def cancel_letter_job(service_id, job_id):
     if request.method == 'POST':
-        job = job_api_client.get_job(service_id, job_id)['data']
+        job = Job.from_id(job_id, service_id=service_id)
-        notification_count = notification_api_client.get_notification_count_for_job_id(
+
-            service_id=service_id, job_id=job_id
+        if job.status != 'finished' or job.notifications_created < job.notification_count:
-        )
-        if job['job_status'] != 'finished' or notification_count < job['notification_count']:
             flash("We are still processing these letters, please try again in a minute.", 'try again')
             return view_job(service_id, job_id)
         try:
-            number_of_letters = job_api_client.cancel_letter_job(current_service.id, job_id)
+            number_of_letters = job.cancel()
         except HTTPError as e:
             flash(e.message, 'dangerous')
             return redirect(url_for('main.view_job', service_id=service_id, job_id=job_id))
         flash("Cancelled {} letters from {}".format(
-            format_thousands(number_of_letters), job['original_file_name']
+            format_thousands(number_of_letters), job.original_file_name
         ), 'default_with_tick')
         return redirect(url_for('main.service_dashboard', service_id=service_id))
 
@@ -200,16 +176,9 @@ def cancel_letter_job(service_id, job_id):
 @user_has_permissions()
 def view_job_updates(service_id, job_id):
 
-    job = job_api_client.get_job(service_id, job_id)['data']
+    job = Job.from_id(job_id, service_id=service_id)
 
-    return jsonify(**get_job_partials(
+    return jsonify(**get_job_partials(job))
-        job,
-        service_api_client.get_service_template(
-            service_id=current_service.id,
-            template_id=job['template'],
-            version=job['template_version']
-        )['data'],
-    ))
 
 
 @main.route('/services/<uuid:service_id>/notifications', methods=['GET', 'POST'])
@@ -386,61 +355,46 @@ def get_status_filters(service, message_type, statistics):
 
 
 def _get_job_counts(job):
-    sending = 0 if job['job_status'] == 'scheduled' else (
-        job.get('notification_count', 0) -
-        job.get('notifications_delivered', 0) -
-        job.get('notifications_failed', 0)
-    )
     return [
         (
             label,
             query_param,
             url_for(
                 ".view_job",
-                service_id=job['service'],
+                service_id=job.service,
-                job_id=job['id'],
+                job_id=job.id,
                 status=query_param,
             ),
             count
         ) for label, query_param, count in [
             [
                 'total', '',
-                job.get('notification_count', 0)
+                job.notification_count
             ],
             [
                 'sending', 'sending',
-                sending
+                job.notifications_sending
             ],
             [
                 'delivered', 'delivered',
-                job.get('notifications_delivered', 0)
+                job.notifications_delivered
             ],
             [
                 'failed', 'failed',
-                job.get('notifications_failed', 0)
+                job.notifications_failed
             ]
         ]
     ]
 
 
-def get_job_partials(job, template):
+def get_job_partials(job):
     filter_args = parse_filter_args(request.args)
     filter_args['status'] = set_status_filters(filter_args)
-    notifications = notification_api_client.get_notifications_for_service(
+    notifications = job.get_notifications(status=filter_args['status'])
-        job['service'], job['id'], status=filter_args['status']
+    if job.template_type == 'letter':
-    )
-
-    if template['template_type'] == 'letter':
-        # there might be no notifications if the job has only just been created and the tasks haven't run yet
-        if notifications['notifications']:
-            postage = notifications['notifications'][0]['postage']
-        else:
-            postage = template['postage']
-
         counts = render_template(
             'partials/jobs/count-letters.html',
-            total=job.get('notification_count', 0),
+            job=job,
-            delivery_estimate=get_letter_timings(job['created_at'], postage=postage).earliest_delivery,
         )
     else:
         counts = render_template(
@@ -448,22 +402,11 @@ def get_job_partials(job, template):
             counts=_get_job_counts(job),
             status=filter_args['status'],
             notifications_deleted=(
-                job['job_status'] == 'finished' and not notifications['notifications']
+                job.status == 'finished' and not notifications['notifications']
             ),
         )
-    service_data_retention_days = current_service.get_days_of_retention(template['template_type'])
+    service_data_retention_days = current_service.get_days_of_retention(job.template_type)
-    can_letter_job_be_cancelled = False
+
-    if template["template_type"] == "letter":
-        not_cancellable = [
-            n for n in notifications["notifications"] if n["status"] not in CANCELLABLE_JOB_LETTER_STATUSES
-        ]
-        job_created = job["created_at"][:-6]
-        if not letter_can_be_cancelled(
-            "created", datetime.strptime(job_created, '%Y-%m-%dT%H:%M:%S.%f')
-        ) or len(not_cancellable) != 0:
-            can_letter_job_be_cancelled = False
-        else:
-            can_letter_job_be_cancelled = True
     return {
         'counts': counts,
         'notifications': render_template(
@@ -472,26 +415,21 @@ def get_job_partials(job, template):
                 add_preview_of_content_to_notifications(notifications['notifications'])
             ),
             more_than_one_page=bool(notifications.get('links', {}).get('next')),
-            percentage_complete=(job['notifications_requested'] / job['notification_count'] * 100),
             download_link=url_for(
                 '.view_job_csv',
                 service_id=current_service.id,
-                job_id=job['id'],
+                job_id=job.id,
                 status=request.args.get('status')
             ),
-            time_left=get_time_left(job['created_at'], service_data_retention_days=service_data_retention_days),
+            time_left=get_time_left(job.created_at, service_data_retention_days=service_data_retention_days),
             job=job,
-            template=template,
-            template_version=job['template_version'],
             service_data_retention_days=service_data_retention_days,
         ),
         'status': render_template(
             'partials/jobs/status.html',
             job=job,
-            template_type=template["template_type"],
+            letter_print_day=get_letter_printing_statement("created", job.created_at)
-            letter_print_day=get_letter_printing_statement("created", job["created_at"])
         ),
-        'can_letter_job_be_cancelled': can_letter_job_be_cancelled,
     }
 
 

app/models/job.py

@@ -0,0 +1,159 @@
+from datetime import datetime
+
+from notifications_utils.letter_timings import (
+    CANCELLABLE_JOB_LETTER_STATUSES,
+    get_letter_timings,
+    letter_can_be_cancelled,
+)
+from werkzeug.utils import cached_property
+
+from app.models import JSONModel
+from app.notify_client.job_api_client import job_api_client
+from app.notify_client.notification_api_client import notification_api_client
+from app.notify_client.service_api_client import service_api_client
+from app.utils import set_status_filters
+
+
+class Job(JSONModel):
+
+    ALLOWED_PROPERTIES = {
+        'id',
+        'service',
+        'template',
+        'template_version',
+        'original_file_name',
+        'created_at',
+        'notification_count',
+        'notifications_sent',
+        'notifications_requested',
+        'job_status',
+        'statistics',
+        'created_by',
+        'scheduled_for',
+    }
+
+    @classmethod
+    def from_id(cls, job_id, service_id):
+        return cls(job_api_client.get_job(service_id, job_id)['data'])
+
+    @property
+    def status(self):
+        return self.job_status
+
+    @property
+    def cancelled(self):
+        return self.status == 'cancelled'
+
+    @property
+    def scheduled(self):
+        return self.status == 'scheduled'
+
+    @property
+    def notification_count(self):
+        return self._dict.get('notification_count', 0)
+
+    @property
+    def notifications_delivered(self):
+        return self._dict.get('notifications_delivered', 0)
+
+    @property
+    def notifications_failed(self):
+        return self._dict.get('notifications_failed', 0)
+
+    @property
+    def notifications_processed(self):
+        return self.notifications_delivered + self.notifications_failed
+
+    @property
+    def notifications_sending(self):
+        if self.scheduled:
+            return 0
+        return (
+            self.notification_count -
+            self.notifications_delivered -
+            self.notifications_failed
+        )
+
+    @property
+    def notifications_created(self):
+        return notification_api_client.get_notification_count_for_job_id(
+            service_id=self.service, job_id=self.id
+        )
+
+    @property
+    def still_processing(self):
+        return (
+            self.status != 'finished' or
+            self.notifications_created < self.notification_count
+        )
+
+    @property
+    def template_id(self):
+        return self._dict['template']
+
+    @cached_property
+    def template(self):
+        return service_api_client.get_service_template(
+            service_id=self.service,
+            template_id=self.template_id,
+            version=self.template_version,
+        )['data']
+
+    @property
+    def template_type(self):
+        return self.template['template_type']
+
+    @property
+    def percentage_complete(self):
+        return self.notifications_requested / self.notification_count * 100
+
+    @property
+    def letter_job_can_be_cancelled(self):
+
+        if self.template['template_type'] != 'letter':
+            return False
+
+        if any(self.uncancellable_notifications):
+            return False
+
+        if not letter_can_be_cancelled(
+            'created', datetime.strptime(self.created_at[:-6], '%Y-%m-%dT%H:%M:%S.%f')
+        ):
+            return False
+
+        return True
+
+    @cached_property
+    def all_notifications(self):
+        return self.get_notifications(set_status_filters({}))['notifications']
+
+    @property
+    def uncancellable_notifications(self):
+        return (
+            n for n in self.all_notifications
+            if n['status'] not in CANCELLABLE_JOB_LETTER_STATUSES
+        )
+
+    @cached_property
+    def postage(self):
+        # There might be no notifications if the job has only just been
+        # created and the tasks haven't run yet
+        try:
+            return self.all_notifications[0]['postage']
+        except IndexError:
+            return self.template['postage']
+
+    @property
+    def letter_timings(self):
+        return get_letter_timings(self.created_at, postage=self.postage)
+
+    def get_notifications(self, status):
+        return notification_api_client.get_notifications_for_service(
+            self.service, self.id, status=status,
+        )
+
+    def cancel(self):
+        if self.template_type == 'letter':
+            return job_api_client.cancel_letter_job(self.service, self.id)
+        else:
+            return job_api_client.cancel_job(self.service, self.id)

app/templates/partials/jobs/count-letters.html

@@ -5,8 +5,8 @@
   <div class="column-half">
     <div class="keyline-block">
       {{ big_number(
-        total,
+        job.notification_count,
-        message_count_label(total, 'letter', suffix='')|capitalize,
+        message_count_label(job.notification_count, 'letter', suffix='')|capitalize,
         smaller=True
       )}}
     </div>
@@ -14,7 +14,7 @@
   <div class="column-half">
     <div class="keyline-block">
       {{ big_number(
-        delivery_estimate|string|format_date_short,
+        job.letter_timings.earliest_delivery|string|format_date_short,
         'Estimated delivery date',
         smaller=True
       )}}

app/templates/partials/jobs/notifications.html

@@ -3,11 +3,11 @@
 {% from "components/form.html" import form_wrapper %}
 
 <div class="ajax-block-container" aria-labelledby='pill-selected-item'>
-  {% if job.job_status == 'scheduled' %}
+  {% if job.scheduled %}
 
     <p>
       Sending
-      <a href="{{ url_for('.view_template_version', service_id=current_service.id, template_id=template.id, version=template_version) }}">{{ template.name }}</a>
+      <a href="{{ url_for('.view_template_version', service_id=current_service.id, template_id=job.template.id, version=job.template_version) }}">{{ job.template.name }}</a>
       {{ job.scheduled_for|format_datetime_relative }}
     </p>
     <div class="page-footer">
@@ -28,12 +28,12 @@
       {% if template.template_type == 'letter' %}
         <div class="keyline-block bottom-gutter-1-2">
       {% endif %}
-        {% if percentage_complete < 100 and job.job_status != 'finished' %}
+        {% if job.percentage_complete < 100 and job.job_status != 'finished' %}
-          <p class="{% if template.template_type != 'letter' %}bottom-gutter{% endif %} hint">
+          <p class="{% if job.template.template_type != 'letter' %}bottom-gutter{% endif %} hint">
-            Report is {{ "{:.0f}%".format(percentage_complete * 0.99) }} complete…
+            Report is {{ "{:.0f}%".format(job.percentage_complete * 0.99) }} complete…
           </p>
         {% elif notifications %}
-          <p class="{% if template.template_type != 'letter' %}bottom-gutter{% endif %}">
+          <p class="{% if job.template.template_type != 'letter' %}bottom-gutter{% endif %}">
             <a href="{{ download_link }}" download class="heading-small">Download this report</a>
             &emsp;
             <span id="time-left">{{ time_left }}</span>

app/templates/partials/jobs/status.html

@@ -3,7 +3,7 @@
     {% if job.scheduled_for %}
       {% if job.processing_started %}
         Sent by {{ job.created_by.name }} on {{ job.processing_started|format_datetime_short }}
-        {% if template_type == "letter" %}
+        {% if job.template.template_type == "letter" %}
           <p id="printing-info">
             {{ letter_print_day }}
           </p>
@@ -13,7 +13,7 @@
       {% endif %}
     {% else %}
       Sent by {{ job.created_by.name }} on {{ job.created_at|format_datetime_short }}
-      {% if template_type == "letter" %}
+      {% if job.template.template_type == "letter" %}
         <p id="printing-info">
           {{ letter_print_day }}
         </p>

app/templates/views/jobs/job.html

@@ -4,13 +4,13 @@
 {% from "components/page-footer.html" import page_footer %}
 
 {% block service_page_title %}
-  {{ uploaded_file_name }}
+  {{ job.original_file_name }}
 {% endblock %}
 
 {% block maincolumn_content %}
 
     <h1 class="heading-large">
-      {{ uploaded_file_name }}
+      {{ job.original_file_name }}
     </h1>
 
     {% if just_sent %}
@@ -21,11 +21,11 @@
     {{ ajax_block(partials, updates_url, 'counts', finished=finished) }}
     {{ ajax_block(partials, updates_url, 'notifications', finished=finished) }}
 
-    {% if can_cancel_letter_job %}
+    {% if job.letter_job_can_be_cancelled %}
       <div class="js-stick-at-bottom-when-scrolling">
         <div class="page-footer">
           <span class="page-footer-delete-link page-footer-delete-link-without-button">
-              <a href="{{ url_for('main.cancel_letter_job', service_id=current_service.id, job_id=job_id) }}">Cancel sending these letters</a>
+              <a href="{{ url_for('main.cancel_letter_job', service_id=current_service.id, job_id=job.id) }}">Cancel sending these letters</a>
           </span>
     {% else %}
       <div>&nbsp;</div>

tests/__init__.py

@@ -380,7 +380,12 @@ def job_json(
         'notifications_sent': notifications_sent,
         'notifications_requested': notifications_requested,
         'job_status': job_status,
-        'statistics': [],
+        'statistics': [
+            {
+                'status': 'blah',
+                'count': notifications_requested,
+            }
+        ],
         'created_by': created_by_json(
             created_by['id'],
             created_by['name'],

tests/app/main/views/test_jobs.py

@@ -514,6 +514,8 @@ def test_should_show_scheduled_job(
 def test_should_cancel_job(
     client_request,
     fake_uuid,
+    mock_get_job,
+    mock_get_service_template,
     mocker,
 ):
     mock_cancel = mocker.patch('app.main.jobs.job_api_client.cancel_job')
@@ -549,6 +551,7 @@ def test_should_not_show_cancelled_job(
 def test_should_cancel_letter_job(
     client_request,
     mocker,
+    mock_get_service_letter_template,
     active_user_with_permissions
 ):
     job_id = str(uuid.uuid4())
@@ -564,7 +567,7 @@ def test_should_cancel_letter_job(
     mocker.patch('app.job_api_client.get_job', side_effect=[{"data": job}])
     mocker.patch('app.notification_api_client.get_notifications_for_service', return_value=notifications_json)
     mocker.patch('app.notification_api_client.get_notification_count_for_job_id', return_value=5)
-    mock_cancel = mocker.patch('app.main.jobs.job_api_client.cancel_letter_job', return_value=5)
+    mock_cancel = mocker.patch('app.job_api_client.cancel_letter_job', return_value=5)
     client_request.post(
         'main.cancel_letter_job',
         service_id=SERVICE_ONE_ID,
@@ -602,7 +605,7 @@ def test_should_not_show_cancel_link_for_letter_job_if_too_late(
     mocker.patch('app.job_api_client.get_job', side_effect=[{"data": job}])
     mocker.patch(
         'app.notification_api_client.get_notifications_for_service',
-        side_effect=[notifications_json]
+        return_value=notifications_json
     )
 
     page = client_request.get(
@@ -639,7 +642,7 @@ def test_should_show_cancel_link_for_letter_job(
     mocker.patch('app.job_api_client.get_job', side_effect=[{"data": job}])
     mocker.patch(
         'app.notification_api_client.get_notifications_for_service',
-        side_effect=[notifications_json]
+        return_value=notifications_json,
     )
 
     page = client_request.get(