Merge pull request #965 from alphagov/paginate-dashboard

Limit jobs on the dashboard

app/main/views/dashboard.py

@@ -1,6 +1,4 @@
 from datetime import datetime, timedelta
-from collections import namedtuple
-from itertools import groupby
 
 import dateutil
 from flask import (
@@ -20,7 +18,7 @@ from app import (
     service_api_client,
     template_statistics_client
 )
-from app.statistics_utils import add_rates_to, get_formatted_percentage, add_rate_to_jobs
+from app.statistics_utils import get_formatted_percentage, add_rate_to_job
 from app.utils import user_has_permissions
 
 
@@ -117,20 +115,21 @@ def aggregate_usage(template_statistics):
 
 
 def get_dashboard_partials(service_id):
+    # all but scheduled and cancelled
+    statuses_to_display = job_api_client.JOB_STATUSES - {'scheduled', 'cancelled'}
 
     template_statistics = aggregate_usage(
         template_statistics_client.get_template_statistics_for_service(service_id, limit_days=7)
     )
 
-    jobs = add_rate_to_jobs([
-        job for job in job_api_client.get_job(service_id, limit_days=7)['data']
-        if job['original_file_name'] != current_app.config['TEST_MESSAGE_FILENAME']
-    ])
-    scheduled_jobs = sorted([
-        job for job in jobs if job['job_status'] == 'scheduled'
-    ], key=lambda job: job['scheduled_for'])
+    scheduled_jobs = sorted(
+        job_api_client.get_jobs(service_id, statuses=['scheduled'])['data'],
+        key=lambda job: job['scheduled_for']
+    )
     immediate_jobs = [
-        job for job in jobs if job['job_status'] not in ['scheduled', 'cancelled']
+        add_rate_to_job(job)
+        for job in job_api_client.get_jobs(service_id, statuses=statuses_to_display)['data']
+        if job['original_file_name'] != current_app.config['TEST_MESSAGE_FILENAME']
     ]
     service = service_api_client.get_detailed_service(service_id)
 

app/main/views/jobs.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 import ago
-import time
 import dateutil
-import json
 from orderedset import OrderedSet
 from datetime import datetime, timedelta, timezone
 from itertools import chain
@@ -29,11 +27,13 @@ from app import (
 from app.main import main
 from app.utils import (
     get_page_from_request,
-    generate_previous_next_dict,
+    generate_next_dict,
+    generate_previous_dict,
     user_has_permissions,
-    generate_notifications_csv)
-from app.statistics_utils import add_rate_to_jobs
-from app.utils import get_help_argument
+    generate_notifications_csv,
+    get_help_argument
+)
+from app.statistics_utils import add_rate_to_job
 
 
 def _parse_filter_args(filter_dict):
@@ -65,12 +65,28 @@ def _set_status_filters(filter_args):
 @login_required
 @user_has_permissions('view_activity', admin_override=True)
 def view_jobs(service_id):
+    page = int(request.args.get('page', 1))
+    # all but scheduled and cancelled
+    statuses_to_display = job_api_client.JOB_STATUSES - {'scheduled', 'cancelled'}
+    jobs_response = job_api_client.get_jobs(service_id, statuses=statuses_to_display, page=page)
+    jobs = [
+        add_rate_to_job(job) for job in jobs_response['data']
+        if job['original_file_name'] != current_app.config['TEST_MESSAGE_FILENAME']
+    ]
+
+    prev_page = None
+    if jobs_response['links'].get('prev', None):
+        prev_page = generate_previous_dict('main.view_jobs', service_id, page)
+    next_page = None
+    if jobs_response['links'].get('next', None):
+        next_page = generate_next_dict('main.view_jobs', service_id, page)
+
     return render_template(
         'views/jobs/jobs.html',
-        jobs=add_rate_to_jobs([
-            job for job in job_api_client.get_job(service_id)['data']
-            if job['job_status'] not in ['scheduled', 'cancelled']
-        ])
+        jobs=jobs,
+        page=page,
+        prev_page=prev_page,
+        next_page=next_page,
     )
 
 
@@ -205,28 +221,18 @@ def get_notifications(service_id, message_type, status_override=None):
         template_type=[message_type],
         status=filter_args.get('status'),
         limit_days=current_app.config['ACTIVITY_STATS_LIMIT_DAYS'])
-    view_dict = dict(
-        message_type=message_type,
-        status=request.args.get('status')
-    )
+
+    url_args = {
+        'message_type': message_type,
+        'status': request.args.get('status')
+    }
     prev_page = None
     if notifications['links'].get('prev', None):
-        prev_page = generate_previous_next_dict(
-            'main.view_notifications',
-            service_id,
-            view_dict,
-            page - 1,
-            'Previous page',
-            'page {}'.format(page - 1))
+        prev_page = generate_previous_dict('main.view_notifications', service_id, page, url_args=url_args)
     next_page = None
     if notifications['links'].get('next', None):
-        next_page = generate_previous_next_dict(
-            'main.view_notifications',
-            service_id,
-            view_dict,
-            page + 1,
-            'Next page',
-            'page {}'.format(page + 1))
+        next_page = generate_next_dict('main.view_notifications', service_id, page, url_args)
+
     if request.path.endswith('csv'):
         csv_content = generate_notifications_csv(
             notification_api_client.get_notifications_for_service(

app/notify_client/job_api_client.py

@@ -5,6 +5,16 @@ from app.notify_client import _attach_current_user
 
 
 class JobApiClient(BaseAPIClient):
+
+    JOB_STATUSES = {
+        'scheduled',
+        'pending',
+        'in progress',
+        'finished',
+        'cancelled',
+        'sending limits exceeded',
+    }
+
     def __init__(self):
         super().__init__("a", "b", "c")
 
@@ -26,24 +36,24 @@ class JobApiClient(BaseAPIClient):
             results['requested'] += outcome['count']
         return results
 
-    def get_job(self, service_id, job_id=None, limit_days=None, status=None):
-        if job_id:
-            params = {}
-            if status is not None:
-                params['status'] = status
-            job = self.get(url='/service/{}/job/{}'.format(service_id, job_id), params=params)
-
-            stats = self.__convert_statistics(job['data'])
-            job['data']['notifications_sent'] = stats['delivered'] + stats['failed']
-            job['data']['notifications_delivered'] = stats['delivered']
-            job['data']['notifications_failed'] = stats['failed']
-            job['data']['notifications_requested'] = stats['requested']
-
-            return job
-
+    def get_job(self, service_id, job_id):
         params = {}
+        job = self.get(url='/service/{}/job/{}'.format(service_id, job_id), params=params)
+
+        stats = self.__convert_statistics(job['data'])
+        job['data']['notifications_sent'] = stats['delivered'] + stats['failed']
+        job['data']['notifications_delivered'] = stats['delivered']
+        job['data']['notifications_failed'] = stats['failed']
+        job['data']['notifications_requested'] = stats['requested']
+
+        return job
+
+    def get_jobs(self, service_id, limit_days=None, statuses=None, page=1):
+        params = {'page': page}
         if limit_days is not None:
             params['limit_days'] = limit_days
+        if statuses is not None:
+            params['statuses'] = ','.join(statuses)
 
         jobs = self.get(url='/service/{}/job'.format(service_id), params=params)
         for job in jobs['data']:

app/statistics_utils.py

@@ -74,17 +74,15 @@ def statistics_by_state(statistics):
 
 def get_failure_rate_for_job(job):
     if not job.get('notifications_delivered'):
-        if job.get('notifications_failed'):
-            return 1
-        return 0
+        return 1 if job.get('notifications_failed') else 0
     return (
         job.get('notifications_failed', 0) /
         (job.get('notifications_failed', 0) + job.get('notifications_delivered', 0))
     )
 
 
-def add_rate_to_jobs(jobs):
-    return [dict(
+def add_rate_to_job(job):
+    return dict(
         failure_rate=(get_failure_rate_for_job(job)) * 100,
         **job
-    ) for job in jobs]
+    )

app/templates/components/previous-next-navigation.html

@@ -8,7 +8,7 @@
             <span class="pagination-label">{{previous_page['label']}}</span>
           </a>
         </li>
-        {% endif %}
+      {% endif %}
       {% if next_page %}
         <li class="next-page">
           <a href="{{next_page['url']}}" rel="next">
@@ -19,4 +19,4 @@
       {% endif %}
     </ul>
   </nav>
-{% endmacro %}
+{% endmacro %}

app/templates/views/jobs/jobs.html

@@ -1,3 +1,4 @@
+{% from "components/previous-next-navigation.html" import previous_next_navigation %}
 {% extends "withnav_template.html" %}
 
 {% block page_title %}
@@ -8,5 +9,6 @@
     <h1 class="heading-large">Uploaded files</h1>
     <div class="dashboard">
       {% include 'views/dashboard/_jobs.html' %}
+      {{ previous_next_navigation(prev_page, next_page) }}
     </div>
 {% endblock %}

app/utils.py

@@ -118,21 +118,26 @@ def get_page_from_request():
         return 1
 
 
-def generate_previous_next_dict(view, service_id, view_dict, page, title, label):
-    if 'page' in view_dict:
-        view_dict.pop('page')
-    if 'service_id' in view_dict:
-        view_dict.pop('service_id')
+def generate_previous_dict(view, service_id, page, url_args=None):
+    return generate_previous_next_dict(view, service_id, page - 1, 'Previous page', url_args or {})
+
+
+def generate_next_dict(view, service_id, page, url_args=None):
+    return generate_previous_next_dict(view, service_id, page + 1, 'Next page', url_args or {})
+
+
+def generate_previous_next_dict(view, service_id, page, title, url_args):
     return {
-        'url': url_for(view, service_id=service_id, page=page, **view_dict),
+        'url': url_for(view, service_id=service_id, page=page, **url_args),
         'title': title,
-        'label': label
+        'label': 'page {}'.format(page)
     }
 
 
 def email_safe(string):
     return "".join([
-        character.lower() if character.isalnum() or character == "." else "" for character in re.sub("\s+", ".", string.strip())  # noqa
+        character.lower() if character.isalnum() or character == "." else ""
+        for character in re.sub(r"\s+", ".", string.strip())
     ])