Merge pull request #1765 from alphagov/download_link-activity-page

Download link activity page

app/main/views/jobs.py

@@ -1,6 +1,4 @@
 # -*- coding: utf-8 -*-
-from orderedset import OrderedSet
-from itertools import chain
 
 from flask import (
     render_template,
@@ -18,7 +16,6 @@ from notifications_utils.template import (
     Template,
     WithSubjectTemplate,
 )
-from werkzeug.datastructures import MultiDict
 
 from app import (
     job_api_client,
@@ -35,39 +32,12 @@ from app.utils import (
     user_has_permissions,
     generate_notifications_csv,
     get_time_left,
-    REQUESTED_STATUSES,
-    FAILURE_STATUSES,
-    SENDING_STATUSES,
-    DELIVERED_STATUSES,
     get_letter_timings,
+    parse_filter_args, set_status_filters
 )
 from app.statistics_utils import add_rate_to_job
 
 
-def _parse_filter_args(filter_dict):
-    if not isinstance(filter_dict, MultiDict):
-        filter_dict = MultiDict(filter_dict)
-
-    return MultiDict(
-        (
-            key,
-            (','.join(filter_dict.getlist(key))).split(',')
-        )
-        for key in filter_dict.keys()
-        if ''.join(filter_dict.getlist(key))
-    )
-
-
-def _set_status_filters(filter_args):
-    status_filters = filter_args.get('status', [])
-    return list(OrderedSet(chain(
-        (status_filters or REQUESTED_STATUSES),
-        DELIVERED_STATUSES if 'delivered' in status_filters else [],
-        SENDING_STATUSES if 'sending' in status_filters else [],
-        FAILURE_STATUSES if 'failed' in status_filters else []
-    )))
-
-
 @main.route("/services/<service_id>/jobs")
 @login_required
 @user_has_permissions('view_activity', admin_override=True)
@@ -104,8 +74,8 @@ def view_job(service_id, job_id):
     if job['job_status'] == 'cancelled':
         abort(404)
 
-    filter_args = _parse_filter_args(request.args)
+    filter_args = parse_filter_args(request.args)
-    filter_args['status'] = _set_status_filters(filter_args)
+    filter_args['status'] = set_status_filters(filter_args)
 
     total_notifications = job.get('notification_count', 0)
     processed_notifications = job.get('notifications_delivered', 0) + job.get('notifications_failed', 0)
@@ -146,8 +116,8 @@ def view_job_csv(service_id, job_id):
         template_id=job['template'],
         version=job['template_version']
     )['data']
-    filter_args = _parse_filter_args(request.args)
+    filter_args = parse_filter_args(request.args)
-    filter_args['status'] = _set_status_filters(filter_args)
+    filter_args['status'] = set_status_filters(filter_args)
 
     return Response(
         stream_with_context(
@@ -206,6 +176,12 @@ def view_notifications(service_id, message_type):
         page=request.args.get('page', 1),
         to=request.form.get('to', ''),
         search_form=SearchNotificationsForm(to=request.form.get('to', '')),
+        download_link=url_for(
+            '.download_notifications_csv',
+            service_id=current_service['id'],
+            message_type=message_type,
+            status=request.args.get('status')
+        )
     )
 
 
@@ -226,8 +202,8 @@ def get_notifications(service_id, message_type, status_override=None):
         abort(404, "Invalid page argument ({}) reverting to page 1.".format(request.args['page'], None))
     if message_type not in ['email', 'sms', 'letter']:
         abort(404)
-    filter_args = _parse_filter_args(request.args)
+    filter_args = parse_filter_args(request.args)
-    filter_args['status'] = _set_status_filters(filter_args)
+    filter_args['status'] = set_status_filters(filter_args)
     if request.path.endswith('csv'):
         return Response(
             generate_notifications_csv(
@@ -250,7 +226,6 @@ def get_notifications(service_id, message_type, status_override=None):
         limit_days=current_app.config['ACTIVITY_STATS_LIMIT_DAYS'],
         to=request.form.get('to', ''),
     )
-
     url_args = {
         'message_type': message_type,
         'status': request.args.get('status')
@@ -361,8 +336,8 @@ def _get_job_counts(job):
 
 
 def get_job_partials(job, template):
-    filter_args = _parse_filter_args(request.args)
+    filter_args = parse_filter_args(request.args)
-    filter_args['status'] = _set_status_filters(filter_args)
+    filter_args['status'] = set_status_filters(filter_args)
     notifications = notification_api_client.get_notifications_for_service(
         job['service'], job['id'], status=filter_args['status']
     )

app/main/views/notifications.py

@@ -1,18 +1,20 @@
 # -*- coding: utf-8 -*-
+from datetime import datetime
+
 from flask import (
     abort,
     render_template,
     jsonify,
     request,
     url_for,
-)
+    Response, stream_with_context)
 from flask_login import login_required
 
 from app import (
     notification_api_client,
     job_api_client,
-    current_service
+    current_service,
-)
+    format_date_numeric)
 from app.main import main
 from app.template_previews import TemplatePreview, get_page_count_for_letter
 from app.utils import (
@@ -23,7 +25,7 @@ from app.utils import (
     get_letter_timings,
     FAILURE_STATUSES,
     DELIVERED_STATUSES,
-)
+    generate_notifications_csv, parse_filter_args, set_status_filters)
 
 
 @main.route("/services/<service_id>/notification/<uuid:notification_id>")
@@ -138,3 +140,31 @@ def get_all_personalisation_from_notification(notification):
         notification['personalisation']['phone_number'] = notification['to']
 
     return notification['personalisation']
+
+
+@main.route("/services/<service_id>/download-notifications.csv")
+@login_required
+@user_has_permissions('view_activity', admin_override=True)
+def download_notifications_csv(service_id):
+    filter_args = parse_filter_args(request.args)
+    filter_args['status'] = set_status_filters(filter_args)
+
+    return Response(
+        stream_with_context(
+            generate_notifications_csv(
+                service_id=service_id,
+                job_id=None,
+                status=filter_args.get('status'),
+                page=request.args.get('page', 1),
+                page_size=5000,
+                format_for_csv=True
+            )
+        ),
+        mimetype='text/csv',
+        headers={
+            'Content-Disposition': 'inline; filename="{} - {} - {} report.csv"'.format(
+                format_date_numeric(datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ")),
+                filter_args['message_type'][0],
+                current_service['name'])
+        }
+    )

app/templates/views/notifications.html

@@ -43,7 +43,11 @@
       <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
     </form>
   {% endif %}
-
+    <p class="bottom-gutter">
+       <a href="{{ download_link }}" download="download" class="heading-small">Download this report</a>
+       &emsp;
+       Data available for 7 days
+    </p>
   {{ ajax_block(
     partials,
     url_for('.get_notifications_as_json', service_id=current_service.id, message_type=message_type, status=status, page=page),

app/utils.py

@@ -1,5 +1,7 @@
 import re
 import csv
+from itertools import chain
+
 import pytz
 from io import StringIO
 from os import path
@@ -28,7 +30,8 @@ from notifications_utils.template import (
     LetterImageTemplate,
     LetterPreviewTemplate,
 )
-
+from orderedset._orderedset import OrderedSet
+from werkzeug.datastructures import MultiDict
 
 SENDING_STATUSES = ['created', 'pending', 'sending']
 DELIVERED_STATUSES = ['delivered', 'sent']
@@ -125,28 +128,20 @@ def get_errors_for_csv(recipients, template_type):
 
 def generate_notifications_csv(**kwargs):
     from app import notification_api_client
+    fieldnames, list_of_keys = _create_key_and_fieldnames_for_csv(kwargs)
 
     if 'page' not in kwargs:
         kwargs['page'] = 1
-    fieldnames = ['Row number', 'Recipient', 'Template', 'Type', 'Job', 'Status', 'Time']
     yield ','.join(fieldnames) + '\n'
 
     while kwargs['page']:
         notifications_resp = notification_api_client.get_notifications_for_service(**kwargs)
         notifications = notifications_resp['notifications']
+
         for notification in notifications:
-            values = [
+            values = [notification[x] for x in list_of_keys]
-                notification['row_number'],
-                notification['recipient'],
-                notification['template_name'],
-                notification['template_type'],
-                notification['job_name'],
-                notification['status'],
-                notification['created_at']
-            ]
             line = ','.join(str(i) for i in values) + '\n'
             yield line
-
         if notifications_resp['links'].get('next'):
             kwargs['page'] += 1
         else:
@@ -154,6 +149,33 @@ def generate_notifications_csv(**kwargs):
     raise Exception("Should never reach here")
 
 
+def _create_key_and_fieldnames_for_csv(kwargs):
+    # if job_id then response looks different
+    if kwargs['job_id']:
+        list_of_keys = [
+            'row_number',
+            'recipient',
+            'template_name',
+            'template_type',
+            'job_name',
+            'status',
+            'created_at'
+        ]
+        fieldnames = ['Row number', 'Recipient', 'Template', 'Type', 'Job', 'Status', 'Time']
+    else:
+        list_of_keys = [
+            'recipient',
+            'template_name',
+            'template_type',
+            'job_name',
+            'status',
+            'created_at',
+            'updated_at'
+        ]
+        fieldnames = ['Recipient', 'Template', 'Type', 'Job', 'Status', 'Time', 'Completed at']
+    return fieldnames, list_of_keys
+
+
 def get_page_from_request():
     if 'page' in request.args:
         try:
@@ -382,3 +404,27 @@ def get_cdn_domain():
     domain = parsed_uri.netloc[len(subdomain + '.'):]
 
     return "static-logos.{}".format(domain)
+
+
+def parse_filter_args(filter_dict):
+    if not isinstance(filter_dict, MultiDict):
+        filter_dict = MultiDict(filter_dict)
+
+    return MultiDict(
+        (
+            key,
+            (','.join(filter_dict.getlist(key))).split(',')
+        )
+        for key in filter_dict.keys()
+        if ''.join(filter_dict.getlist(key))
+    )
+
+
+def set_status_filters(filter_args):
+    status_filters = filter_args.get('status', [])
+    return list(OrderedSet(chain(
+        (status_filters or REQUESTED_STATUSES),
+        DELIVERED_STATUSES if 'delivered' in status_filters else [],
+        SENDING_STATUSES if 'sending' in status_filters else [],
+        FAILURE_STATUSES if 'failed' in status_filters else []
+    )))

tests/app/test_utils.py

@@ -15,6 +15,7 @@ from app.utils import (
     get_letter_timings,
     get_cdn_domain
 )
+from tests.conftest import fake_uuid
 
 
 def _get_notifications_csv(
@@ -28,7 +29,8 @@ def _get_notifications_csv(
     status='Delivered',
     created_at='Thursday 19 April at 12:00',
     rows=1,
-    with_links=False
+    with_links=False,
+    job_id=fake_uuid
 ):
     links = {}
     if with_links:
@@ -58,7 +60,8 @@ def _get_notifications_csv(
 @pytest.fixture(scope='function')
 def _get_notifications_csv_mock(
     mocker,
-    api_user_active
+    api_user_active,
+    job_id=fake_uuid
 ):
     return mocker.patch(
         'app.notification_api_client.get_notifications_for_service',
@@ -122,13 +125,13 @@ def test_can_create_spreadsheet_from_dict_with_filename():
 
 
 def test_generate_notifications_csv_returns_correct_csv_file(_get_notifications_csv_mock):
-    csv_content = generate_notifications_csv(service_id='1234')
+    csv_content = generate_notifications_csv(service_id='1234', job_id=fake_uuid)
     csv_file = DictReader(StringIO('\n'.join(csv_content)))
     assert csv_file.fieldnames == ['Row number', 'Recipient', 'Template', 'Type', 'Job', 'Status', 'Time']
 
 
 def test_generate_notifications_csv_only_calls_once_if_no_next_link(_get_notifications_csv_mock):
-    list(generate_notifications_csv(service_id='1234'))
+    list(generate_notifications_csv(service_id='1234', job_id=fake_uuid))
 
     assert _get_notifications_csv_mock.call_count == 1
 
@@ -146,7 +149,7 @@ def test_generate_notifications_csv_calls_twice_if_next_link(mocker):
         ]
     )
 
-    csv_content = generate_notifications_csv(service_id=service_id)
+    csv_content = generate_notifications_csv(service_id=service_id, job_id=fake_uuid)
     csv = DictReader(StringIO('\n'.join(csv_content)))
 
     assert len(list(csv)) == 10