From d4a173c78cdab5cad16930731d85476546189bfc Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 19 Feb 2020 13:47:35 +0000 Subject: [PATCH 01/63] Upgrade itsdangerous Version 1.1.0 has reverted the breaking change (moving from sha1 to sha256) that was introduced in version 1.0.0. Upgrading now so that we can take advantage of this bug fix: https://github.com/pallets/itsdangerous/issues/46 --- requirements-app.txt | 4 +--- requirements.txt | 8 +++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/requirements-app.txt b/requirements-app.txt index d2bd53f6d..7ac863e6d 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -19,9 +19,7 @@ notifications-python-client==5.5.1 # PaaS awscli-cwlogs>=1.4,<1.5 - -# Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default -itsdangerous==0.24 # pyup: <1.0.0 +itsdangerous==1.1.0 git+https://github.com/alphagov/notifications-utils.git@36.6.0#egg=notifications-utils==36.6.0 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha diff --git a/requirements.txt b/requirements.txt index 89e522545..335cee4a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,18 +21,16 @@ notifications-python-client==5.5.1 # PaaS awscli-cwlogs>=1.4,<1.5 - -# Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default -itsdangerous==0.24 # pyup: <1.0.0 +itsdangerous==1.1.0 git+https://github.com/alphagov/notifications-utils.git@36.6.0#egg=notifications-utils==36.6.0 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha ## The following requirements were added by pip freeze: -awscli==1.17.15 +awscli==1.18.2 bleach==3.1.0 boto3==1.10.38 -botocore==1.14.15 +botocore==1.15.2 certifi==2019.11.28 chardet==3.0.4 Click==7.0 From 5b818dcd68e7aa8d897c0ab9200eb434afcc1586 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 16 Mar 2020 10:31:37 +0000 Subject: [PATCH 02/63] Increase all ajax calls to 5 seconds The default is 2 seconds and this will mean that we are halving traffic for these ajax calls which can only be good for trying to limit queries on the database. I think the user impact on this will likely not be noticable. Debatable whether we should up them all even further to 10 seconds but this is definitely a quick although maybe small win. --- app/templates/components/ajax-block.html | 2 +- app/templates/views/dashboard/dashboard.html | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/templates/components/ajax-block.html b/app/templates/components/ajax-block.html index 4d40a25b3..c0aa403d8 100644 --- a/app/templates/components/ajax-block.html +++ b/app/templates/components/ajax-block.html @@ -1,4 +1,4 @@ -{% macro ajax_block(partials, url, key, interval=2, finished=False, form='') %} +{% macro ajax_block(partials, url, key, interval=5, finished=False, form='') %} {% if not finished %}
In the last 7 days - {{ ajax_block(partials, updates_url, 'inbox', interval=5) }} + {{ ajax_block(partials, updates_url, 'inbox') }} - {{ ajax_block(partials, updates_url, 'totals', interval=5) }} + {{ ajax_block(partials, updates_url, 'totals') }} {{ show_more( url_for('.monthly', service_id=current_service.id), 'See messages sent per month' ) }} - {{ ajax_block(partials, updates_url, 'template-statistics', interval=5) }} + {{ ajax_block(partials, updates_url, 'template-statistics') }} {% if current_service.immediate_jobs and not current_service.has_permission('upload_letters') %} - {{ ajax_block(partials, updates_url, 'jobs', interval=5) }} + {{ ajax_block(partials, updates_url, 'jobs') }} {{ show_more( url_for('.view_jobs', service_id=current_service.id), 'See all uploaded files' From c7a56c8bdfea9234c293d387719fa26726abad16 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 12:08:38 +0000 Subject: [PATCH 03/63] Remove jobs from the dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’ve done this already for services with the upload letters permission. And all services can upload letters now. But we’re still returning it in the JSON response we use to AJAX-ify the page. Since the jobs response can query stats for up to 50 jobs at a time this puts some load on the API/database. Hopefully this might drop that load a bit. --- app/main/views/dashboard.py | 4 - app/templates/main_nav.html | 8 +- app/templates/views/dashboard/dashboard.html | 8 -- tests/app/main/test_permissions.py | 2 + tests/app/main/views/test_dashboard.py | 85 -------------------- tests/app/test_navigation.py | 2 +- 6 files changed, 5 insertions(+), 104 deletions(-) diff --git a/app/main/views/dashboard.py b/app/main/views/dashboard.py index b66eedf8f..9af826f07 100644 --- a/app/main/views/dashboard.py +++ b/app/main/views/dashboard.py @@ -309,10 +309,6 @@ def get_dashboard_partials(service_id): [row['count'] for row in template_statistics] or [0] ), ), - 'jobs': render_template( - 'views/dashboard/_jobs.html', - jobs=current_service.immediate_jobs, - ), 'usage': render_template( 'views/dashboard/_usage.html', **calculate_usage(yearly_usage, free_sms_allowance), diff --git a/app/templates/main_nav.html b/app/templates/main_nav.html index 8bc80dbad..3e5d8b2bd 100644 --- a/app/templates/main_nav.html +++ b/app/templates/main_nav.html @@ -8,14 +8,10 @@
  • Dashboard
    • {% endif %}
  • Templates
    • - {% if current_user.has_permissions('view_activity') %} -
  • Uploads
    • - {% else %} + {% if not current_user.has_permissions('view_activity') %}
  • Sent messages
    • - {% if current_service.has_jobs or current_service.can_upload_letters %} -
  • Uploads
    • - {% endif %} {% endif %} +
  • Uploads
  • Team members
    • {% if current_user.has_permissions('manage_service', allow_org_user=True) %}
  • Usage
    • diff --git a/app/templates/views/dashboard/dashboard.html b/app/templates/views/dashboard/dashboard.html index 0eee5be5d..fa8405690 100644 --- a/app/templates/views/dashboard/dashboard.html +++ b/app/templates/views/dashboard/dashboard.html @@ -35,14 +35,6 @@ {{ ajax_block(partials, updates_url, 'template-statistics') }} - {% if current_service.immediate_jobs and not current_service.has_permission('upload_letters') %} - {{ ajax_block(partials, updates_url, 'jobs') }} - {{ show_more( - url_for('.view_jobs', service_id=current_service.id), - 'See all uploaded files' - ) }} - {% endif %} - {% if current_user.has_permissions('manage_service') %}

    This year

    {{ ajax_block(partials, updates_url, 'usage') }} diff --git a/tests/app/main/test_permissions.py b/tests/app/main/test_permissions.py index 4156001f2..7a16e5012 100644 --- a/tests/app/main/test_permissions.py +++ b/tests/app/main/test_permissions.py @@ -384,6 +384,7 @@ def test_service_navigation_for_org_user( ( 'Templates', 'Sent messages', + 'Uploads', 'Team members', ), 403, @@ -393,6 +394,7 @@ def test_service_navigation_for_org_user( ( 'Templates', 'Sent messages', + 'Uploads', 'Team members', 'Usage', ), diff --git a/tests/app/main/views/test_dashboard.py b/tests/app/main/views/test_dashboard.py index 0675d9472..208cd1e20 100644 --- a/tests/app/main/views/test_dashboard.py +++ b/tests/app/main/views/test_dashboard.py @@ -1,7 +1,6 @@ import copy import json from datetime import datetime -from unittest.mock import call import pytest from bs4 import BeautifulSoup @@ -903,56 +902,6 @@ def test_correct_font_size_for_big_numbers( ) == 3 -@freeze_time("2016-01-01 11:09:00.061258") -def test_should_show_recent_jobs_on_dashboard( - client_request, - mock_get_service_templates, - mock_get_template_statistics, - mock_get_service_statistics, - mock_get_jobs, - mock_get_usage, - mock_get_free_sms_fragment_limit, - mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, -): - page = client_request.get( - 'main.service_dashboard', - service_id=SERVICE_ONE_ID, - ) - - third_call = mock_get_jobs.call_args_list[2] - assert third_call[0] == (SERVICE_ONE_ID,) - assert third_call[1]['limit_days'] == 7 - assert 'scheduled' not in third_call[1]['statuses'] - - table_rows = page.select_one('tbody').select('tr') - - assert len(table_rows) == 4 - - for index, filename in enumerate(( - "export 1/1/2016.xls", - "all email addresses.xlsx", - "applicants.ods", - "thisisatest.csv", - )): - assert filename in table_rows[index].find_all('th')[0].text - assert 'Sent today at 11:09' in table_rows[index].find_all('th')[0].text - assert normalize_spaces( - table_rows[index].select_one('td').text - ) == ( - '1 sending 0 delivered 0 failed' - ) - - -@pytest.mark.parametrize('extra_permissions', ( - pytest.param( - [], - marks=pytest.mark.xfail(raises=AssertionError), - ), - pytest.param( - ['upload_letters'] - ), -)) def test_should_not_show_jobs_on_dashboard_for_users_with_uploads_page( client_request, service_one, @@ -964,9 +913,7 @@ def test_should_not_show_jobs_on_dashboard_for_users_with_uploads_page( mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, mock_get_returned_letter_summary_with_no_returned_letters, - extra_permissions, ): - service_one['permissions'] += extra_permissions page = client_request.get( 'main.service_dashboard', service_id=SERVICE_ONE_ID, @@ -1546,38 +1493,6 @@ def test_get_tuples_of_financial_years_defaults_to_2015(): ))[0] -@freeze_time("2016-01-01 11:09:00.061258") -def test_should_show_all_jobs_with_valid_statuses( - logged_in_client, - mock_get_template_statistics, - mock_get_service_statistics, - mock_get_service_templates_when_no_templates_exist, - mock_get_jobs, - mock_get_usage, - mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, - mock_get_free_sms_fragment_limit, -): - logged_in_client.get(url_for('main.service_dashboard', service_id=SERVICE_ONE_ID)) - - first_call = mock_get_jobs.call_args_list[0] - # first call - checking for any jobs - assert first_call == call(SERVICE_ONE_ID) - second_call = mock_get_jobs.call_args_list[1] - # second call - scheduled jobs only - assert second_call == call(SERVICE_ONE_ID, statuses=['scheduled']) - # third call - everything but scheduled and cancelled - third_call = mock_get_jobs.call_args_list[2] - assert third_call == call(SERVICE_ONE_ID, limit_days=7, statuses={ - 'pending', - 'in progress', - 'finished', - 'sending limits exceeded', - 'ready to send', - 'sent to dvla' - }) - - def test_org_breadcrumbs_do_not_show_if_service_has_no_org( client_request, mock_get_template_statistics, diff --git a/tests/app/test_navigation.py b/tests/app/test_navigation.py index 55a6f8c02..14b95212c 100644 --- a/tests/app/test_navigation.py +++ b/tests/app/test_navigation.py @@ -191,7 +191,7 @@ def test_caseworkers_get_caseworking_navigation( ) page = client_request.get('main.choose_template', service_id=SERVICE_ONE_ID) assert normalize_spaces(page.select_one('header + .govuk-width-container nav').text) == ( - 'Templates Sent messages Team members' + 'Templates Sent messages Uploads Team members' ) From 1c02476ee7b3d66cf4db4f9b8616918084caeeda Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 12 Mar 2020 16:13:18 +0000 Subject: [PATCH 04/63] Let users upload a contact list to use later MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We increasingly have teams wanting to do business-continuity type messaging. They might be without access to their normal systems, which is where they would otherwise go to get the list of email addresses or phone numbers. So we want to give them a place in Notify where they can store their spreadsheets and use them at a later date. For the initial pass we’re going to scope this to only allowing spreadsheets with one column, ie just phone numbers/email addresses. This is because: - it minimises the amount of personal info we’re storing - it reduces the chance of getting a placeholder error when you go to send the message, which is probably a high-stress situation where you might not be able to re-generate the file The code for this is mostly copied from the existing upload CSV journey. It’s quite duplicative, but that’s what I needed to do to get this out quickly. There are opportunities for refactoring later. Similarly, I would have liked to split this up into better commit messages, but it really was a case of just bashing code out until it worked 😳 This commit does not: - implement the ‘view a contact list page’ (it just has a placeholder because the API isn’t ready at the moment) - link to this page (because it’s not ready to use yet) --- app/__init__.py | 2 + app/main/views/uploads.py | 159 ++++++- app/models/contact_list.py | 24 ++ app/models/service.py | 4 + app/navigation.py | 16 + app/notify_client/contact_list_api_client.py | 28 ++ app/s3_client/s3_csv_client.py | 10 + .../uploads/contact-list/column-errors.html | 91 ++++ .../views/uploads/contact-list/ok.html | 70 +++ .../uploads/contact-list/row-errors.html | 108 +++++ .../contact-list/too-many-columns.html | 112 +++++ .../views/uploads/contact-list/upload.html | 84 ++++ tests/app/main/views/test_uploads.py | 402 +++++++++++++++++- tests/conftest.py | 23 + 14 files changed, 1131 insertions(+), 2 deletions(-) create mode 100644 app/models/contact_list.py create mode 100644 app/notify_client/contact_list_api_client.py create mode 100644 app/templates/views/uploads/contact-list/column-errors.html create mode 100644 app/templates/views/uploads/contact-list/ok.html create mode 100644 app/templates/views/uploads/contact-list/row-errors.html create mode 100644 app/templates/views/uploads/contact-list/too-many-columns.html create mode 100644 app/templates/views/uploads/contact-list/upload.html diff --git a/app/__init__.py b/app/__init__.py index 625a697b0..79c315225 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -64,6 +64,7 @@ from app.notify_client import InviteTokenError from app.notify_client.api_key_api_client import api_key_api_client from app.notify_client.billing_api_client import billing_api_client from app.notify_client.complaint_api_client import complaint_api_client +from app.notify_client.contact_list_api_client import contact_list_api_client from app.notify_client.email_branding_client import email_branding_client from app.notify_client.events_api_client import events_api_client from app.notify_client.inbound_number_client import inbound_number_client @@ -140,6 +141,7 @@ def create_app(application): # API clients api_key_api_client, billing_api_client, + contact_list_api_client, complaint_api_client, email_branding_client, events_api_client, diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index a6b32f28d..ea9fc5770 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -1,25 +1,38 @@ import base64 +import itertools import json import urllib import uuid from io import BytesIO +from zipfile import BadZipFile from flask import ( abort, current_app, + flash, redirect, render_template, request, url_for, ) +from notifications_utils.columns import Columns from notifications_utils.pdf import pdf_page_count +from notifications_utils.recipients import RecipientCSV +from notifications_utils.sanitise_text import SanitiseASCII from PyPDF2.utils import PdfReadError from requests import RequestException +from xlrd.biffh import XLRDError +from xlrd.xldate import XLDateError from app import current_service, notification_api_client, service_api_client from app.extensions import antivirus_client from app.main import main -from app.main.forms import LetterUploadPostageForm, PDFUploadForm +from app.main.forms import CsvUploadForm, LetterUploadPostageForm, PDFUploadForm +from app.s3_client.s3_csv_client import ( + s3download, + s3upload, + set_metadata_on_csv_upload, +) from app.s3_client.s3_letter_upload_client import ( get_letter_metadata, get_letter_pdf_and_metadata, @@ -28,10 +41,13 @@ from app.s3_client.s3_letter_upload_client import ( ) from app.template_previews import TemplatePreview, sanitise_letter from app.utils import ( + Spreadsheet, generate_next_dict, generate_previous_dict, + get_errors_for_csv, get_letter_validation_error, get_template, + unicode_truncate, user_has_permissions, ) @@ -282,3 +298,144 @@ def send_uploaded_letter(service_id): service_id=service_id, notification_id=file_id, )) + + +@main.route("/services//upload-a-contact-list", methods=['GET', 'POST']) +@user_has_permissions('send_messages') +def upload_contact_list(service_id): + form = CsvUploadForm() + + if form.validate_on_submit(): + try: + upload_id = s3upload( + service_id, + Spreadsheet.from_file(form.file.data, filename=form.file.data.filename).as_dict, + current_app.config['AWS_REGION'], + ) + return redirect(url_for( + '.check_contact_list', + service_id=service_id, + upload_id=upload_id, + original_file_name=form.file.data.filename, + )) + except (UnicodeDecodeError, BadZipFile, XLRDError): + flash('Could not read {}. Try using a different file format.'.format( + form.file.data.filename + )) + except (XLDateError): + flash(( + '{} contains numbers or dates that Notify cannot understand. ' + 'Try formatting all columns as ‘text’ or export your file as CSV.' + ).format( + form.file.data.filename + )) + + return render_template( + 'views/uploads/contact-list/upload.html', + form=form, + ) + + +@main.route( + "/services//check-contact-list/", + methods=['GET', 'POST'], +) +@user_has_permissions('send_messages') +def check_contact_list(service_id, upload_id): + + form = CsvUploadForm() + + contents = s3download(service_id, upload_id).strip() + first_row = contents.splitlines()[0].strip().rstrip(',') if contents else '' + + template_type = { + 'emailaddress': 'email', + 'phonenumber': 'sms', + }.get(Columns.make_key(first_row)) + + original_file_name = SanitiseASCII.encode(request.args.get('original_file_name', '')) + + recipients = RecipientCSV( + contents, + template_type=template_type or 'sms', + whitelist=itertools.chain.from_iterable( + [user.name, user.mobile_number, user.email_address] + for user in current_service.active_users + ) if current_service.trial_mode else None, + international_sms=current_service.has_permission('international_sms'), + max_initial_rows_shown=50, + max_errors_shown=50, + ) + + non_empty_column_headers = list(filter(None, recipients.column_headers)) + + if len(non_empty_column_headers) > 1 or not template_type or not recipients: + return render_template( + 'views/uploads/contact-list/too-many-columns.html', + recipients=recipients, + original_file_name=original_file_name, + template_type=template_type, + form=form, + ) + + if recipients.too_many_rows or not len(recipients): + return render_template( + 'views/uploads/contact-list/column-errors.html', + recipients=recipients, + original_file_name=original_file_name, + form=form, + ) + + row_errors = get_errors_for_csv(recipients, template_type) + if row_errors: + return render_template( + 'views/uploads/contact-list/row-errors.html', + recipients=recipients, + original_file_name=original_file_name, + row_errors=row_errors, + form=form, + ) + + if recipients.has_errors: + return render_template( + 'views/uploads/contact-list/column-errors.html', + recipients=recipients, + original_file_name=original_file_name, + form=form, + ) + + metadata_kwargs = { + 'row_count': len(recipients), + 'valid': True, + 'original_file_name': unicode_truncate( + original_file_name, + 1600, + ), + 'template_type': template_type + } + + set_metadata_on_csv_upload(service_id, upload_id, **metadata_kwargs) + + return render_template( + 'views/uploads/contact-list/ok.html', + recipients=recipients, + original_file_name=original_file_name, + upload_id=upload_id, + ) + + +@main.route("/services//save-contact-list/", methods=['POST']) +@user_has_permissions('send_messages') +def save_contact_list(service_id, upload_id): + current_service.save_contact_list(upload_id) + return redirect(url_for( + '.contact_list', + service_id=current_service.id, + contact_list_id=upload_id, + )) + + +@main.route("/services//contact-list/", methods=['GET']) +@user_has_permissions('send_messages') +def contact_list(service_id, contact_list_id): + return 'page for contact list {}'.format(contact_list_id) diff --git a/app/models/contact_list.py b/app/models/contact_list.py new file mode 100644 index 000000000..67cabc6d9 --- /dev/null +++ b/app/models/contact_list.py @@ -0,0 +1,24 @@ +from flask import abort + +from app.models import JSONModel +from app.notify_client.contact_list_api_client import contact_list_api_client +from app.s3_client.s3_csv_client import get_csv_metadata + + +class ContactList(JSONModel): + + @classmethod + def create(cls, service_id, upload_id): + + metadata = get_csv_metadata(service_id, upload_id) + + if not metadata.get('valid'): + abort(403) + + return cls(contact_list_api_client.create_contact_list( + service_id=service_id, + upload_id=upload_id, + original_file_name=metadata['original_file_name'], + row_count=int(metadata['row_count']), + template_type=metadata['template_type'], + )) diff --git a/app/models/service.py b/app/models/service.py index 132ac157c..6b883724b 100644 --- a/app/models/service.py +++ b/app/models/service.py @@ -6,6 +6,7 @@ from notifications_utils.timezones import local_timezone from werkzeug.utils import cached_property from app.models import JSONModel +from app.models.contact_list import ContactList from app.models.job import ( ImmediateJobs, PaginatedJobs, @@ -132,6 +133,9 @@ class Service(JSONModel): return [] return ScheduledJobs(self.id) + def save_contact_list(self, upload_id): + return ContactList.create(self.id, upload_id) + @cached_property def invited_users(self): return InvitedUsers(self.id) diff --git a/app/navigation.py b/app/navigation.py index aae89b581..491d3989a 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -317,6 +317,10 @@ class HeaderNavigation(Navigation): 'template_history', 'template_usage', 'trial_mode', + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -385,6 +389,10 @@ class MainNavigation(Navigation): 'view_template_versions', }, 'uploads': { + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -661,6 +669,10 @@ class CaseworkNavigation(Navigation): 'uploads': { 'view_jobs', 'view_job', + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -1195,6 +1207,10 @@ class OrgNavigation(Navigation): 'two_factor_email_sent', 'update_email_branding', 'update_letter_branding', + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', diff --git a/app/notify_client/contact_list_api_client.py b/app/notify_client/contact_list_api_client.py new file mode 100644 index 000000000..67f55950f --- /dev/null +++ b/app/notify_client/contact_list_api_client.py @@ -0,0 +1,28 @@ +from app.notify_client import NotifyAdminAPIClient, _attach_current_user + + +class ContactListApiClient(NotifyAdminAPIClient): + + def create_contact_list( + self, + *, + service_id, + upload_id, + original_file_name, + row_count, + template_type, + ): + data = { + "id": upload_id, + "original_file_name": original_file_name, + "row_count": row_count, + "template_type": template_type, + } + + data = _attach_current_user(data) + job = self.post(url='/service/{}/contact-list'.format(service_id), data=data) + + return job + + +contact_list_api_client = ContactListApiClient() diff --git a/app/s3_client/s3_csv_client.py b/app/s3_client/s3_csv_client.py index f9d0b5b9f..7999426e1 100644 --- a/app/s3_client/s3_csv_client.py +++ b/app/s3_client/s3_csv_client.py @@ -55,3 +55,13 @@ def set_metadata_on_csv_upload(service_id, upload_id, **kwargs): }, MetadataDirective='REPLACE', ) + + +def get_csv_metadata(service_id, upload_id): + try: + key = get_csv_upload(service_id, upload_id) + return key.get()['Metadata'] + except botocore.exceptions.ClientError as e: + current_app.logger.error("Unable to download s3 file {}".format( + FILE_LOCATION_STRUCTURE.format(service_id, upload_id))) + raise e diff --git a/app/templates/views/uploads/contact-list/column-errors.html b/app/templates/views/uploads/contact-list/column-errors.html new file mode 100644 index 000000000..34dd9b13e --- /dev/null +++ b/app/templates/views/uploads/contact-list/column-errors.html @@ -0,0 +1,91 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/back-link/macro.njk" import govukBackLink %} +{% from "components/message-count-label.html" import message_count_label, recipient_count_label %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

    + Skip to file contents +

    +{% endmacro %} + +{% block service_page_title %} + Error +{% endblock %} + +{% block maincolumn_content %} + + {{ govukBackLink({ "href": url_for('main.upload_contact_list', service_id=current_service.id) }) }} + +
    + {% call banner_wrapper(type='dangerous') %} + + {% if recipients.too_many_rows %} + +

    + Your file has too many rows +

    +

    + Notify can store files up to + {{ "{:,}".format(recipients.max_rows) }} rows in size. Your + file has {{ "{:,}".format(recipients|length) }} rows. +

    + + {% elif not recipients.allowed_to_send_to %} + +

    + You cannot save + {{ 'this' if recipients|length == 1 else 'these' }} + {{ recipient_count_label(recipients|length, recipients.template_type) }} +

    +

    + In trial mode you can only + send to yourself and members of your team +

    + + {% endif %} + + {{ skip_to_file_contents() }} + + {% endcall %} +
    + + +
    +
    + {{ file_upload( + form.file, + action=url_for('.upload_contact_list', service_id=current_service.id), + button_text='Upload your file again' + ) }} +
    + Back to top +
    + +

    {{ original_file_name }}

    + + {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients.column_headers + ) %} + {{ index_field(item.index + 2) }} + {% for column in recipients.column_headers %} + {{ text_field(item[column].data or '') }} + {% endfor %} + {% endcall %} + + {% if recipients.displayed_rows|list|length < recipients|length %} +

    + Only showing the first {{ recipients.displayed_rows|list|length }} rows +

    + {% endif %} + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/ok.html b/app/templates/views/uploads/contact-list/ok.html new file mode 100644 index 000000000..0dfa68a4a --- /dev/null +++ b/app/templates/views/uploads/contact-list/ok.html @@ -0,0 +1,70 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/page-header.html" import page_header %} +{% from "components/message-count-label.html" import message_count_label, recipient_count_label %} +{% from "components/button/macro.njk" import govukButton %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

    + Skip to file contents +

    +{% endmacro %} + +{% block service_page_title %} + {{ original_file_name }} +{% endblock %} + +{% block maincolumn_content %} + + {{ page_header( + original_file_name, + back_link=url_for('main.upload_contact_list', service_id=current_service.id) + ) }} + +

    + {{ recipients|length|format_thousands }} {{ recipient_count_label(recipients|length, recipients.template_type) }} found +

    + +
    +
    + + {{ govukButton({ "text": "Save contact list" }) }} +
    +
    + +

    + File preview +

    + {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients.column_headers + ) %} + {{ index_field(item.index + 2) }} + {% for column in recipients.column_headers %} + {% if item[column].ignore %} + {{ text_field(item[column].data or '', status='default') }} + {% else %} + {{ text_field(item[column].data or '') }} + {% endif %} + {% endfor %} + {% if item[None].data %} + {% for column in item[None].data %} + {{ text_field(column, status='default') }} + {% endfor %} + {% endif %} + {% endcall %} + + {% if recipients.displayed_rows|list|length < recipients|length %} +

    + Only showing the first {{ recipients.displayed_rows|list|length }} rows +

    + {% endif %} + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/row-errors.html b/app/templates/views/uploads/contact-list/row-errors.html new file mode 100644 index 000000000..c164a5ad1 --- /dev/null +++ b/app/templates/views/uploads/contact-list/row-errors.html @@ -0,0 +1,108 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/back-link/macro.njk" import govukBackLink %} +{% from "components/message-count-label.html" import message_count_label %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

    + Skip to file contents +

    +{% endmacro %} + +{% block service_page_title %} + Error +{% endblock %} + +{% block maincolumn_content %} + + {{ govukBackLink({ "href": url_for('main.upload_contact_list', service_id=current_service.id) }) }} + +
    + {% call banner_wrapper(type='dangerous') %} + {% if row_errors|length == 1 %} +

    + There’s a problem with {{ original_file_name }} +

    +

    + You need to {{ row_errors[0] }}. +

    + {% else %} +

    + There are some problems with {{ original_file_name }} +

    +

    + You need to: +

    +
      + {% for error in row_errors %} +
    • {{ error }}
      • + {% endfor %} +
    + {% endif %} + {{ skip_to_file_contents() }} + {% endcall %} +
    + +
    +
    + {{ file_upload( + form.file, + action=url_for('.upload_contact_list', service_id=current_service.id), + button_text='Upload your file again' + ) }} +
    + Back to top +
    + + {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients.column_headers + ) %} + {% call index_field() %} + + {{ item.index + 2 }} + + {% endcall %} + {% for column in recipients.column_headers %} + {% if item[column].error and not recipients.missing_column_headers %} + {% call field() %} + + {{ item[column].error }} + {{ item[column].data if item[column].data != None }} + + {% endcall %} + {% elif item[column].ignore %} + {{ text_field(item[column].data or '', status='default') }} + {% else %} + {{ text_field(item[column].data or '') }} + {% endif %} + {% endfor %} + {% if item[None].data %} + {% for column in item[None].data %} + {{ text_field(column, status='default') }} + {% endfor %} + {% endif %} + {% endcall %} + + {% if recipients.displayed_rows|list|length < recipients|length %} + {% if recipients.displayed_rows|list|length < recipients.rows_with_errors|list|length %} +

    + Only showing the first {{ recipients.displayed_rows|list|length }} rows with errors +

    + {% else %} +

    + Only showing rows with errors +

    + {% endif %} + {% endif %} + + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/too-many-columns.html b/app/templates/views/uploads/contact-list/too-many-columns.html new file mode 100644 index 000000000..41ea59a86 --- /dev/null +++ b/app/templates/views/uploads/contact-list/too-many-columns.html @@ -0,0 +1,112 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/back-link/macro.njk" import govukBackLink %} +{% from "components/message-count-label.html" import message_count_label %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

    + Skip to file contents +

    +{% endmacro %} + +{% block service_page_title %} + Error +{% endblock %} + +{% block maincolumn_content %} + + {{ govukBackLink({ "href": url_for('main.upload_contact_list', service_id=current_service.id) }) }} + +
    + {% call banner_wrapper(type='dangerous') %} + + {% if not recipients|length %} + +

    + Your file is missing some rows +

    +

    + It needs at least one row of data + {%- if template_type %}.{% else %}, in a column called ‘email address’ or ‘phone number’.{% endif %} +

    + + {% elif recipients.column_headers|length == 1 %} + +

    + Your file needs a column called ‘email address’ or ‘phone number’. +

    +

    + Right now it has 1 column called ‘{{ recipients._raw_column_headers[0] }}’. +

    + + {% else %} + +

    + Your file has too many columns +

    +

    + It needs to have 1 column, called ‘email address’ or ‘phone number’. +

    +

    + Right now it has {{ recipients._raw_column_headers|length }} columns called {{ recipients._raw_column_headers | formatted_list }}. +

    + + {% endif %} + + {{ skip_to_file_contents() }} + + {% endcall %} +
    + + +
    +
    + {{ file_upload( + form.file, + action=url_for('.upload_contact_list', service_id=current_service.id), + button_text='Upload your file again' + ) }} +
    + Back to top +
    + + {% set column_headers = recipients._raw_column_headers if recipients.duplicate_recipient_column_headers else recipients.column_headers %} + +

    {{ original_file_name }}

    + +
    + {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients._raw_column_headers + ) %} + {{ index_field(item.index + 2) }} + {% for column in column_headers %} + {% if item[column].ignore %} + {{ text_field(item[column].data or '', status='default') }} + {% else %} + {{ text_field(item[column].data or '') }} + {% endif %} + {% endfor %} + {% if item[None].data %} + {% for column in item[None].data %} + {{ text_field(column, status='default') }} + {% endfor %} + {% endif %} + {% endcall %} +
    + + {% if recipients.displayed_rows|list|length < recipients|length %} +

    + Only showing the first {{ recipients.displayed_rows|list|length }} rows +

    + {% endif %} + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/upload.html b/app/templates/views/uploads/contact-list/upload.html new file mode 100644 index 000000000..7b6355a38 --- /dev/null +++ b/app/templates/views/uploads/contact-list/upload.html @@ -0,0 +1,84 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/page-header.html" import page_header %} +{% from "components/table.html" import list_table, text_field, index_field, index_field_heading %} + +{% block service_page_title %} + Upload an emergency contact list +{% endblock %} + +{% block maincolumn_content %} + {% if error %} + {% call banner_wrapper(type='dangerous') %} +

    {{ error.title }}

    + {% if error.detail %} +

    {{ error.detail | safe }}

    + {% endif %} + {% endcall %} + {% else %} + {{ page_header( + 'Upload an emergency contact list', + back_link=url_for('main.uploads', service_id=current_service.id) + ) }} +

    Upload a list of phone numbers or email addresses.

    +

    Don’t put members of the public in here.

    + {% endif %} + +
    + {{ file_upload( + form.file, + button_text='Upload your file again' if error else 'Choose file', + show_errors=False + )}} +
    + +

    Your file needs to look like one of these examples

    + +

    + Save your file as a + CSV, + TSV, + ODS, + or Microsoft Excel spreadsheet +

    + +
    +
    +
    + {% call(item, row_number) list_table( + [ + ['email address'], + ['test@example.gov.uk'], + ], + caption="Example", + caption_visible=False, + field_headings=['', 'A'] + ) %} + {{ index_field(row_number - 1) }} + {% for column in item %} + {{ text_field(column) }} + {% endfor %} + {% endcall %} +
    +
    +
    +
    + {% call(item, row_number) list_table( + [ + ['phone number'], + ['07700 900123'], + ], + caption="Example", + caption_visible=False, + field_headings=['', 'A'] + ) %} + {{ index_field(row_number - 1) }} + {% for column in item %} + {{ text_field(column) }} + {% endfor %} + {% endcall %} +
    +
    +
    +{% endblock %} diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 2033c5852..fa9b12d9d 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -1,6 +1,7 @@ import re import urllib -from unittest.mock import Mock +from io import BytesIO +from unittest.mock import ANY, Mock import pytest from flask import make_response, url_for @@ -697,3 +698,402 @@ def test_get_uploads_shows_pagination( 'Previous page ' 'page 0' ) + + +def test_upload_contact_list_page(client_request): + page = client_request.get( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + ) + assert 'action' not in page.select_one('form') + assert page.select_one('form input')['name'] == 'file' + assert page.select_one('form input')['type'] == 'file' + + assert normalize_spaces(page.select('.spreadsheet')[0].text) == ( + 'Example A ' + '1 email address ' + '2 test@example.gov.uk' + ) + assert normalize_spaces(page.select('.spreadsheet')[1].text) == ( + 'Example A ' + '1 phone number ' + '2 07700 900123' + ) + + +@pytest.mark.parametrize('file_contents, expected_error, expected_thead, expected_tbody,', [ + ( + """ + telephone,name + +447700900986 + """, + ( + 'Your file has too many columns ' + 'It needs to have 1 column, called ‘email address’ or ‘phone number’. ' + 'Right now it has 2 columns called ‘telephone’ and ‘name’. ' + 'Skip to file contents' + ), + 'Row in file 1 telephone name', + '2 +447700900986', + ), + ( + """ + phone number, email address + +447700900986, test@example.com + """, + ( + 'Your file has too many columns ' + 'It needs to have 1 column, called ‘email address’ or ‘phone number’. ' + 'Right now it has 2 columns called ‘phone number’ and ‘email address’. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number email address', + '2 +447700900986 test@example.com', + ), + ( + """ + email address + +447700900986 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 1 email address. ' + 'Skip to file contents' + ), + 'Row in file 1 email address', + '2 Not a valid email address +447700900986', + ), + ( + """ + phone number + test@example.com + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 1 phone number. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + '2 Must not contain letters or symbols test@example.com', + ), + ( + """ + phone number, phone number, PHONE_NUMBER + +447700900111,+447700900222,+447700900333, + """, + ( + 'Your file has too many columns ' + 'It needs to have 1 column, called ‘email address’ or ‘phone number’. ' + 'Right now it has 3 columns called ‘phone number’, ‘phone number’ and ‘PHONE_NUMBER’. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number phone number PHONE_NUMBER', + '2 +447700900333 +447700900333 +447700900333', + ), + ( + """ + phone number + """, + ( + 'Your file is missing some rows ' + 'It needs at least one row of data. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + '', + ), + ( + "+447700900986", + ( + 'Your file is missing some rows ' + 'It needs at least one row of data, in a column called ' + '‘email address’ or ‘phone number’. ' + 'Skip to file contents' + ), + 'Row in file 1 +447700900986', + '', + ), + ( + "", + ( + 'Your file is missing some rows ' + 'It needs at least one row of data, in a column called ' + '‘email address’ or ‘phone number’. ' + 'Skip to file contents' + ), + 'Row in file 1', + '', + ), + ( + """ + phone number + +447700900986 + + +447700900986 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to enter missing data in 1 row. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + ( + '3 Missing' + ) + ), + ( + """ + phone number + +447700900 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 1 phone number. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + '2 Not enough digits +447700900', + ), + ( + """ + email address + ok@example.com + bad@example1 + bad@example2 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 2 email addresses. ' + 'Skip to file contents' + ), + 'Row in file 1 email address', + ( + '3 Not a valid email address bad@example1 ' + '4 Not a valid email address bad@example2' + ), + ), +]) +def test_upload_csv_file_shows_error_banner( + client_request, + mocker, + mock_s3_upload, + mock_get_job_doesnt_exist, + mock_get_users_by_service, + fake_uuid, + file_contents, + expected_error, + expected_thead, + expected_tbody, +): + + mock_upload = mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) + mock_download = mocker.patch('app.main.views.uploads.s3download', return_value=file_contents) + + page = client_request.post( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + _data={'file': (BytesIO(''.encode('utf-8')), 'invalid.csv')}, + _follow_redirects=True, + ) + mock_upload.assert_called_once_with( + SERVICE_ONE_ID, {'data': '', 'file_name': 'invalid.csv'}, ANY, + ) + mock_download.assert_called_once_with(SERVICE_ONE_ID, fake_uuid) + + assert normalize_spaces(page.select_one('.banner-dangerous').text) == expected_error + + assert page.select_one('form')['action'] == url_for( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + ) + assert page.select_one('form input')['type'] == 'file' + + assert normalize_spaces(page.select_one('thead').text) == expected_thead + assert normalize_spaces(page.select_one('tbody').text) == expected_tbody + + +def test_upload_csv_file_shows_error_banner_for_too_many_rows( + client_request, + mocker, + mock_s3_upload, + mock_get_job_doesnt_exist, + mock_get_users_by_service, + fake_uuid, +): + + mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) + mocker.patch('app.main.views.uploads.s3download', return_value='\n'.join( + ['phone number'] + (['07700900986'] * 50001) + )) + + page = client_request.post( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + _data={'file': (BytesIO(''.encode('utf-8')), 'invalid.csv')}, + _follow_redirects=True, + ) + + assert normalize_spaces(page.select_one('.banner-dangerous').text) == ( + 'Your file has too many rows ' + 'Notify can store files up to 50,000 rows in size. ' + 'Your file has 50,001 rows. ' + 'Skip to file contents' + ) + assert len(page.select('tbody tr')) == 50 + assert normalize_spaces(page.select_one('.table-show-more-link').text) == ( + 'Only showing the first 50 rows' + ) + + +def test_upload_csv_shows_trial_mode_error( + client_request, + mock_get_users_by_service, + mock_get_job_doesnt_exist, + fake_uuid, + mocker +): + mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) + mocker.patch('app.main.views.uploads.s3download', return_value=( + 'phone number\n' + '07900900321' # Not in team + )) + + page = client_request.get( + 'main.check_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + _test_page_title=False, + ) + + assert normalize_spaces(page.select_one('.banner-dangerous').text) == ( + 'You cannot save this phone number ' + 'In trial mode you can only send to yourself and members of your team ' + 'Skip to file contents' + ) + assert page.select_one('.banner-dangerous a')['href'] == url_for( + 'main.trial_mode_new' + ) + + +def test_upload_csv_shows_ok_page( + client_request, + mock_get_live_service, + mock_get_users_by_service, + mock_get_job_doesnt_exist, + fake_uuid, + mocker +): + mocker.patch('app.main.views.uploads.s3download', return_value='\n'.join( + ['email address'] + ['test@example.com'] * 51 + )) + mock_metadata_set = mocker.patch('app.main.views.uploads.set_metadata_on_csv_upload') + + page = client_request.get( + 'main.check_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + original_file_name='good times.xlsx', + _test_page_title=False, + ) + + mock_metadata_set.assert_called_once_with( + SERVICE_ONE_ID, + fake_uuid, + row_count=51, + original_file_name='good times.xlsx', + template_type='email', + valid=True, + ) + + assert normalize_spaces(page.select_one('h1').text) == ( + 'good times.xlsx' + ) + assert normalize_spaces(page.select_one('main p').text) == ( + '51 email addresses found' + ) + assert page.select_one('form')['action'] == url_for( + 'main.save_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + ) + + assert normalize_spaces(page.select_one('form [type=submit]').text) == ( + 'Save contact list' + ) + assert normalize_spaces(page.select_one('thead').text) == ( + 'Row in file 1 email address' + ) + assert len(page.select('tbody tr')) == 50 + assert normalize_spaces(page.select_one('tbody tr').text) == ( + '2 test@example.com' + ) + assert normalize_spaces(page.select_one('.table-show-more-link').text) == ( + 'Only showing the first 50 rows' + ) + + +def test_save_contact_list( + mocker, + client_request, + fake_uuid, + mock_create_contact_list, +): + mocker.patch('app.models.contact_list.get_csv_metadata', return_value={ + 'row_count': 999, + 'valid': True, + 'original_file_name': 'example.csv', + 'template_type': 'email' + }) + client_request.post( + 'main.save_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + _expected_status=302, + _expected_redirect=url_for( + 'main.contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + _external=True, + ) + ) + mock_create_contact_list.assert_called_once_with( + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + original_file_name='example.csv', + row_count=999, + template_type='email', + ) + + +def test_cant_save_bad_contact_list( + mocker, + client_request, + fake_uuid, + mock_create_contact_list, +): + mocker.patch('app.models.contact_list.get_csv_metadata', return_value={ + 'row_count': 999, + 'valid': False, + 'original_file_name': 'example.csv', + 'template_type': 'email' + }) + client_request.post( + 'main.save_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + _expected_status=403, + ) + assert mock_create_contact_list.called is False + + +def test_view_contact_list( + client_request, + fake_uuid, +): + page = client_request.get( + 'main.contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + _test_page_title=False, + ) + assert page.text == 'page for contact list 6ce466d0-fd6a-11e5-82f5-e0accb9d11a6' diff --git a/tests/conftest.py b/tests/conftest.py index f01c9150c..d44aa12eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1819,6 +1819,29 @@ def mock_get_no_uploads(mocker, api_user_active): ) +@pytest.fixture(scope='function') +def mock_create_contact_list(mocker, api_user_active): + def _create( + service_id, + upload_id, + original_file_name, + row_count, + template_type, + ): + return { + 'service_id': service_id, + 'upload_id': upload_id, + 'original_file_name': original_file_name, + 'row_count': row_count, + 'template_type': template_type, + } + + return mocker.patch( + 'app.contact_list_api_client.create_contact_list', + side_effect=_create, + ) + + @pytest.fixture(scope='function') def mock_get_notifications( mocker, From 03f2368debae21d75d9d1d9adec8909e98ab6fb2 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 13 Mar 2020 13:53:18 +0000 Subject: [PATCH 05/63] Use the correct bucket for storing contact lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don’t want to muddy them up with the normal CSV uploads. I’ve tried to reuse the existing S3 code where possible because it’s well tested. Buckets have already been created. --- app/config.py | 6 +++ app/main/views/send.py | 2 +- app/main/views/uploads.py | 19 ++++----- app/models/contact_list.py | 50 +++++++++++++++++++++-- app/models/service.py | 4 -- app/s3_client/s3_csv_client.py | 35 ++++++++++------ app/utils.py | 7 ++++ tests/app/main/views/test_uploads.py | 43 +++++++++++++------ tests/app/s3_client/test_s3_csv_client.py | 2 +- 9 files changed, 121 insertions(+), 47 deletions(-) diff --git a/app/config.py b/app/config.py index 7b8984287..85a72ccb9 100644 --- a/app/config.py +++ b/app/config.py @@ -68,6 +68,7 @@ class Config(object): WTF_CSRF_ENABLED = True WTF_CSRF_TIME_LIMIT = None CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload' + CONTACT_LIST_UPLOAD_BUCKET_NAME = 'local-contact-list' ACTIVITY_STATS_LIMIT_DAYS = 7 TEST_MESSAGE_FILENAME = 'Report' @@ -98,6 +99,7 @@ class Development(Config): SESSION_PROTECTION = None STATSD_ENABLED = False CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload' + CONTACT_LIST_UPLOAD_BUCKET_NAME = 'development-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-tools' MOU_BUCKET_NAME = 'notify.tools-mou' TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters' @@ -121,6 +123,7 @@ class Test(Development): STATSD_ENABLED = False WTF_CSRF_ENABLED = False CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload' + CONTACT_LIST_UPLOAD_BUCKET_NAME = 'test-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test' MOU_BUCKET_NAME = 'test-mou' TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters' @@ -140,6 +143,7 @@ class Preview(Config): HEADER_COLOUR = '#F499BE' # $baby-pink STATSD_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload' + CONTACT_LIST_UPLOAD_BUCKET_NAME = 'preview-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview' MOU_BUCKET_NAME = 'notify.works-mou' TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters' @@ -158,6 +162,7 @@ class Staging(Config): HEADER_COLOUR = '#6F72AF' # $mauve STATSD_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'staging-notifications-csv-upload' + CONTACT_LIST_UPLOAD_BUCKET_NAME = 'staging-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging' MOU_BUCKET_NAME = 'staging-notify.works-mou' TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters' @@ -173,6 +178,7 @@ class Live(Config): HTTP_PROTOCOL = 'https' STATSD_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload' + CONTACT_LIST_UPLOAD_BUCKET_NAME = 'live-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-production' MOU_BUCKET_NAME = 'notifications.service.gov.uk-mou' TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' diff --git a/app/main/views/send.py b/app/main/views/send.py index dbfb3421b..f8833723b 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -151,7 +151,7 @@ def send_messages(service_id, template_id): try: upload_id = s3upload( service_id, - Spreadsheet.from_file(form.file.data, filename=form.file.data.filename).as_dict, + Spreadsheet.from_file_form(form).as_dict, current_app.config['AWS_REGION'] ) return redirect(url_for( diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index ea9fc5770..feedb58f5 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -28,11 +28,7 @@ from app import current_service, notification_api_client, service_api_client from app.extensions import antivirus_client from app.main import main from app.main.forms import CsvUploadForm, LetterUploadPostageForm, PDFUploadForm -from app.s3_client.s3_csv_client import ( - s3download, - s3upload, - set_metadata_on_csv_upload, -) +from app.models.contact_list import ContactList from app.s3_client.s3_letter_upload_client import ( get_letter_metadata, get_letter_pdf_and_metadata, @@ -307,10 +303,9 @@ def upload_contact_list(service_id): if form.validate_on_submit(): try: - upload_id = s3upload( - service_id, - Spreadsheet.from_file(form.file.data, filename=form.file.data.filename).as_dict, - current_app.config['AWS_REGION'], + upload_id = ContactList.upload( + current_service.id, + Spreadsheet.from_file_form(form).as_dict, ) return redirect(url_for( '.check_contact_list', @@ -345,7 +340,7 @@ def check_contact_list(service_id, upload_id): form = CsvUploadForm() - contents = s3download(service_id, upload_id).strip() + contents = ContactList.download(service_id, upload_id) first_row = contents.splitlines()[0].strip().rstrip(',') if contents else '' template_type = { @@ -414,7 +409,7 @@ def check_contact_list(service_id, upload_id): 'template_type': template_type } - set_metadata_on_csv_upload(service_id, upload_id, **metadata_kwargs) + ContactList.set_metadata(service_id, upload_id, **metadata_kwargs) return render_template( 'views/uploads/contact-list/ok.html', @@ -427,7 +422,7 @@ def check_contact_list(service_id, upload_id): @main.route("/services//save-contact-list/", methods=['POST']) @user_has_permissions('send_messages') def save_contact_list(service_id, upload_id): - current_service.save_contact_list(upload_id) + ContactList.create(current_service.id, upload_id) return redirect(url_for( '.contact_list', service_id=current_service.id, diff --git a/app/models/contact_list.py b/app/models/contact_list.py index 67cabc6d9..fb32a4327 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -1,16 +1,60 @@ -from flask import abort +from flask import abort, current_app +from notifications_utils.formatters import strip_whitespace from app.models import JSONModel from app.notify_client.contact_list_api_client import contact_list_api_client -from app.s3_client.s3_csv_client import get_csv_metadata +from app.s3_client.s3_csv_client import ( + get_csv_metadata, + s3download, + s3upload, + set_metadata_on_csv_upload, +) class ContactList(JSONModel): + @staticmethod + def get_bucket_name(): + return current_app.config['CONTACT_LIST_UPLOAD_BUCKET_NAME'] + + @staticmethod + def upload(service_id, file_dict): + return s3upload( + service_id, + file_dict, + current_app.config['AWS_REGION'], + bucket=ContactList.get_bucket_name(), + ) + + @staticmethod + def download(service_id, upload_id): + return strip_whitespace(s3download( + service_id, + upload_id, + bucket=ContactList.get_bucket_name(), + )) + + @staticmethod + def set_metadata(service_id, upload_id, **kwargs): + return set_metadata_on_csv_upload( + service_id, + upload_id, + bucket=ContactList.get_bucket_name(), + **kwargs, + ) + + @staticmethod + def get_metadata(service_id, upload_id): + return get_csv_metadata( + service_id, + upload_id, + bucket=ContactList.get_bucket_name(), + ) + @classmethod def create(cls, service_id, upload_id): - metadata = get_csv_metadata(service_id, upload_id) + metadata = cls.get_metadata(service_id, upload_id) if not metadata.get('valid'): abort(403) diff --git a/app/models/service.py b/app/models/service.py index 6b883724b..132ac157c 100644 --- a/app/models/service.py +++ b/app/models/service.py @@ -6,7 +6,6 @@ from notifications_utils.timezones import local_timezone from werkzeug.utils import cached_property from app.models import JSONModel -from app.models.contact_list import ContactList from app.models.job import ( ImmediateJobs, PaginatedJobs, @@ -133,9 +132,6 @@ class Service(JSONModel): return [] return ScheduledJobs(self.id) - def save_contact_list(self, upload_id): - return ContactList.create(self.id, upload_id) - @cached_property def invited_users(self): return InvitedUsers(self.id) diff --git a/app/s3_client/s3_csv_client.py b/app/s3_client/s3_csv_client.py index 7999426e1..8ddfb1a57 100644 --- a/app/s3_client/s3_csv_client.py +++ b/app/s3_client/s3_csv_client.py @@ -9,20 +9,20 @@ from app.s3_client.s3_logo_client import get_s3_object FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv' -def get_csv_location(service_id, upload_id): +def get_csv_location(service_id, upload_id, bucket=None): return ( - current_app.config['CSV_UPLOAD_BUCKET_NAME'], + bucket or current_app.config['CSV_UPLOAD_BUCKET_NAME'], FILE_LOCATION_STRUCTURE.format(service_id, upload_id), ) -def get_csv_upload(service_id, upload_id): - return get_s3_object(*get_csv_location(service_id, upload_id)) +def get_csv_upload(service_id, upload_id, bucket=None): + return get_s3_object(*get_csv_location(service_id, upload_id, bucket)) -def s3upload(service_id, filedata, region): +def s3upload(service_id, filedata, region, bucket=None): upload_id = str(uuid.uuid4()) - bucket_name, file_location = get_csv_location(service_id, upload_id) + bucket_name, file_location = get_csv_location(service_id, upload_id, bucket) utils_s3upload( filedata=filedata['data'], region=region, @@ -32,10 +32,10 @@ def s3upload(service_id, filedata, region): return upload_id -def s3download(service_id, upload_id): +def s3download(service_id, upload_id, bucket=None): contents = '' try: - key = get_csv_upload(service_id, upload_id) + key = get_csv_upload(service_id, upload_id, bucket) contents = key.get()['Body'].read().decode('utf-8') except botocore.exceptions.ClientError as e: current_app.logger.error("Unable to download s3 file {}".format( @@ -44,11 +44,11 @@ def s3download(service_id, upload_id): return contents -def set_metadata_on_csv_upload(service_id, upload_id, **kwargs): +def set_metadata_on_csv_upload(service_id, upload_id, bucket=None, **kwargs): get_csv_upload( - service_id, upload_id + service_id, upload_id, bucket=bucket ).copy_from( - CopySource='{}/{}'.format(*get_csv_location(service_id, upload_id)), + CopySource='{}/{}'.format(*get_csv_location(service_id, upload_id, bucket=bucket)), ServerSideEncryption='AES256', Metadata={ key: str(value) for key, value in kwargs.items() @@ -57,9 +57,18 @@ def set_metadata_on_csv_upload(service_id, upload_id, **kwargs): ) -def get_csv_metadata(service_id, upload_id): +def set_metadata_on_contact_list(service_id, upload_id, **kwargs): + return set_metadata_on_csv_upload( + service_id, + upload_id, + bucket=current_app.config['CONTACT_LIST_UPLOAD_BUCKET_NAME'], + **kwargs, + ) + + +def get_csv_metadata(service_id, upload_id, bucket=None): try: - key = get_csv_upload(service_id, upload_id) + key = get_csv_upload(service_id, upload_id, bucket) return key.get()['Metadata'] except botocore.exceptions.ClientError as e: current_app.logger.error("Unable to download s3 file {}".format( diff --git a/app/utils.py b/app/utils.py index 07d7b78c2..0fae647f0 100644 --- a/app/utils.py +++ b/app/utils.py @@ -323,6 +323,13 @@ class Spreadsheet(): pyexcel.free_resources() return instance + @classmethod + def from_file_form(cls, form): + return cls.from_file( + form.file.data, + filename=form.file.data.filename, + ) + @property def as_rows(self): if not self._rows: diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index fa9b12d9d..de794df60 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -885,9 +885,14 @@ def test_upload_csv_file_shows_error_banner( expected_thead, expected_tbody, ): - - mock_upload = mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) - mock_download = mocker.patch('app.main.views.uploads.s3download', return_value=file_contents) + mock_upload = mocker.patch( + 'app.models.contact_list.s3upload', + return_value=fake_uuid, + ) + mock_download = mocker.patch( + 'app.models.contact_list.s3download', + return_value=file_contents, + ) page = client_request.post( 'main.upload_contact_list', @@ -896,9 +901,16 @@ def test_upload_csv_file_shows_error_banner( _follow_redirects=True, ) mock_upload.assert_called_once_with( - SERVICE_ONE_ID, {'data': '', 'file_name': 'invalid.csv'}, ANY, + SERVICE_ONE_ID, + {'data': '', 'file_name': 'invalid.csv'}, + ANY, + bucket='test-contact-list', + ) + mock_download.assert_called_once_with( + SERVICE_ONE_ID, + fake_uuid, + bucket='test-contact-list', ) - mock_download.assert_called_once_with(SERVICE_ONE_ID, fake_uuid) assert normalize_spaces(page.select_one('.banner-dangerous').text) == expected_error @@ -920,9 +932,8 @@ def test_upload_csv_file_shows_error_banner_for_too_many_rows( mock_get_users_by_service, fake_uuid, ): - - mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) - mocker.patch('app.main.views.uploads.s3download', return_value='\n'.join( + mocker.patch('app.models.contact_list.s3upload', return_value=fake_uuid) + mocker.patch('app.models.contact_list.s3download', return_value='\n'.join( ['phone number'] + (['07700900986'] * 50001) )) @@ -952,8 +963,8 @@ def test_upload_csv_shows_trial_mode_error( fake_uuid, mocker ): - mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) - mocker.patch('app.main.views.uploads.s3download', return_value=( + mocker.patch('app.models.contact_list.s3upload', return_value=fake_uuid) + mocker.patch('app.models.contact_list.s3download', return_value=( 'phone number\n' '07900900321' # Not in team )) @@ -983,10 +994,10 @@ def test_upload_csv_shows_ok_page( fake_uuid, mocker ): - mocker.patch('app.main.views.uploads.s3download', return_value='\n'.join( + mocker.patch('app.models.contact_list.s3download', return_value='\n'.join( ['email address'] + ['test@example.com'] * 51 )) - mock_metadata_set = mocker.patch('app.main.views.uploads.set_metadata_on_csv_upload') + mock_metadata_set = mocker.patch('app.models.contact_list.set_metadata_on_csv_upload') page = client_request.get( 'main.check_contact_list', @@ -999,6 +1010,7 @@ def test_upload_csv_shows_ok_page( mock_metadata_set.assert_called_once_with( SERVICE_ONE_ID, fake_uuid, + bucket='test-contact-list', row_count=51, original_file_name='good times.xlsx', template_type='email', @@ -1038,7 +1050,7 @@ def test_save_contact_list( fake_uuid, mock_create_contact_list, ): - mocker.patch('app.models.contact_list.get_csv_metadata', return_value={ + mock_get_metadata = mocker.patch('app.models.contact_list.get_csv_metadata', return_value={ 'row_count': 999, 'valid': True, 'original_file_name': 'example.csv', @@ -1056,6 +1068,11 @@ def test_save_contact_list( _external=True, ) ) + mock_get_metadata.assert_called_once_with( + SERVICE_ONE_ID, + fake_uuid, + bucket='test-contact-list', + ) mock_create_contact_list.assert_called_once_with( service_id=SERVICE_ONE_ID, upload_id=fake_uuid, diff --git a/tests/app/s3_client/test_s3_csv_client.py b/tests/app/s3_client/test_s3_csv_client.py index f4014065f..c8e07092e 100644 --- a/tests/app/s3_client/test_s3_csv_client.py +++ b/tests/app/s3_client/test_s3_csv_client.py @@ -12,7 +12,7 @@ def test_sets_metadata(client, mocker): set_metadata_on_csv_upload('1234', '5678', foo='bar', baz=True) - mocked_get_s3_object.assert_called_once_with('1234', '5678') + mocked_get_s3_object.assert_called_once_with('1234', '5678', bucket=None) mocked_s3_object.copy_from.assert_called_once_with( CopySource='test-notifications-csv-upload/service-1234-notify/5678.csv', Metadata={'baz': 'True', 'foo': 'bar'}, From 460b779d11d4f9e272f84746c91862171fd97d0e Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 13 Mar 2020 14:38:43 +0000 Subject: [PATCH 06/63] Add endpoint to redirect from contact list to send MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to use a contact list we’re going to put you in the normal upload a spreadsheet journey. Except without having to upload again. So this commit adds an endpoint that takes the file from the contact list bucket, puts it in the same place it would have gone if you’d uploaded it, then sends you on your way. --- app/main/views/send.py | 15 +++++++++++ app/models/contact_list.py | 16 ++++++++++++ app/navigation.py | 4 +++ tests/app/main/views/test_send.py | 41 +++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+) diff --git a/app/main/views/send.py b/app/main/views/send.py index f8833723b..8e1debe71 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -42,6 +42,7 @@ from app.main.forms import ( SetSenderForm, get_placeholder_form_instance, ) +from app.models.contact_list import ContactList from app.models.user import Users from app.s3_client.s3_csv_client import ( s3download, @@ -491,6 +492,20 @@ def send_test_preview(service_id, template_id, filetype): return TemplatePreview.from_utils_template(template, filetype, page=request.args.get('page')) +@main.route( + '/services//send/' + '/from-contact-list/' +) +@user_has_permissions('send_messages') +def send_from_contact_list(service_id, template_id, contact_list_id): + return redirect(url_for( + 'main.check_messages', + service_id=current_service.id, + template_id=template_id, + upload_id=ContactList.copy_to_uploads(service_id, contact_list_id), + )) + + def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_pdf=False): try: diff --git a/app/models/contact_list.py b/app/models/contact_list.py index fb32a4327..db20313f3 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -51,6 +51,22 @@ class ContactList(JSONModel): bucket=ContactList.get_bucket_name(), ) + @staticmethod + def copy_to_uploads(service_id, upload_id): + contents = ContactList.download(service_id, upload_id) + metadata = ContactList.get_metadata(service_id, upload_id) + new_upload_id = s3upload( + service_id, + contents, + current_app.config['AWS_REGION'], + ) + set_metadata_on_csv_upload( + service_id, + new_upload_id, + **metadata, + ) + return new_upload_id + @classmethod def create(cls, service_id, upload_id): diff --git a/app/navigation.py b/app/navigation.py index 491d3989a..a78c8ea84 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -259,6 +259,7 @@ class HeaderNavigation(Navigation): 'send_test', 'no_cookie.send_test_preview', 'send_test_step', + 'send_from_contact_list', 'send_uploaded_letter', 'service_add_email_reply_to', 'service_add_letter_contact', @@ -594,6 +595,7 @@ class MainNavigation(Navigation): 'robots', 'security', 'send_notification', + 'send_from_contact_list', 'send_uploaded_letter', 'service_dashboard_updates', 'service_delete_email_reply_to', @@ -847,6 +849,7 @@ class CaseworkNavigation(Navigation): 'send_messages', 'send_notification', 'no_cookie.send_test_preview', + 'send_from_contact_list', 'send_uploaded_letter', 'service_add_email_reply_to', 'service_add_letter_contact', @@ -1136,6 +1139,7 @@ class OrgNavigation(Navigation): 'send_test', 'no_cookie.send_test_preview', 'send_test_step', + 'send_from_contact_list', 'send_uploaded_letter', 'service_add_email_reply_to', 'service_add_letter_contact', diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 79d7f8758..23c31beb1 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -6,6 +6,7 @@ from glob import glob from io import BytesIO from itertools import repeat from os import path +from unittest.mock import ANY from uuid import uuid4 from zipfile import BadZipFile @@ -3780,3 +3781,43 @@ def test_redirects_to_template_if_job_exists_already( _external=True, ) ) + + +def test_send_from_contact_list( + mocker, + client_request, + fake_uuid, +): + new_uuid = uuid.uuid4() + mock_download = mocker.patch('app.models.contact_list.s3download', return_value='contents') + mock_get_metadata = mocker.patch('app.models.contact_list.get_csv_metadata', return_value={ + 'example_key': 'example value', + }) + mock_upload = mocker.patch('app.models.contact_list.s3upload', return_value=new_uuid) + mock_set_metadata = mocker.patch('app.models.contact_list.set_metadata_on_csv_upload') + client_request.get( + 'main.send_from_contact_list', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + contact_list_id=fake_uuid, + _expected_status=302, + _expected_redirect=url_for( + 'main.check_messages', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + upload_id=new_uuid, + _external=True, + ) + ) + mock_download.assert_called_once_with( + SERVICE_ONE_ID, fake_uuid, bucket='test-contact-list' + ) + mock_get_metadata.assert_called_once_with( + SERVICE_ONE_ID, fake_uuid, bucket='test-contact-list' + ) + mock_upload.assert_called_once_with( + SERVICE_ONE_ID, 'contents', ANY + ) + mock_set_metadata.assert_called_once_with( + SERVICE_ONE_ID, new_uuid, example_key='example value' + ) From f15b9b8d383a5c1603d2b51e2fe14e94eb054ada Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 13 Mar 2020 15:39:51 +0000 Subject: [PATCH 07/63] Rename URL for consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We say `…/upload-letter` not `…/upload-a-letter` --- app/main/views/uploads.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index feedb58f5..ba53d0ec6 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -296,7 +296,7 @@ def send_uploaded_letter(service_id): )) -@main.route("/services//upload-a-contact-list", methods=['GET', 'POST']) +@main.route("/services//upload-contact-list", methods=['GET', 'POST']) @user_has_permissions('send_messages') def upload_contact_list(service_id): form = CsvUploadForm() From aa85870ee1093a697a13ee07f7f8c33d33dba662 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 13 Mar 2020 16:58:43 +0000 Subject: [PATCH 08/63] Add a page to view a single contact list --- app/main/views/uploads.py | 5 +- app/models/contact_list.py | 29 ++++++++++++ app/notify_client/contact_list_api_client.py | 3 ++ .../uploads/contact-list/contact-list.html | 47 +++++++++++++++++++ tests/app/main/views/test_uploads.py | 44 ++++++++++++++++- tests/conftest.py | 27 +++++++++++ 6 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 app/templates/views/uploads/contact-list/contact-list.html diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index ba53d0ec6..b3a1021ef 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -433,4 +433,7 @@ def save_contact_list(service_id, upload_id): @main.route("/services//contact-list/", methods=['GET']) @user_has_permissions('send_messages') def contact_list(service_id, contact_list_id): - return 'page for contact list {}'.format(contact_list_id) + return render_template( + 'views/uploads/contact-list/contact-list.html', + contact_list=ContactList.from_id(contact_list_id, service_id=service_id), + ) diff --git a/app/models/contact_list.py b/app/models/contact_list.py index db20313f3..f61ee1861 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -1,5 +1,7 @@ from flask import abort, current_app from notifications_utils.formatters import strip_whitespace +from notifications_utils.recipients import RecipientCSV +from werkzeug.utils import cached_property from app.models import JSONModel from app.notify_client.contact_list_api_client import contact_list_api_client @@ -13,6 +15,24 @@ from app.s3_client.s3_csv_client import ( class ContactList(JSONModel): + ALLOWED_PROPERTIES = { + 'id', + 'created_at', + 'created_by', + 'service_id', + 'original_file_name', + 'row_count', + 'template_type', + } + + @classmethod + def from_id(cls, contact_list_id, *, service_id): + # This is temporary until we have a get single list endpoint + for contact_list in contact_list_api_client.get_contact_lists(service_id): + if contact_list['id'] == contact_list_id: + return cls(contact_list) + abort(404) + @staticmethod def get_bucket_name(): return current_app.config['CONTACT_LIST_UPLOAD_BUCKET_NAME'] @@ -82,3 +102,12 @@ class ContactList(JSONModel): row_count=int(metadata['row_count']), template_type=metadata['template_type'], )) + + @cached_property + def recipients(self): + return RecipientCSV( + self.download(self.service_id, self.id), + template_type=self.template_type, + international_sms=True, + max_initial_rows_shown=50, + ) diff --git a/app/notify_client/contact_list_api_client.py b/app/notify_client/contact_list_api_client.py index 67f55950f..9062addf0 100644 --- a/app/notify_client/contact_list_api_client.py +++ b/app/notify_client/contact_list_api_client.py @@ -24,5 +24,8 @@ class ContactListApiClient(NotifyAdminAPIClient): return job + def get_contact_lists(self, service_id): + return self.get('/service/{}/contact-list'.format(service_id)) + contact_list_api_client = ContactListApiClient() diff --git a/app/templates/views/uploads/contact-list/contact-list.html b/app/templates/views/uploads/contact-list/contact-list.html new file mode 100644 index 000000000..f674e99ed --- /dev/null +++ b/app/templates/views/uploads/contact-list/contact-list.html @@ -0,0 +1,47 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/page-header.html" import page_header %} +{% from "components/message-count-label.html" import message_count_label, recipient_count_label %} +{% from "components/button/macro.njk" import govukButton %} + +{% block service_page_title %} + {{ contact_list.original_file_name }} +{% endblock %} + +{% block maincolumn_content %} + + {{ page_header( + contact_list.original_file_name, + back_link=url_for('main.uploads', service_id=current_service.id) + ) }} + +

    + Uploaded by {{ contact_list.created_by }} {{ contact_list.created_at|format_datetime_human }} +

    + +

    + {{ contact_list.recipients|length|format_thousands }} + {{ recipient_count_label(contact_list.recipients|length, contact_list.recipients.template_type) }} +

    + + {% set recipient_column = contact_list.recipients.column_headers[0] %} + + {% call(item, row_number) list_table( + contact_list.recipients.displayed_rows, + caption=recipient_count_label(contact_list.recipients|length, contact_list.template_type)|capitalize, + caption_visible=False, + field_headings=['1', recipient_column], + ) %} + {{ index_field(row_number) }} + {{ text_field(item[recipient_column].data) }} + {% endcall %} + + {% if contact_list.recipients.displayed_rows|list|length < contact_list.recipients|length %} +

    + Only showing the first {{ contact_list.recipients.displayed_rows|list|length }} rows +

    + {% endif %} + +{% endblock %} diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index de794df60..ecc739d02 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -1,5 +1,6 @@ import re import urllib +import uuid from io import BytesIO from unittest.mock import ANY, Mock @@ -1103,14 +1104,53 @@ def test_cant_save_bad_contact_list( assert mock_create_contact_list.called is False +@freeze_time('2020-03-13 16:51:56') def test_view_contact_list( + mocker, client_request, + mock_get_contact_lists, fake_uuid, ): + mocker.patch('app.models.contact_list.s3download', return_value='\n'.join( + ['email address'] + ['test@example.com'] * 51 + )) page = client_request.get( 'main.contact_list', service_id=SERVICE_ONE_ID, contact_list_id=fake_uuid, - _test_page_title=False, ) - assert page.text == 'page for contact list 6ce466d0-fd6a-11e5-82f5-e0accb9d11a6' + assert normalize_spaces(page.select_one('h1').text) == ( + 'EmergencyContactList.xls' + ) + assert normalize_spaces(page.select('main p')[0].text) == ( + 'Uploaded by Test User today at 10:59am' + ) + assert normalize_spaces(page.select('main p')[1].text) == ( + '51 email addresses' + ) + assert normalize_spaces(page.select_one('table').text).startswith( + 'Email addresses ' + '1 email address ' + '2 test@example.com ' + '3 test@example.com ' + ) + assert normalize_spaces(page.select_one('table').text).endswith( + '50 test@example.com ' + '51 test@example.com' + ) + assert normalize_spaces(page.select_one('.table-show-more-link').text) == ( + 'Only showing the first 50 rows' + ) + + +def test_view_contact_list_404s_for_non_existing_list( + client_request, + mock_get_contact_lists, + fake_uuid, +): + client_request.get( + 'main.contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=uuid.uuid4(), + _expected_status=404, + ) diff --git a/tests/conftest.py b/tests/conftest.py index d44aa12eb..9874c4c0b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1842,6 +1842,33 @@ def mock_create_contact_list(mocker, api_user_active): ) +@pytest.fixture(scope='function') +def mock_get_contact_lists(mocker, api_user_active, fake_uuid): + def _get(service_id): + return [{ + 'created_at': '2020-03-13 10:59:56', + 'created_by': 'Test User', + 'id': fake_uuid, + 'original_file_name': 'EmergencyContactList.xls', + 'row_count': 100, + 'service_id': service_id, + 'template_type': 'email', + }, { + 'created_at': '2020-03-13 13:00:00', + 'created_by': 'Test User', + 'id': uuid4(), + 'original_file_name': 'another ist.csv', + 'row_count': 123, + 'service_id': service_id, + 'template_type': 'sms', + }] + + return mocker.patch( + 'app.contact_list_api_client.get_contact_lists', + side_effect=_get, + ) + + @pytest.fixture(scope='function') def mock_get_notifications( mocker, From e24083ce9d7035e2aae308d4e335ba83c84bb25c Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 13 Mar 2020 17:51:22 +0000 Subject: [PATCH 09/63] Add a page to choose a contact list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit You’ll be able to use a contact list by first choosing a template, then choosing the list you want to use. At the moment this shows all lists, not just the ones that are compatible with your template. --- app/main/views/send.py | 15 +++++- app/models/contact_list.py | 15 ++++-- app/navigation.py | 4 ++ app/templates/views/send-contact-list.html | 53 ++++++++++++++++++++++ tests/app/main/views/test_send.py | 34 +++++++++++++- tests/conftest.py | 2 +- 6 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 app/templates/views/send-contact-list.html diff --git a/app/main/views/send.py b/app/main/views/send.py index 8e1debe71..2ace356f7 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -42,7 +42,7 @@ from app.main.forms import ( SetSenderForm, get_placeholder_form_instance, ) -from app.models.contact_list import ContactList +from app.models.contact_list import ContactList, ContactLists from app.models.user import Users from app.s3_client.s3_csv_client import ( s3download, @@ -492,6 +492,19 @@ def send_test_preview(service_id, template_id, filetype): return TemplatePreview.from_utils_template(template, filetype, page=request.args.get('page')) +@main.route( + '/services//send/' + '/from-contact-list' +) +@user_has_permissions('send_messages') +def choose_from_contact_list(service_id, template_id): + return render_template( + 'views/send-contact-list.html', + contact_lists=ContactLists(current_service.id), + template=current_service.get_template(template_id), + ) + + @main.route( '/services//send/' '/from-contact-list/' diff --git a/app/models/contact_list.py b/app/models/contact_list.py index f61ee1861..dbf58a1d4 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -3,7 +3,7 @@ from notifications_utils.formatters import strip_whitespace from notifications_utils.recipients import RecipientCSV from werkzeug.utils import cached_property -from app.models import JSONModel +from app.models import JSONModel, ModelList from app.notify_client.contact_list_api_client import contact_list_api_client from app.s3_client.s3_csv_client import ( get_csv_metadata, @@ -28,9 +28,9 @@ class ContactList(JSONModel): @classmethod def from_id(cls, contact_list_id, *, service_id): # This is temporary until we have a get single list endpoint - for contact_list in contact_list_api_client.get_contact_lists(service_id): - if contact_list['id'] == contact_list_id: - return cls(contact_list) + for contact_list in ContactLists(service_id): + if contact_list.id == contact_list_id: + return contact_list abort(404) @staticmethod @@ -77,7 +77,7 @@ class ContactList(JSONModel): metadata = ContactList.get_metadata(service_id, upload_id) new_upload_id = s3upload( service_id, - contents, + {'data': contents}, current_app.config['AWS_REGION'], ) set_metadata_on_csv_upload( @@ -111,3 +111,8 @@ class ContactList(JSONModel): international_sms=True, max_initial_rows_shown=50, ) + + +class ContactLists(ModelList): + client_method = contact_list_api_client.get_contact_lists + model = ContactList diff --git a/app/navigation.py b/app/navigation.py index a78c8ea84..3221dd61b 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -149,6 +149,7 @@ class HeaderNavigation(Navigation): 'check_notification', 'no_cookie.check_notification_preview', 'choose_account', + 'choose_from_contact_list', 'choose_service', 'choose_template', 'choose_template_to_copy', @@ -368,6 +369,7 @@ class MainNavigation(Navigation): 'add_service_template', 'check_messages', 'check_notification', + 'choose_from_contact_list', 'choose_template', 'choose_template_to_copy', 'confirm_redact_template', @@ -658,6 +660,7 @@ class CaseworkNavigation(Navigation): mapping = { 'send-one-off': { + 'choose_from_contact_list', 'choose_template', 'send_one_off', 'send_one_off_step', @@ -1018,6 +1021,7 @@ class OrgNavigation(Navigation): 'check_notification', 'no_cookie.check_notification_preview', 'choose_account', + 'choose_from_contact_list', 'choose_service', 'choose_template', 'choose_template_to_copy', diff --git a/app/templates/views/send-contact-list.html b/app/templates/views/send-contact-list.html new file mode 100644 index 000000000..6bf5c9e01 --- /dev/null +++ b/app/templates/views/send-contact-list.html @@ -0,0 +1,53 @@ +{% extends "withnav_template.html" %} +{% from "components/big-number.html" import big_number -%} +{% from "components/message-count-label.html" import recipient_count_label %} +{% from "components/page-header.html" import page_header %} +{% from "components/table.html" import list_table, field, right_aligned_field_heading, row_heading %} + +{% block service_page_title %} + Choose a saved contact list +{% endblock %} + +{% block maincolumn_content %} + + {{ page_header( + 'Choose a saved contact list', + back_link=url_for('.send_one_off', service_id=current_service.id, template_id=template.id) + ) }} + +
    + {% call(item, row_number) list_table( + contact_lists, + caption="Existing contact lists", + caption_visible=False, + empty_message=( + 'You don’t have any contact lists yet' + ), + field_headings=[ + 'File', + 'Status' + ], + field_headings_visible=False + ) %} + {% call row_heading() %} +
    + {{ item.original_file_name }} + + Uploaded {{ item.created_at|format_datetime_relative }} + +
    + {% endcall %} + {% call field() %} + {{ big_number( + item.row_count, + smallest=True, + label=recipient_count_label( + item.row_count, + item.template_type + ) + ) }} + {% endcall %} + {% endcall %} +
    + +{% endblock %} diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 23c31beb1..488421cd9 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -13,6 +13,7 @@ from zipfile import BadZipFile import pytest from bs4 import BeautifulSoup from flask import url_for +from freezegun import freeze_time from notifications_python_client.errors import HTTPError from notifications_utils.recipients import RecipientCSV from notifications_utils.template import ( @@ -3783,6 +3784,37 @@ def test_redirects_to_template_if_job_exists_already( ) +@freeze_time('2020-03-13 13:00') +def test_choose_from_contact_list( + mocker, + client_request, + mock_get_service_template, + mock_get_contact_lists, + fake_uuid, +): + page = client_request.get( + 'main.choose_from_contact_list', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + ) + assert len(page.select('.file-list-filename-large')) == 2 + assert normalize_spaces(page.select_one('.file-list-filename-large').text) == ( + 'EmergencyContactList.xls' + ) + assert page.select_one('a.file-list-filename-large')['href'] == url_for( + 'main.send_from_contact_list', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + contact_list_id=fake_uuid, + ) + assert normalize_spaces(page.select_one('.file-list-hint').text) == ( + 'Uploaded today at 10:59am' + ) + assert normalize_spaces(page.select_one('.big-number-smallest').text) == ( + '100 email addresses' + ) + + def test_send_from_contact_list( mocker, client_request, @@ -3816,7 +3848,7 @@ def test_send_from_contact_list( SERVICE_ONE_ID, fake_uuid, bucket='test-contact-list' ) mock_upload.assert_called_once_with( - SERVICE_ONE_ID, 'contents', ANY + SERVICE_ONE_ID, {'data': 'contents'}, ANY ) mock_set_metadata.assert_called_once_with( SERVICE_ONE_ID, new_uuid, example_key='example value' diff --git a/tests/conftest.py b/tests/conftest.py index 9874c4c0b..ca0dac789 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1864,7 +1864,7 @@ def mock_get_contact_lists(mocker, api_user_active, fake_uuid): }] return mocker.patch( - 'app.contact_list_api_client.get_contact_lists', + 'app.models.contact_list.ContactLists.client_method', side_effect=_get, ) From b22ea172b91eb9e2e884016e9921ce784b810f8d Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Sun, 15 Mar 2020 14:43:39 +0000 Subject: [PATCH 10/63] Add button to uploads page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need a place where people can get into the contact list journey. This is going to be the uploads page, because that’s where stuff you upload lives. At the moment this is platform-admin only, until we’re ready to launch the feature. --- app/templates/views/jobs/jobs.html | 13 ++++++++++++- tests/app/main/views/test_uploads.py | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/app/templates/views/jobs/jobs.html b/app/templates/views/jobs/jobs.html index 347ac2a6d..3516c2daf 100644 --- a/app/templates/views/jobs/jobs.html +++ b/app/templates/views/jobs/jobs.html @@ -31,7 +31,10 @@

    {% endif %} {{ previous_next_navigation(prev_page, next_page) }} - {% if current_service.can_upload_letters and current_user.has_permissions('send_messages') %} + {% if ( + current_service.can_upload_letters and + current_user.has_permissions('send_messages') + ) or current_user.platform_admin %}
    {{ govukButton({ "element": "a", @@ -39,6 +42,14 @@ "href": url_for('.upload_letter', service_id=current_service.id), "classes": "govuk-button--secondary" }) }} + {% if current_user.platform_admin %} + {{ govukButton({ + "element": "a", + "text": "Upload a contact list", + "href": url_for('.upload_contact_list', service_id=current_service.id), + "classes": "govuk-button--secondary govuk-!-margin-left-3" + }) }} + {% endif %}
    {% endif %}
    diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index ecc739d02..6ee68999e 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -15,6 +15,7 @@ from tests.conftest import ( SERVICE_ONE_ID, create_active_caseworking_user, create_active_user_with_permissions, + create_platform_admin_user, ) @@ -39,6 +40,30 @@ def test_no_upload_letters_button_without_permission( assert not page.find('a', text=re.compile('Upload a letter')) +@pytest.mark.parametrize('user', ( + pytest.param( + create_platform_admin_user(), + ), + pytest.param( + create_active_user_with_permissions(), + marks=pytest.mark.xfail(raises=AssertionError), + ), +)) +def test_platform_admin_has_upload_contact_list( + client_request, + mock_get_uploads, + mock_get_jobs, + user, +): + client_request.login(user) + page = client_request.get('main.uploads', service_id=SERVICE_ONE_ID) + button = page.find('a', text=re.compile('Upload a contact list')) + assert button + assert button['href'] == url_for( + 'main.upload_contact_list', service_id=SERVICE_ONE_ID, + ) + + @pytest.mark.parametrize('extra_permissions, expected_empty_message', ( (['letter'], ( 'You have not uploaded any files recently. ' From f76a9e787a3d074216b5f436d57b1240b19f422f Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Sun, 15 Mar 2020 14:56:52 +0000 Subject: [PATCH 11/63] Move skip links underneath the text box If we add another link under it will look weird having two under and one off to the side. It looked sort of balanced when it was just one off to the side and one under. Having all the links under the box makes it less visually noisy. --- app/templates/views/send-test.html | 17 ++++++++++------- tests/app/main/views/test_send.py | 18 +++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/app/templates/views/send-test.html b/app/templates/views/send-test.html index 9f6d1e326..cfde9cb31 100644 --- a/app/templates/views/send-test.html +++ b/app/templates/views/send-test.html @@ -22,24 +22,27 @@ data_kwargs={'force-focus': True} ) %}
    -
    +
    {{ textbox( form.placeholder_value, hint='Optional' if optional_placeholder else None, width='1-1', ) }}
    - {% if skip_link %} -
    - {{ skip_link[0] }} + {% if skip_link or link_to_upload %} +
    + {% if link_to_upload %} + Upload a list of {{ recipient_count_label(999, template.template_type) }} + {% endif %} + {% if skip_link %} + {{ skip_link[0] }} + {% endif %}
    {% endif %}
    {% if link_to_upload %}

    - - Upload a list of {{ recipient_count_label(999, template.template_type) }} - +

    {% endif %} {{ page_footer('Continue') }} diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 488421cd9..c038c9e43 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -1310,16 +1310,17 @@ def test_send_one_off_has_skip_link( _follow_redirects=True, ) - skip_links = page.select('a.top-gutter-4-3') + skip_links = page.select('form a') if expected_link_text and expected_link_url: - assert skip_links[0].text.strip() == expected_link_text - assert skip_links[0]['href'] == expected_link_url( + assert skip_links[1].text.strip() == expected_link_text + assert skip_links[1]['href'] == expected_link_url( service_id=service_one['id'], template_id=fake_uuid, ) else: - assert not skip_links + with pytest.raises(IndexError): + skip_links[1] @pytest.mark.parametrize('template_type, expected_sticky', [ @@ -1376,9 +1377,9 @@ def test_skip_link_will_not_show_on_sms_one_off_if_service_has_no_mobile_number( assert not skip_links -@pytest.mark.parametrize('user, link_index', ( - (create_active_user_with_permissions(), 2), - (create_active_caseworking_user(), 1), +@pytest.mark.parametrize('user', ( + create_active_user_with_permissions(), + create_active_caseworking_user(), )) def test_send_one_off_offers_link_to_upload( client_request, @@ -1386,7 +1387,6 @@ def test_send_one_off_offers_link_to_upload( mock_get_service_template, mock_has_jobs, user, - link_index, ): client_request.login(user) @@ -1398,7 +1398,7 @@ def test_send_one_off_offers_link_to_upload( ) back_link = page.select('main a')[0] - link = page.select('main a')[link_index] + link = page.select_one('form a') assert back_link.text.strip() == 'Back' From ec2f36d054f03a49c34a202ef4f41ed76d18bfd2 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Sun, 15 Mar 2020 15:54:49 +0000 Subject: [PATCH 12/63] Let platform admin users use a contact list Platform admin users now have a link to upload a contact list. This commit gives them access to the page where they can use this. Once this commit is merged we have a way of testing the whole flow end to end, without giving it to all users yet. --- app/templates/views/send-test.html | 3 ++ tests/app/main/views/test_send.py | 55 ++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/app/templates/views/send-test.html b/app/templates/views/send-test.html index cfde9cb31..f2b71ec9e 100644 --- a/app/templates/views/send-test.html +++ b/app/templates/views/send-test.html @@ -33,6 +33,9 @@
    {% if link_to_upload %} Upload a list of {{ recipient_count_label(999, template.template_type) }} + {% if current_user.platform_admin %} + Use a saved list + {% endif %} {% endif %} {% if skip_link %} {{ skip_link[0] }} diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index c038c9e43..01eae7d22 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -38,6 +38,7 @@ from tests.conftest import ( create_active_user_with_permissions, create_multiple_email_reply_to_addresses, create_multiple_sms_senders, + create_platform_admin_user, create_template, mock_get_service_email_template, mock_get_service_letter_template, @@ -1410,6 +1411,60 @@ def test_send_one_off_offers_link_to_upload( ) +@pytest.mark.parametrize('user', ( + pytest.param( + create_platform_admin_user(), + ), + pytest.param( + create_active_user_with_permissions(), + marks=pytest.mark.xfail(raises=AssertionError), + ), +)) +def test_platform_admin_has_link_to_use_existing_list( + client_request, + mock_get_service_template, + mock_has_jobs, + fake_uuid, + user, +): + client_request.login(user) + page = client_request.get( + 'main.send_one_off', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + _follow_redirects=True, + ) + + assert [ + (link.text, link['href']) for link in page.select('form a') + ] == [ + ( + 'Upload a list of phone numbers', + url_for( + 'main.send_messages', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + ), + ), + ( + 'Use a saved list', + url_for( + 'main.choose_from_contact_list', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + ), + ), + ( + 'Use my phone number', + url_for( + 'main.send_test', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + ), + ), + ] + + @pytest.mark.parametrize('user', ( create_active_user_with_permissions(), create_active_caseworking_user(), From b598af1ecaf9bad5db2f94f879b4599a73ce653f Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Sun, 15 Mar 2020 19:13:07 +0000 Subject: [PATCH 13/63] Add an endpoint to download a contact list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In case you need to download it, modify it, and re-upload it when you don’t have access to the existing list. --- app/main/views/uploads.py | 14 ++++++- app/models/contact_list.py | 13 ++++++- app/navigation.py | 4 ++ .../uploads/contact-list/contact-list.html | 1 + tests/app/main/views/test_uploads.py | 37 ++++++++++++++++++- 5 files changed, 66 insertions(+), 3 deletions(-) diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index b3a1021ef..329d8591c 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -3,7 +3,7 @@ import itertools import json import urllib import uuid -from io import BytesIO +from io import BytesIO, StringIO from zipfile import BadZipFile from flask import ( @@ -13,6 +13,7 @@ from flask import ( redirect, render_template, request, + send_file, url_for, ) from notifications_utils.columns import Columns @@ -437,3 +438,14 @@ def contact_list(service_id, contact_list_id): 'views/uploads/contact-list/contact-list.html', contact_list=ContactList.from_id(contact_list_id, service_id=service_id), ) + + +@main.route("/services//contact-list/.csv", methods=['GET']) +@user_has_permissions('send_messages') +def download_contact_list(service_id, contact_list_id): + contact_list = ContactList.from_id(contact_list_id, service_id=service_id) + return send_file( + filename_or_fp=StringIO(contact_list.contents), + attachment_filename=contact_list.saved_file_name, + as_attachment=True, + ) diff --git a/app/models/contact_list.py b/app/models/contact_list.py index dbf58a1d4..b617a4463 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -1,3 +1,5 @@ +from os import path + from flask import abort, current_app from notifications_utils.formatters import strip_whitespace from notifications_utils.recipients import RecipientCSV @@ -103,15 +105,24 @@ class ContactList(JSONModel): template_type=metadata['template_type'], )) + @property + def contents(self): + return self.download(self.service_id, self.id) + @cached_property def recipients(self): return RecipientCSV( - self.download(self.service_id, self.id), + self.contents, template_type=self.template_type, international_sms=True, max_initial_rows_shown=50, ) + @property + def saved_file_name(self): + file_name, extention = path.splitext(self.original_file_name) + return f'{file_name}.csv' + class ContactLists(ModelList): client_method = contact_list_api_client.get_contact_lists diff --git a/app/navigation.py b/app/navigation.py index 3221dd61b..4969af085 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -170,6 +170,7 @@ class HeaderNavigation(Navigation): 'delivery_and_failure', 'delivery_status_callback', 'design_content', + 'download_contact_list', 'download_notifications_csv', 'edit_data_retention', 'edit_organisation_agreement', @@ -506,6 +507,7 @@ class MainNavigation(Navigation): 'delivery_and_failure', 'design_content', 'documentation', + 'download_contact_list', 'download_notifications_csv', 'edit_data_retention', 'edit_organisation_agreement', @@ -744,6 +746,7 @@ class CaseworkNavigation(Navigation): 'delivery_status_callback', 'design_content', 'documentation', + 'download_contact_list', 'download_notifications_csv', 'edit_data_retention', 'edit_organisation_agreement', @@ -1045,6 +1048,7 @@ class OrgNavigation(Navigation): 'delivery_status_callback', 'design_content', 'documentation', + 'download_contact_list', 'download_notifications_csv', 'edit_data_retention', 'edit_provider', diff --git a/app/templates/views/uploads/contact-list/contact-list.html b/app/templates/views/uploads/contact-list/contact-list.html index f674e99ed..50c0cef79 100644 --- a/app/templates/views/uploads/contact-list/contact-list.html +++ b/app/templates/views/uploads/contact-list/contact-list.html @@ -22,6 +22,7 @@

    + Download this list  {{ contact_list.recipients|length|format_thousands }} {{ recipient_count_label(contact_list.recipients|length, contact_list.recipients.template_type) }}

    diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 6ee68999e..073b045da 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -1151,7 +1151,12 @@ def test_view_contact_list( 'Uploaded by Test User today at 10:59am' ) assert normalize_spaces(page.select('main p')[1].text) == ( - '51 email addresses' + 'Download this list 51 email addresses' + ) + assert page.select_one('a[download]')['href'] == url_for( + 'main.download_contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, ) assert normalize_spaces(page.select_one('table').text).startswith( 'Email addresses ' @@ -1179,3 +1184,33 @@ def test_view_contact_list_404s_for_non_existing_list( contact_list_id=uuid.uuid4(), _expected_status=404, ) + + +def test_download_contact_list( + mocker, + logged_in_client, + fake_uuid, + mock_get_contact_lists, +): + mocker.patch( + 'app.models.contact_list.s3download', + return_value='phone number\n07900900321' + ) + response = logged_in_client.get(url_for( + 'main.download_contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + )) + assert response.status_code == 200 + assert response.headers['Content-Type'] == ( + 'text/csv; ' + 'charset=utf-8' + ) + assert response.headers['Content-Disposition'] == ( + 'attachment; ' + 'filename=EmergencyContactList.csv' + ) + assert response.get_data(as_text=True) == ( + 'phone number\n' + '07900900321' + ) From 6c2021aeb286342ae21a8fed9a38c6aa8739eb9d Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 09:41:42 +0000 Subject: [PATCH 14/63] Only show contact lists relevant to template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit You can’t send an email message template to a list of phone numbers. So we shouldn’t show you the lists of phone numbers when you’ve chosen an email template. --- app/main/views/send.py | 11 +++- app/models/contact_list.py | 8 +++ app/templates/views/send-contact-list.html | 73 ++++++++++++---------- tests/app/main/views/test_send.py | 73 +++++++++++++++++++--- tests/conftest.py | 6 +- 5 files changed, 128 insertions(+), 43 deletions(-) diff --git a/app/main/views/send.py b/app/main/views/send.py index 2ace356f7..065862ce2 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -498,9 +498,18 @@ def send_test_preview(service_id, template_id, filetype): ) @user_has_permissions('send_messages') def choose_from_contact_list(service_id, template_id): + db_template = current_service.get_template_with_user_permission_or_403( + template_id, current_user + ) + template = get_template( + db_template, current_service, + ) return render_template( 'views/send-contact-list.html', - contact_lists=ContactLists(current_service.id), + contact_lists=ContactLists( + current_service.id, + template_type=template.template_type, + ), template=current_service.get_template(template_id), ) diff --git a/app/models/contact_list.py b/app/models/contact_list.py index b617a4463..c781df97f 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -125,5 +125,13 @@ class ContactList(JSONModel): class ContactLists(ModelList): + client_method = contact_list_api_client.get_contact_lists model = ContactList + + def __init__(self, service_id, template_type=None): + super().__init__(service_id) + self.items = [ + item for item in self.items + if template_type in {item['template_type'], None} + ] diff --git a/app/templates/views/send-contact-list.html b/app/templates/views/send-contact-list.html index 6bf5c9e01..1827485a0 100644 --- a/app/templates/views/send-contact-list.html +++ b/app/templates/views/send-contact-list.html @@ -15,39 +15,48 @@ back_link=url_for('.send_one_off', service_id=current_service.id, template_id=template.id) ) }} -
    - {% call(item, row_number) list_table( - contact_lists, - caption="Existing contact lists", - caption_visible=False, - empty_message=( - 'You don’t have any contact lists yet' - ), - field_headings=[ - 'File', - 'Status' - ], - field_headings_visible=False - ) %} - {% call row_heading() %} -
    - {{ item.original_file_name }} - - Uploaded {{ item.created_at|format_datetime_relative }} - -
    - {% endcall %} - {% call field() %} - {{ big_number( - item.row_count, - smallest=True, - label=recipient_count_label( + {% if contact_lists %} +
    + {% call(item, row_number) list_table( + contact_lists, + caption="Existing contact lists", + caption_visible=False, + empty_message=( + 'You don’t have any contact lists yet' + ), + field_headings=[ + 'File', + 'Status' + ], + field_headings_visible=False + ) %} + {% call row_heading() %} +
    + {{ item.original_file_name }} + + Uploaded {{ item.created_at|format_datetime_relative }} + +
    + {% endcall %} + {% call field() %} + {{ big_number( item.row_count, - item.template_type - ) - ) }} + smallest=True, + label=recipient_count_label( + item.row_count, + item.template_type + ) + ) }} + {% endcall %} {% endcall %} - {% endcall %} -
    +
    + {% else %} +

    + You have not saved any lists of {{ recipient_count_label(99, template.template_type) }} yet. +

    +

    + To upload and save a new contact list, go to the uploads page. +

    + {% endif %} {% endblock %} diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 01eae7d22..db80d72d6 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -3839,37 +3839,96 @@ def test_redirects_to_template_if_job_exists_already( ) +@pytest.mark.parametrize(( + 'template_type, ' + 'expected_list_id, ' + 'expected_filename, ' + 'expected_time, ' + 'expected_count' +), ( + ( + 'email', + '6ce466d0-fd6a-11e5-82f5-e0accb9d11a6', + 'EmergencyContactList.xls', + 'Uploaded today at 10:59am', + '100 email addresses', + ), + ( + 'sms', + 'd7b0bd1a-d1c7-4621-be5c-3c1b4278a2ad', + 'phone number list.csv', + 'Uploaded today at 1:00pm', + '123 phone numbers', + ), +)) @freeze_time('2020-03-13 13:00') def test_choose_from_contact_list( mocker, client_request, - mock_get_service_template, mock_get_contact_lists, fake_uuid, + template_type, + expected_list_id, + expected_filename, + expected_time, + expected_count, ): + template = create_template(template_type=template_type) + mocker.patch( + 'app.service_api_client.get_service_template', + return_value={'data': template}, + ) page = client_request.get( 'main.choose_from_contact_list', service_id=SERVICE_ONE_ID, template_id=fake_uuid, ) - assert len(page.select('.file-list-filename-large')) == 2 + assert len(page.select('.file-list-filename-large')) == 1 assert normalize_spaces(page.select_one('.file-list-filename-large').text) == ( - 'EmergencyContactList.xls' + expected_filename ) assert page.select_one('a.file-list-filename-large')['href'] == url_for( 'main.send_from_contact_list', service_id=SERVICE_ONE_ID, - template_id=fake_uuid, - contact_list_id=fake_uuid, + template_id=template['id'], + contact_list_id=expected_list_id, ) assert normalize_spaces(page.select_one('.file-list-hint').text) == ( - 'Uploaded today at 10:59am' + expected_time ) assert normalize_spaces(page.select_one('.big-number-smallest').text) == ( - '100 email addresses' + expected_count ) +def test_choose_from_contact_list_with_no_lists( + mocker, + client_request, + mock_get_service_template, + fake_uuid, +): + mocker.patch( + 'app.models.contact_list.ContactLists.client_method', + return_value=[], + ) + page = client_request.get( + 'main.choose_from_contact_list', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + ) + assert [ + normalize_spaces(p.text) for p in page.select('main p') + ] == [ + 'You have not saved any lists of phone numbers yet.', + 'To upload and save a new contact list, go to the uploads page.', + ] + assert page.select_one('main p a')['href'] == url_for( + 'main.uploads', + service_id=SERVICE_ONE_ID, + ) + assert not page.select('table') + + def test_send_from_contact_list( mocker, client_request, diff --git a/tests/conftest.py b/tests/conftest.py index ca0dac789..308a78a2a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1844,7 +1844,7 @@ def mock_create_contact_list(mocker, api_user_active): @pytest.fixture(scope='function') def mock_get_contact_lists(mocker, api_user_active, fake_uuid): - def _get(service_id): + def _get(service_id, template_type=None): return [{ 'created_at': '2020-03-13 10:59:56', 'created_by': 'Test User', @@ -1856,8 +1856,8 @@ def mock_get_contact_lists(mocker, api_user_active, fake_uuid): }, { 'created_at': '2020-03-13 13:00:00', 'created_by': 'Test User', - 'id': uuid4(), - 'original_file_name': 'another ist.csv', + 'id': 'd7b0bd1a-d1c7-4621-be5c-3c1b4278a2ad', + 'original_file_name': 'phone number list.csv', 'row_count': 123, 'service_id': service_id, 'template_type': 'sms', From 31d2fd9b5821f2b8b88aa95d031ceb50a3ea8c3c Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 10:14:11 +0000 Subject: [PATCH 15/63] Only show the link to use list if service has lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most services won’t be interested in the contact list feature, so we shouldn’t clutter up the interface with the extra link. --- app/models/service.py | 5 +++++ app/templates/views/send-test.html | 2 +- tests/app/main/views/test_send.py | 27 +++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/app/models/service.py b/app/models/service.py index 132ac157c..041ef5e5a 100644 --- a/app/models/service.py +++ b/app/models/service.py @@ -6,6 +6,7 @@ from notifications_utils.timezones import local_timezone from werkzeug.utils import cached_property from app.models import JSONModel +from app.models.contact_list import ContactLists from app.models.job import ( ImmediateJobs, PaginatedJobs, @@ -695,3 +696,7 @@ class Service(JSONModel): for report in self.returned_letter_summary if parse(report['reported_at'] + " 00:00:00") >= seven_days_ago ) + + @property + def contact_lists(self): + return ContactLists(self.id) diff --git a/app/templates/views/send-test.html b/app/templates/views/send-test.html index f2b71ec9e..9ea19106c 100644 --- a/app/templates/views/send-test.html +++ b/app/templates/views/send-test.html @@ -33,7 +33,7 @@
    {% if link_to_upload %} Upload a list of {{ recipient_count_label(999, template.template_type) }} - {% if current_user.platform_admin %} + {% if current_user.platform_admin and current_service.contact_lists %} Use a saved list {% endif %} {% endif %} diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index db80d72d6..c7112599b 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -1424,6 +1424,7 @@ def test_platform_admin_has_link_to_use_existing_list( client_request, mock_get_service_template, mock_has_jobs, + mock_get_contact_lists, fake_uuid, user, ): @@ -1465,6 +1466,32 @@ def test_platform_admin_has_link_to_use_existing_list( ] +def test_no_link_to_use_existing_list_for_service_without_lists( + mocker, + client_request, + mock_get_service_template, + mock_has_jobs, + fake_uuid, +): + mocker.patch( + 'app.models.contact_list.ContactLists.client_method', + return_value=[], + ) + client_request.login(create_platform_admin_user()) + page = client_request.get( + 'main.send_one_off', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + _follow_redirects=True, + ) + assert [ + link.text for link in page.select('form a') + ] == [ + 'Upload a list of phone numbers', + 'Use my phone number', + ] + + @pytest.mark.parametrize('user', ( create_active_user_with_permissions(), create_active_caseworking_user(), From 8ba75cf6ba0b7556da23fb393adf59bdfb3651be Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 10:34:05 +0000 Subject: [PATCH 16/63] Warn users trying to use a personalised template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since contact lists only store the email address or phone number, this won’t work. So we should warn people, and highlight that the personalisation in the template is the problem, since this is what they will need to change. --- app/main/views/send.py | 2 +- app/templates/views/send-contact-list.html | 32 +++++++++++++++++----- tests/app/main/views/test_send.py | 28 +++++++++++++++++++ 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/app/main/views/send.py b/app/main/views/send.py index 065862ce2..72d0166ee 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -510,7 +510,7 @@ def choose_from_contact_list(service_id, template_id): current_service.id, template_type=template.template_type, ), - template=current_service.get_template(template_id), + template=template, ) diff --git a/app/templates/views/send-contact-list.html b/app/templates/views/send-contact-list.html index 1827485a0..e5634d927 100644 --- a/app/templates/views/send-contact-list.html +++ b/app/templates/views/send-contact-list.html @@ -1,5 +1,6 @@ {% extends "withnav_template.html" %} {% from "components/big-number.html" import big_number -%} +{% from "components/list.html" import list_of_placeholders %} {% from "components/message-count-label.html" import recipient_count_label %} {% from "components/page-header.html" import page_header %} {% from "components/table.html" import list_table, field, right_aligned_field_heading, row_heading %} @@ -15,7 +16,20 @@ back_link=url_for('.send_one_off', service_id=current_service.id, template_id=template.id) ) }} - {% if contact_lists %} + {% if template.placeholders %} +
    +
    +

    + You cannot use a saved contact list with this template because it + is personalised with {{ list_of_placeholders(template.placeholders) }}. +

    +

    + Saved contact lists can only store email addresses or phone + numbers. +

    +
    +
    + {% elif contact_lists %}
    {% call(item, row_number) list_table( contact_lists, @@ -51,12 +65,16 @@ {% endcall %}
    {% else %} -

    - You have not saved any lists of {{ recipient_count_label(99, template.template_type) }} yet. -

    -

    - To upload and save a new contact list, go to the uploads page. -

    +
    +
    +

    + You have not saved any lists of {{ recipient_count_label(99, template.template_type) }} yet. +

    +

    + To upload and save a new contact list, go to the uploads page. +

    +
    +
    {% endif %} {% endblock %} diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index c7112599b..c96ae090d 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -3928,6 +3928,34 @@ def test_choose_from_contact_list( ) +def test_choose_from_contact_list_with_personalised_template( + mocker, + client_request, + mock_get_contact_lists, + fake_uuid, +): + template = create_template( + content="Hey ((name)) ((thing)) is happening" + ) + mocker.patch( + 'app.service_api_client.get_service_template', + return_value={'data': template}, + ) + page = client_request.get( + 'main.choose_from_contact_list', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + ) + assert [ + normalize_spaces(p.text) for p in page.select('main p') + ] == [ + 'You cannot use a saved contact list with this template because ' + 'it is personalised with ((name)) and ((thing)).', + 'Saved contact lists can only store email addresses or phone numbers.', + ] + assert not page.select('table') + + def test_choose_from_contact_list_with_no_lists( mocker, client_request, From 050f98fea692833bc65350c61280542ff6f275f1 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 10:55:58 +0000 Subject: [PATCH 17/63] Show contact lists on the uploads page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uploads page is where all the stuff you’ve uploaded lives. Now you can upload contact lists they should live here too. They always come first because they’re the most-removed from stuff you’ve sent. --- app/main/views/uploads.py | 6 ++- app/models/contact_list.py | 2 + app/templates/views/dashboard/_jobs.html | 19 ++++++++- tests/app/main/views/test_uploads.py | 51 ++++++++++++++++++++++++ tests/conftest.py | 8 ++++ 5 files changed, 84 insertions(+), 2 deletions(-) diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index 329d8591c..a3c3ae63b 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -66,7 +66,11 @@ def uploads(service_id): next_page = generate_next_dict('main.uploads', service_id, uploads.current_page) if uploads.current_page == 1: - listed_uploads = current_service.scheduled_jobs + uploads + listed_uploads = ( + current_service.contact_lists + + current_service.scheduled_jobs + + uploads + ) else: listed_uploads = uploads diff --git a/app/models/contact_list.py b/app/models/contact_list.py index c781df97f..ca14bcbdc 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -27,6 +27,8 @@ class ContactList(JSONModel): 'template_type', } + upload_type = 'contact_list' + @classmethod def from_id(cls, contact_list_id, *, service_id): # This is temporary until we have a get single list endpoint diff --git a/app/templates/views/dashboard/_jobs.html b/app/templates/views/dashboard/_jobs.html index 9efb9f777..5b5e3bda9 100644 --- a/app/templates/views/dashboard/_jobs.html +++ b/app/templates/views/dashboard/_jobs.html @@ -1,6 +1,6 @@ {% from "components/table.html" import list_table, field, right_aligned_field_heading, row_heading %} {% from "components/big-number.html" import big_number -%} -{% from "components/message-count-label.html" import message_count_label -%} +{% from "components/message-count-label.html" import message_count_label, recipient_count_label -%}
    {% call(item, row_number) list_table( @@ -20,6 +20,8 @@
    {% if item.upload_type == 'letter' %} {{ item.original_file_name }} + {% elif item.upload_type == 'contact_list' %} + {{ item.original_file_name }} {% else %} {{ item.original_file_name }} {% endif %} @@ -29,6 +31,12 @@ item.scheduled_for|format_datetime_relative }} + {% elif item.upload_type == 'contact_list' %} + + Uploaded {{ + item.created_at|format_datetime_relative + }} + {% else %} Sent {{ @@ -60,6 +68,15 @@ suffix='' ) ) }} + {% elif item.upload_type == 'contact_list' %} + {{ big_number( + item.row_count, + smallest=True, + label="saved {}".format(recipient_count_label( + item.row_count, + item.template_type + )) + ) }} {% elif item.pdf_letter %}

    {% for line in item.recipient.splitlines() %} diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 073b045da..a3bc2f1ab 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -33,6 +33,7 @@ def test_no_upload_letters_button_without_permission( service_one, mock_get_uploads, mock_get_jobs, + mock_get_no_contact_lists, extra_permissions, ): service_one['permissions'] += extra_permissions @@ -53,6 +54,7 @@ def test_platform_admin_has_upload_contact_list( client_request, mock_get_uploads, mock_get_jobs, + mock_get_no_contact_lists, user, ): client_request.login(user) @@ -80,6 +82,7 @@ def test_get_upload_hub_with_no_uploads( client_request, service_one, mock_get_no_uploads, + mock_get_no_contact_lists, extra_permissions, expected_empty_message, ): @@ -97,6 +100,7 @@ def test_get_upload_hub_page( client_request, service_one, mock_get_uploads, + mock_get_no_contact_lists, ): mocker.patch('app.job_api_client.get_jobs', return_value={'data': []}) service_one['permissions'] += ['letter', 'upload_letters'] @@ -683,6 +687,7 @@ def test_uploads_page_shows_scheduled_jobs( client_request, mock_get_no_uploads, mock_get_jobs, + mock_get_no_contact_lists, user, ): client_request.login(user) @@ -708,11 +713,57 @@ def test_uploads_page_shows_scheduled_jobs( assert not page.select('.table-empty-message') +@freeze_time('2020-03-15') +def test_uploads_page_shows_contact_lists_first( + mocker, + client_request, + fake_uuid, + mock_get_no_uploads, + mock_get_jobs, + mock_get_contact_lists, +): + page = client_request.get('main.uploads', service_id=SERVICE_ONE_ID) + + assert [ + normalize_spaces(row.text) for row in page.select('tr') + ] == [ + ( + 'File Status' + ), + ( + 'EmergencyContactList.xls ' + 'Uploaded 13 March at 10:59am ' + '100 saved email addresses' + ), + ( + 'phone number list.csv ' + 'Uploaded 13 March at 1:00pm ' + '123 saved phone numbers' + ), + ( + 'even_later.csv ' + 'Sending 1 January 2016 at 11:09pm ' + '1 text message waiting to send' + ), + ( + 'send_me_later.csv ' + 'Sending 1 January 2016 at 11:09am ' + '1 text message waiting to send' + ), + ] + assert page.select_one('.file-list-filename-large')['href'] == url_for( + 'main.contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + ) + + def test_get_uploads_shows_pagination( client_request, active_user_with_permissions, mock_get_jobs, mock_get_uploads, + mock_get_no_contact_lists, ): page = client_request.get('main.uploads', service_id=SERVICE_ONE_ID) diff --git a/tests/conftest.py b/tests/conftest.py index 308a78a2a..c4360c140 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1869,6 +1869,14 @@ def mock_get_contact_lists(mocker, api_user_active, fake_uuid): ) +@pytest.fixture(scope='function') +def mock_get_no_contact_lists(mocker): + return mocker.patch( + 'app.models.contact_list.ContactLists.client_method', + return_value=[], + ) + + @pytest.fixture(scope='function') def mock_get_notifications( mocker, From b99216173e7437f9b84bbf7038144dd9b0b8efcc Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 11:05:26 +0000 Subject: [PATCH 18/63] Sort contact lists appropriately On the uploads page this should be newest first, same as the scheduled and immediate jobs on this page. On the _choose_ page this should be alphabetical, same as choosing a template. --- app/main/views/send.py | 4 ++-- app/models/contact_list.py | 18 ++++++++++++++++-- tests/app/main/views/test_uploads.py | 13 ++++++------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/app/main/views/send.py b/app/main/views/send.py index 72d0166ee..f8bcd1451 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -42,7 +42,7 @@ from app.main.forms import ( SetSenderForm, get_placeholder_form_instance, ) -from app.models.contact_list import ContactList, ContactLists +from app.models.contact_list import ContactList, ContactListsAlphabetical from app.models.user import Users from app.s3_client.s3_csv_client import ( s3download, @@ -506,7 +506,7 @@ def choose_from_contact_list(service_id, template_id): ) return render_template( 'views/send-contact-list.html', - contact_lists=ContactLists( + contact_lists=ContactListsAlphabetical( current_service.id, template_type=template.template_type, ), diff --git a/app/models/contact_list.py b/app/models/contact_list.py index ca14bcbdc..2d7e685bf 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -1,3 +1,4 @@ +from functools import partial from os import path from flask import abort, current_app @@ -130,10 +131,23 @@ class ContactLists(ModelList): client_method = contact_list_api_client.get_contact_lists model = ContactList + sort_function = partial( + sorted, + key=lambda item: item['created_at'], + reverse=True, + ) def __init__(self, service_id, template_type=None): super().__init__(service_id) - self.items = [ + self.items = self.sort_function([ item for item in self.items if template_type in {item['template_type'], None} - ] + ]) + + +class ContactListsAlphabetical(ContactLists): + + sort_function = partial( + sorted, + key=lambda item: item['original_file_name'].lower(), + ) diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index a3bc2f1ab..555cb1662 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -717,7 +717,6 @@ def test_uploads_page_shows_scheduled_jobs( def test_uploads_page_shows_contact_lists_first( mocker, client_request, - fake_uuid, mock_get_no_uploads, mock_get_jobs, mock_get_contact_lists, @@ -730,16 +729,16 @@ def test_uploads_page_shows_contact_lists_first( ( 'File Status' ), - ( - 'EmergencyContactList.xls ' - 'Uploaded 13 March at 10:59am ' - '100 saved email addresses' - ), ( 'phone number list.csv ' 'Uploaded 13 March at 1:00pm ' '123 saved phone numbers' ), + ( + 'EmergencyContactList.xls ' + 'Uploaded 13 March at 10:59am ' + '100 saved email addresses' + ), ( 'even_later.csv ' 'Sending 1 January 2016 at 11:09pm ' @@ -754,7 +753,7 @@ def test_uploads_page_shows_contact_lists_first( assert page.select_one('.file-list-filename-large')['href'] == url_for( 'main.contact_list', service_id=SERVICE_ONE_ID, - contact_list_id=fake_uuid, + contact_list_id='d7b0bd1a-d1c7-4621-be5c-3c1b4278a2ad', ) From 93f862621ce93b91f03d7f5f6f074ba2838f2e2e Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 11:25:07 +0000 Subject: [PATCH 19/63] Pass original file name when sending contact list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise the new file doesn’t get a name, which is very confusing. --- app/main/views/send.py | 7 ++++++- app/models/contact_list.py | 12 +++++------- tests/app/main/views/test_send.py | 2 ++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/app/main/views/send.py b/app/main/views/send.py index f8bcd1451..bde1b3a85 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -520,11 +520,16 @@ def choose_from_contact_list(service_id, template_id): ) @user_has_permissions('send_messages') def send_from_contact_list(service_id, template_id, contact_list_id): + contact_list = ContactList.from_id( + contact_list_id, + service_id=current_service.id, + ) return redirect(url_for( 'main.check_messages', service_id=current_service.id, template_id=template_id, - upload_id=ContactList.copy_to_uploads(service_id, contact_list_id), + upload_id=contact_list.copy_to_uploads(), + original_file_name=contact_list.original_file_name, )) diff --git a/app/models/contact_list.py b/app/models/contact_list.py index 2d7e685bf..6afbf9911 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -76,17 +76,15 @@ class ContactList(JSONModel): bucket=ContactList.get_bucket_name(), ) - @staticmethod - def copy_to_uploads(service_id, upload_id): - contents = ContactList.download(service_id, upload_id) - metadata = ContactList.get_metadata(service_id, upload_id) + def copy_to_uploads(self): + metadata = self.get_metadata(self.service_id, self.id) new_upload_id = s3upload( - service_id, - {'data': contents}, + self.service_id, + {'data': self.contents}, current_app.config['AWS_REGION'], ) set_metadata_on_csv_upload( - service_id, + self.service_id, new_upload_id, **metadata, ) diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index c96ae090d..7cb5d2df6 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -3988,6 +3988,7 @@ def test_send_from_contact_list( mocker, client_request, fake_uuid, + mock_get_contact_lists, ): new_uuid = uuid.uuid4() mock_download = mocker.patch('app.models.contact_list.s3download', return_value='contents') @@ -4007,6 +4008,7 @@ def test_send_from_contact_list( service_id=SERVICE_ONE_ID, template_id=fake_uuid, upload_id=new_uuid, + original_file_name='EmergencyContactList.xls', _external=True, ) ) From f563c4a18122a05690150ca6b41814410e7c7b01 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 11:35:45 +0000 Subject: [PATCH 20/63] =?UTF-8?q?Update=20content=20based=20on=20KC?= =?UTF-8?q?=E2=80=99s=20suggestions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/templates/views/jobs/jobs.html | 2 +- .../views/uploads/contact-list/upload.html | 13 ++++++++++--- tests/app/main/views/test_uploads.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/app/templates/views/jobs/jobs.html b/app/templates/views/jobs/jobs.html index 3516c2daf..77b51794a 100644 --- a/app/templates/views/jobs/jobs.html +++ b/app/templates/views/jobs/jobs.html @@ -45,7 +45,7 @@ {% if current_user.platform_admin %} {{ govukButton({ "element": "a", - "text": "Upload a contact list", + "text": "Upload an emergency contact list", "href": url_for('.upload_contact_list', service_id=current_service.id), "classes": "govuk-button--secondary govuk-!-margin-left-3" }) }} diff --git a/app/templates/views/uploads/contact-list/upload.html b/app/templates/views/uploads/contact-list/upload.html index 7b6355a38..265b366b5 100644 --- a/app/templates/views/uploads/contact-list/upload.html +++ b/app/templates/views/uploads/contact-list/upload.html @@ -20,9 +20,16 @@ {{ page_header( 'Upload an emergency contact list', back_link=url_for('main.uploads', service_id=current_service.id) - ) }} -

    Upload a list of phone numbers or email addresses.

    -

    Don’t put members of the public in here.

    + ) }} +

    + Save a list of staff email addresses or phone numbers in Notify. +

    +

    + In an emergency, you can send a message to everyone on the list. +

    +

    + Do not include contact details for members of the public. +

    {% endif %}
    diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 555cb1662..03a4b3e73 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -59,7 +59,7 @@ def test_platform_admin_has_upload_contact_list( ): client_request.login(user) page = client_request.get('main.uploads', service_id=SERVICE_ONE_ID) - button = page.find('a', text=re.compile('Upload a contact list')) + button = page.find('a', text=re.compile('Upload an emergency contact list')) assert button assert button['href'] == url_for( 'main.upload_contact_list', service_id=SERVICE_ONE_ID, From 00f953de6d4a2c3fed2a070028dd98d9eb68d076 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 13:47:10 +0000 Subject: [PATCH 21/63] Give Flask bytes to send, not string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For some reason this passed the tests, but didn’t work running locally. --- app/main/views/uploads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index a3c3ae63b..e3c6d09ad 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -3,7 +3,7 @@ import itertools import json import urllib import uuid -from io import BytesIO, StringIO +from io import BytesIO from zipfile import BadZipFile from flask import ( @@ -449,7 +449,7 @@ def contact_list(service_id, contact_list_id): def download_contact_list(service_id, contact_list_id): contact_list = ContactList.from_id(contact_list_id, service_id=service_id) return send_file( - filename_or_fp=StringIO(contact_list.contents), + filename_or_fp=BytesIO(contact_list.contents.encode('utf-8')), attachment_filename=contact_list.saved_file_name, as_attachment=True, ) From 341b9009f3309667c72708ad4031780d66f11e4d Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 14:14:29 +0000 Subject: [PATCH 22/63] Harmonise the typography when listing uploads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 24px with 19px is what we use on the uploads page. On notifications page we use 19px with 16px. There’s some loose idea that the bigger size is for items that contain other items. This also increases the line height for recipients of PDF letters to make things line up. --- app/assets/stylesheets/components/letter.scss | 5 +++++ app/assets/stylesheets/views/dashboard.scss | 11 +++++++++++ app/templates/views/dashboard/_jobs.html | 8 ++++---- app/templates/views/send-contact-list.html | 2 +- tests/app/main/views/test_send.py | 2 +- tests/app/main/views/test_uploads.py | 2 +- 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/app/assets/stylesheets/components/letter.scss b/app/assets/stylesheets/components/letter.scss index 49f25bc3c..84503da16 100644 --- a/app/assets/stylesheets/components/letter.scss +++ b/app/assets/stylesheets/components/letter.scss @@ -89,3 +89,8 @@ $iso-paper-ratio: 141.42135624%; } } + +.letter-recipient-summary { + line-height: 28px; + margin-bottom: 0; +} diff --git a/app/assets/stylesheets/views/dashboard.scss b/app/assets/stylesheets/views/dashboard.scss index b0035347b..1db274e5f 100644 --- a/app/assets/stylesheets/views/dashboard.scss +++ b/app/assets/stylesheets/views/dashboard.scss @@ -83,6 +83,17 @@ max-width: 580px; } + &-hint-large { + @include core-19; + display: block; + color: $secondary-text-colour; + pointer-events: none; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + max-width: 580px; + } + } .failure-highlight { diff --git a/app/templates/views/dashboard/_jobs.html b/app/templates/views/dashboard/_jobs.html index 5b5e3bda9..f7d6ab271 100644 --- a/app/templates/views/dashboard/_jobs.html +++ b/app/templates/views/dashboard/_jobs.html @@ -26,19 +26,19 @@ {{ item.original_file_name }} {% endif %} {% if item.scheduled %} - + Sending {{ item.scheduled_for|format_datetime_relative }} {% elif item.upload_type == 'contact_list' %} - + Uploaded {{ item.created_at|format_datetime_relative }} {% else %} - + Sent {{ (item.scheduled_for or item.created_at)|format_datetime_relative }} @@ -78,7 +78,7 @@ )) ) }} {% elif item.pdf_letter %} -

    +

    {% for line in item.recipient.splitlines() %} {% if loop.index < 3 %} {{ line }}
    diff --git a/app/templates/views/send-contact-list.html b/app/templates/views/send-contact-list.html index e5634d927..e08c020fb 100644 --- a/app/templates/views/send-contact-list.html +++ b/app/templates/views/send-contact-list.html @@ -47,7 +47,7 @@ {% call row_heading() %}

    {{ item.original_file_name }} - + Uploaded {{ item.created_at|format_datetime_relative }}
    diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 7cb5d2df6..3a4b671ee 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -3920,7 +3920,7 @@ def test_choose_from_contact_list( template_id=template['id'], contact_list_id=expected_list_id, ) - assert normalize_spaces(page.select_one('.file-list-hint').text) == ( + assert normalize_spaces(page.select_one('.file-list-hint-large').text) == ( expected_time ) assert normalize_spaces(page.select_one('.big-number-smallest').text) == ( diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 03a4b3e73..13daaf5dc 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -128,7 +128,7 @@ def test_get_upload_hub_page( '123 Example Street' ) assert normalize_spaces(str(uploads[1].select_one('.govuk-body'))) == ( - '

    ' + '

    ' 'Firstname Lastname
    ' '123 Example Street
    ' '

    ' From b5d279e890bc398114f3010c7dbf69bb6f7d4993 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 15:07:34 +0000 Subject: [PATCH 23/63] Redirect to uploads after save contact list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The saved uploads page just shows you the same thing you’ve seen on the previous page – a list of email addresses. It’s more useful to see the uploads page, so you can see where the thing you’ve saved has gone. You can see it more in context. --- app/main/views/uploads.py | 3 +-- tests/app/main/views/test_uploads.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index e3c6d09ad..4398152eb 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -429,9 +429,8 @@ def check_contact_list(service_id, upload_id): def save_contact_list(service_id, upload_id): ContactList.create(current_service.id, upload_id) return redirect(url_for( - '.contact_list', + '.uploads', service_id=current_service.id, - contact_list_id=upload_id, )) diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 13daaf5dc..980b087e5 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -1138,9 +1138,8 @@ def test_save_contact_list( upload_id=fake_uuid, _expected_status=302, _expected_redirect=url_for( - 'main.contact_list', + 'main.uploads', service_id=SERVICE_ONE_ID, - contact_list_id=fake_uuid, _external=True, ) ) From 585fdd90768af1e6fb87b3e8b92f6d52b772b5c2 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 16:10:25 +0000 Subject: [PATCH 24/63] Use the correct name for the contact list bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything else is production. The bucket is currently called production. The fact that the CSV bucket is called `live-` is a legacy thing that’s hard to change. --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 85a72ccb9..9d470974e 100644 --- a/app/config.py +++ b/app/config.py @@ -178,7 +178,7 @@ class Live(Config): HTTP_PROTOCOL = 'https' STATSD_ENABLED = True CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload' - CONTACT_LIST_UPLOAD_BUCKET_NAME = 'live-contact-list' + CONTACT_LIST_UPLOAD_BUCKET_NAME = 'production-contact-list' LOGO_UPLOAD_BUCKET_NAME = 'public-logos-production' MOU_BUCKET_NAME = 'notifications.service.gov.uk-mou' TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters' From fc58547be30f5fe7b09fa8b7c538815b94e458e4 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 17:00:34 +0000 Subject: [PATCH 25/63] =?UTF-8?q?Tell=20the=20API=20which=20contact=20list?= =?UTF-8?q?=20a=20job=E2=80=99s=20come=20from?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So that we can make enhancements to the UI in the future, for example grouping jobs within their associated contact list. --- app/main/views/send.py | 4 ++- app/notify_client/job_api_client.py | 5 ++- app/templates/views/check/ok.html | 1 + tests/app/main/views/test_send.py | 38 ++++++++++++++++++++-- tests/app/notify_client/test_job_client.py | 15 +++++++++ tests/conftest.py | 2 +- 6 files changed, 60 insertions(+), 5 deletions(-) diff --git a/app/main/views/send.py b/app/main/views/send.py index bde1b3a85..7784dd902 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -530,6 +530,7 @@ def send_from_contact_list(service_id, template_id, contact_list_id): template_id=template_id, upload_id=contact_list.copy_to_uploads(), original_file_name=contact_list.original_file_name, + contact_list_id=contact_list.id, )) @@ -742,7 +743,8 @@ def start_job(service_id, upload_id): job_api_client.create_job( upload_id, service_id, - scheduled_for=request.form.get('scheduled_for', '') + scheduled_for=request.form.get('scheduled_for', ''), + contact_list_id=request.form.get('contact_list_id', ''), ) session.pop('sender_id', None) diff --git a/app/notify_client/job_api_client.py b/app/notify_client/job_api_client.py index 803bbe26e..f143da94b 100644 --- a/app/notify_client/job_api_client.py +++ b/app/notify_client/job_api_client.py @@ -78,12 +78,15 @@ class JobApiClient(NotifyAdminAPIClient): def has_jobs(self, service_id): return bool(self.get_jobs(service_id)['data']) - def create_job(self, job_id, service_id, scheduled_for=None): + def create_job(self, job_id, service_id, scheduled_for=None, contact_list_id=None): data = {"id": job_id} if scheduled_for: data.update({'scheduled_for': scheduled_for}) + if contact_list_id: + data.update({'contact_list_id': contact_list_id}) + data = _attach_current_user(data) job = self.post(url='/service/{}/job'.format(service_id), data=data) diff --git a/app/templates/views/check/ok.html b/app/templates/views/check/ok.html index 3ec9d2ac9..7554f18c0 100644 --- a/app/templates/views/check/ok.html +++ b/app/templates/views/check/ok.html @@ -37,6 +37,7 @@
    + {% if choose_time_form and template.template_type != 'letter' %} {{ radio_select( choose_time_form.scheduled_for, diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 3a4b671ee..f5a582b2b 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -2264,6 +2264,31 @@ def test_upload_csvfile_with_international_validates( assert mock_recipients.call_args[1]['international_sms'] == should_allow_international +def test_job_from_contact_list_knows_where_its_come_from( + client_request, + mocker, + service_one, + mock_get_service_template, + mock_s3_download, + mock_get_users_by_service, + mock_get_service_statistics, + mock_get_job_doesnt_exist, + mock_get_jobs, + mock_s3_set_metadata, + fake_uuid +): + page = client_request.get( + 'main.check_messages', + service_id=service_one['id'], + upload_id=fake_uuid, + template_id=fake_uuid, + contact_list_id=unchanging_fake_uuid, + ) + assert page.select_one( + 'form input[type=hidden][name=contact_list_id]' + )['value'] == str(unchanging_fake_uuid) + + def test_test_message_can_only_be_sent_now( client_request, mocker, @@ -2352,6 +2377,9 @@ def test_send_button_is_correctly_labelled( @pytest.mark.parametrize('when', [ '', '2016-08-25T13:04:21.767198' ]) +@pytest.mark.parametrize('contact_list_id', [ + '', unchanging_fake_uuid, +]) def test_create_job_should_call_api( client_request, mock_create_job, @@ -2361,7 +2389,8 @@ def test_create_job_should_call_api( mock_get_service_data_retention, mocker, fake_uuid, - when + when, + contact_list_id, ): data = mock_get_job(SERVICE_ONE_ID, fake_uuid)['data'] job_id = data['id'] @@ -2382,7 +2411,10 @@ def test_create_job_should_call_api( service_id=SERVICE_ONE_ID, upload_id=job_id, original_file_name=original_file_name, - _data={'scheduled_for': when}, + _data={ + 'scheduled_for': when, + 'contact_list_id': contact_list_id, + }, _follow_redirects=True, _expected_status=200, ) @@ -2393,6 +2425,7 @@ def test_create_job_should_call_api( job_id, SERVICE_ONE_ID, scheduled_for=when, + contact_list_id=str(contact_list_id), ) @@ -4009,6 +4042,7 @@ def test_send_from_contact_list( template_id=fake_uuid, upload_id=new_uuid, original_file_name='EmergencyContactList.xls', + contact_list_id=fake_uuid, _external=True, ) ) diff --git a/tests/app/notify_client/test_job_client.py b/tests/app/notify_client/test_job_client.py index b3bffbbe2..e85f4c4c2 100644 --- a/tests/app/notify_client/test_job_client.py +++ b/tests/app/notify_client/test_job_client.py @@ -50,6 +50,21 @@ def test_client_schedules_job(mocker, fake_uuid): assert mock_post.call_args[1]['data']['scheduled_for'] == when +def test_client_links_job_to_contact_list(mocker, fake_uuid): + + mocker.patch('app.notify_client.current_user', id='1') + + contact_list_id = uuid.uuid4() + + mock_post = mocker.patch('app.notify_client.job_api_client.JobApiClient.post') + + JobApiClient().create_job( + fake_uuid, 1, contact_list_id=contact_list_id + ) + + assert mock_post.call_args[1]['data']['contact_list_id'] == contact_list_id + + def test_client_gets_job_by_service_and_job(mocker): service_id = 'service_id' job_id = 'job_id' diff --git a/tests/conftest.py b/tests/conftest.py index c4360c140..123437186 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1638,7 +1638,7 @@ def mock_check_verify_code_code_expired(mocker): @pytest.fixture(scope='function') def mock_create_job(mocker, api_user_active): - def _create(job_id, service_id, scheduled_for=None): + def _create(job_id, service_id, scheduled_for=None, contact_list_id=None): return job_json( service_id, api_user_active, From 3dfa401de1c73fc866b6bf8d1d125dddec7c1481 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Mar 2020 14:57:36 +0000 Subject: [PATCH 26/63] Ship the contact lists feature This reveals the links that we were only showing to platform admin users so that everyone can discover the new feature. --- app/templates/views/jobs/jobs.html | 25 +++++++++++-------------- app/templates/views/send-test.html | 2 +- tests/app/main/views/test_send.py | 22 ++++++++++------------ tests/app/main/views/test_uploads.py | 26 ++++++++++++-------------- 4 files changed, 34 insertions(+), 41 deletions(-) diff --git a/app/templates/views/jobs/jobs.html b/app/templates/views/jobs/jobs.html index 77b51794a..0b88d0c34 100644 --- a/app/templates/views/jobs/jobs.html +++ b/app/templates/views/jobs/jobs.html @@ -31,25 +31,22 @@

    {% endif %} {{ previous_next_navigation(prev_page, next_page) }} - {% if ( - current_service.can_upload_letters and - current_user.has_permissions('send_messages') - ) or current_user.platform_admin %} + {% if current_user.has_permissions('send_messages') %}
    - {{ govukButton({ - "element": "a", - "text": "Upload a letter", - "href": url_for('.upload_letter', service_id=current_service.id), - "classes": "govuk-button--secondary" - }) }} - {% if current_user.platform_admin %} + {% if current_service.has_permission('letter') %} {{ govukButton({ "element": "a", - "text": "Upload an emergency contact list", - "href": url_for('.upload_contact_list', service_id=current_service.id), - "classes": "govuk-button--secondary govuk-!-margin-left-3" + "text": "Upload a letter", + "href": url_for('.upload_letter', service_id=current_service.id), + "classes": "govuk-button--secondary" }) }} {% endif %} + {{ govukButton({ + "element": "a", + "text": "Upload an emergency contact list", + "href": url_for('.upload_contact_list', service_id=current_service.id), + "classes": "govuk-button--secondary govuk-!-margin-left-3" + }) }}
    {% endif %}
    diff --git a/app/templates/views/send-test.html b/app/templates/views/send-test.html index 9ea19106c..ad3915678 100644 --- a/app/templates/views/send-test.html +++ b/app/templates/views/send-test.html @@ -33,7 +33,7 @@
    {% if link_to_upload %} Upload a list of {{ recipient_count_label(999, template.template_type) }} - {% if current_user.platform_admin and current_service.contact_lists %} + {% if current_service.contact_lists %} Use a saved list {% endif %} {% endif %} diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index f5a582b2b..99bcfe264 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -1066,6 +1066,7 @@ def test_send_test_step_redirects_if_session_not_setup( mock_get_service_statistics, mock_get_users_by_service, mock_has_no_jobs, + mock_get_no_contact_lists, fake_uuid, user, endpoint, @@ -1180,6 +1181,7 @@ def test_send_one_off_or_test_has_correct_page_titles( logged_in_client, service_one, mock_has_no_jobs, + mock_get_no_contact_lists, fake_uuid, mocker, template_type, @@ -1241,6 +1243,7 @@ def test_send_one_off_or_test_shows_placeholders_in_correct_order( client_request, fake_uuid, mock_has_no_jobs, + mock_get_no_contact_lists, mock_get_service_template_with_multiple_placeholders, endpoint, step_index, @@ -1292,6 +1295,7 @@ def test_send_one_off_has_skip_link( fake_uuid, mock_get_service_email_template, mock_has_no_jobs, + mock_get_no_contact_lists, mocker, template_type, expected_link_text, @@ -1334,6 +1338,7 @@ def test_send_one_off_has_sticky_header_for_email_and_letter( client_request, fake_uuid, mock_has_no_jobs, + mock_get_no_contact_lists, template_type, expected_sticky, ): @@ -1362,6 +1367,7 @@ def test_skip_link_will_not_show_on_sms_one_off_if_service_has_no_mobile_number( fake_uuid, mock_get_service_template, mock_has_no_jobs, + mock_get_no_contact_lists, mocker, user, ): @@ -1387,6 +1393,7 @@ def test_send_one_off_offers_link_to_upload( fake_uuid, mock_get_service_template, mock_has_jobs, + mock_get_no_contact_lists, user, ): client_request.login(user) @@ -1411,24 +1418,13 @@ def test_send_one_off_offers_link_to_upload( ) -@pytest.mark.parametrize('user', ( - pytest.param( - create_platform_admin_user(), - ), - pytest.param( - create_active_user_with_permissions(), - marks=pytest.mark.xfail(raises=AssertionError), - ), -)) -def test_platform_admin_has_link_to_use_existing_list( +def test_send_one_off_has_link_to_use_existing_list( client_request, mock_get_service_template, mock_has_jobs, mock_get_contact_lists, fake_uuid, - user, ): - client_request.login(user) page = client_request.get( 'main.send_one_off', service_id=SERVICE_ONE_ID, @@ -3787,6 +3783,7 @@ def test_reply_to_is_previewed_if_chosen( mock_get_service_statistics, mock_get_job_doesnt_exist, mock_get_jobs, + mock_get_no_contact_lists, get_default_reply_to_email_address, fake_uuid, endpoint, @@ -3838,6 +3835,7 @@ def test_sms_sender_is_previewed( mock_get_service_statistics, mock_get_job_doesnt_exist, mock_get_jobs, + mock_get_no_contact_lists, get_default_sms_sender, fake_uuid, endpoint, diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 980b087e5..6fa7a4199 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -20,15 +20,18 @@ from tests.conftest import ( @pytest.mark.parametrize('extra_permissions', ( - [], - ['letter'], - ['upload_letters'], pytest.param( - ['letter', 'upload_letters'], + [], marks=pytest.mark.xfail(raises=AssertionError), ), + pytest.param( + ['upload_letters'], + marks=pytest.mark.xfail(raises=AssertionError), + ), + ['letter'], + ['letter', 'upload_letters'], )) -def test_no_upload_letters_button_without_permission( +def test_upload_letters_button_only_with_letters_permission( client_request, service_one, mock_get_uploads, @@ -38,19 +41,14 @@ def test_no_upload_letters_button_without_permission( ): service_one['permissions'] += extra_permissions page = client_request.get('main.uploads', service_id=SERVICE_ONE_ID) - assert not page.find('a', text=re.compile('Upload a letter')) + assert page.find('a', text=re.compile('Upload a letter')) @pytest.mark.parametrize('user', ( - pytest.param( - create_platform_admin_user(), - ), - pytest.param( - create_active_user_with_permissions(), - marks=pytest.mark.xfail(raises=AssertionError), - ), + create_platform_admin_user(), + create_active_user_with_permissions(), )) -def test_platform_admin_has_upload_contact_list( +def test_all_users_have_upload_contact_list( client_request, mock_get_uploads, mock_get_jobs, From 82634f79de11a2753793de3c63fdb3a8ab19ad84 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 17 Mar 2020 11:30:27 +0000 Subject: [PATCH 27/63] Use new API endpoint to get single contact list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When we first built the contact list feature this endpoint didn’t exist. Now it does we can remove the slightly cludgy looping-through-all-the-lists code. --- app/models/contact_list.py | 9 +++--- app/notify_client/contact_list_api_client.py | 5 +++- tests/app/main/views/test_send.py | 2 +- tests/app/main/views/test_uploads.py | 6 ++-- tests/conftest.py | 30 ++++++++++++++++++++ 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/app/models/contact_list.py b/app/models/contact_list.py index 6afbf9911..70378cb12 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -32,11 +32,10 @@ class ContactList(JSONModel): @classmethod def from_id(cls, contact_list_id, *, service_id): - # This is temporary until we have a get single list endpoint - for contact_list in ContactLists(service_id): - if contact_list.id == contact_list_id: - return contact_list - abort(404) + return cls(contact_list_api_client.get_contact_list( + service_id=service_id, + contact_list_id=contact_list_id, + )) @staticmethod def get_bucket_name(): diff --git a/app/notify_client/contact_list_api_client.py b/app/notify_client/contact_list_api_client.py index 9062addf0..9315d57d8 100644 --- a/app/notify_client/contact_list_api_client.py +++ b/app/notify_client/contact_list_api_client.py @@ -25,7 +25,10 @@ class ContactListApiClient(NotifyAdminAPIClient): return job def get_contact_lists(self, service_id): - return self.get('/service/{}/contact-list'.format(service_id)) + return self.get(f'/service/{service_id}/contact-list') + + def get_contact_list(self, *, service_id, contact_list_id): + return self.get(f'/service/{service_id}/contact-list/{contact_list_id}') contact_list_api_client = ContactListApiClient() diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 3a4b671ee..29891809d 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -3988,7 +3988,7 @@ def test_send_from_contact_list( mocker, client_request, fake_uuid, - mock_get_contact_lists, + mock_get_contact_list, ): new_uuid = uuid.uuid4() mock_download = mocker.patch('app.models.contact_list.s3download', return_value='contents') diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 13daaf5dc..25847566b 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -1183,7 +1183,7 @@ def test_cant_save_bad_contact_list( def test_view_contact_list( mocker, client_request, - mock_get_contact_lists, + mock_get_contact_list, fake_uuid, ): mocker.patch('app.models.contact_list.s3download', return_value='\n'.join( @@ -1225,7 +1225,7 @@ def test_view_contact_list( def test_view_contact_list_404s_for_non_existing_list( client_request, - mock_get_contact_lists, + mock_get_no_contact_list, fake_uuid, ): client_request.get( @@ -1240,7 +1240,7 @@ def test_download_contact_list( mocker, logged_in_client, fake_uuid, - mock_get_contact_lists, + mock_get_contact_list, ): mocker.patch( 'app.models.contact_list.s3download', diff --git a/tests/conftest.py b/tests/conftest.py index c4360c140..b8fdaf7aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1869,6 +1869,36 @@ def mock_get_contact_lists(mocker, api_user_active, fake_uuid): ) +@pytest.fixture(scope='function') +def mock_get_contact_list(mocker, api_user_active, fake_uuid): + def _get(*, service_id, contact_list_id): + return { + 'created_at': '2020-03-13 10:59:56', + 'created_by': 'Test User', + 'id': fake_uuid, + 'original_file_name': 'EmergencyContactList.xls', + 'row_count': 100, + 'service_id': service_id, + 'template_type': 'email', + } + + return mocker.patch( + 'app.models.contact_list.contact_list_api_client.get_contact_list', + side_effect=_get, + ) + + +@pytest.fixture(scope='function') +def mock_get_no_contact_list(mocker, api_user_active, fake_uuid): + def _get(*, service_id, contact_list_id): + raise HTTPError(response=Mock(status_code=404)) + + return mocker.patch( + 'app.models.contact_list.contact_list_api_client.get_contact_list', + side_effect=_get, + ) + + @pytest.fixture(scope='function') def mock_get_no_contact_lists(mocker): return mocker.patch( From 39fe27be39c351386d986ff1ec3878ac8e21dccc Mon Sep 17 00:00:00 2001 From: David McDonald Date: Thu, 12 Mar 2020 17:19:57 +0000 Subject: [PATCH 28/63] Pull in version of utils that validates postcodes for CSVs This will mean that a user can not send a CSV for letters with invalid postcodes and must fix their data. See https://github.com/alphagov/notifications-utils/pull/704 --- requirements-app.txt | 2 +- requirements.txt | 10 +++++----- tests/app/main/test_errors_for_csv.py | 10 ++++++++++ tests/app/main/views/test_send.py | 9 ++++++--- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/requirements-app.txt b/requirements-app.txt index d64f192c6..95332f00a 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -24,5 +24,5 @@ awscli-cwlogs>=1.4,<1.5 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default itsdangerous==0.24 # pyup: <1.0.0 -git+https://github.com/alphagov/notifications-utils.git@36.6.2#egg=notifications-utils==36.6.2 +git+https://github.com/alphagov/notifications-utils.git@36.9.1#egg=notifications-utils==36.9.1 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha diff --git a/requirements.txt b/requirements.txt index df55b1d98..49e7f5607 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,17 +26,17 @@ awscli-cwlogs>=1.4,<1.5 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default itsdangerous==0.24 # pyup: <1.0.0 -git+https://github.com/alphagov/notifications-utils.git@36.6.2#egg=notifications-utils==36.6.2 +git+https://github.com/alphagov/notifications-utils.git@36.9.1#egg=notifications-utils==36.9.1 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha ## The following requirements were added by pip freeze: -awscli==1.18.13 -bleach==3.1.1 +awscli==1.18.23 +bleach==3.1.3 boto3==1.10.38 -botocore==1.15.13 +botocore==1.15.23 certifi==2019.11.28 chardet==3.0.4 -Click==7.0 +click==7.1.1 colorama==0.4.3 dnspython==1.16.0 docopt==0.6.2 diff --git a/tests/app/main/test_errors_for_csv.py b/tests/app/main/test_errors_for_csv.py index 3e1fc3add..d455c54b1 100644 --- a/tests/app/main/test_errors_for_csv.py +++ b/tests/app/main/test_errors_for_csv.py @@ -44,6 +44,16 @@ MockRecipients = namedtuple( 'email', ['fix 3 email addresses'] ), + ( + {2}, [], [], [], + 'letter', + ['fix 1 address'] + ), + ( + {2, 4}, [], [], [], + 'letter', + ['fix 2 addresses'] + ), ( {2}, {3}, [], [], 'sms', diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 99bcfe264..04a80f689 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -2321,7 +2321,10 @@ def test_letter_can_only_be_sent_now( mock_get_jobs, fake_uuid, ): - mocker.patch('app.main.views.send.s3download', return_value="addressline1, addressline2, postcode\na,b,c") + mocker.patch( + 'app.main.views.send.s3download', + return_value="addressline1, addressline2, postcode\na,b,sw1 1aa" + ) mocker.patch('app.main.views.send.set_metadata_on_csv_upload') mocker.patch('app.main.views.send.get_page_count_for_letter', return_value=1) @@ -3021,8 +3024,8 @@ def test_check_messages_shows_data_errors_before_trial_mode_errors_for_letters( mocker.patch('app.main.views.send.s3download', return_value='\n'.join( ['address_line_1,address_line_2,postcode,'] + - [' , ,11SW1 1AA'] + - [' , ,11SW1 1AA'] + [' , ,SW1 1AA'] + + [' , ,SW1 1AA'] )) mocker.patch( From a5d8de502841b274e787a4bb032104b40e23c197 Mon Sep 17 00:00:00 2001 From: Tom Byers Date: Wed, 18 Mar 2020 16:54:00 +0000 Subject: [PATCH 29/63] Remove POST param from agreement test This test sent an empty string to represent the 'who' field not having a value. This is represented by a group of radios in the page. In browsers, radios that aren't selected result in that field not being included in the HTTP parameters. This change makes the test match that. The expected validation message is also changed to the one you see in a browser. Turns out [the WTForms DataRequired validator](https://wtforms.readthedocs.io/en/stable/validators.html#wtforms.validators.DataRequired) produces a different error message for an empty string vs the value being missing ("This field is required." vs "Not a valid choice"). --- tests/app/main/views/test_agreement.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/app/main/views/test_agreement.py b/tests/app/main/views/test_agreement.py index 81c106d40..24ae7ebc3 100644 --- a/tests/app/main/views/test_agreement.py +++ b/tests/app/main/views/test_agreement.py @@ -272,12 +272,11 @@ def test_accept_agreement_page_populates( ( { 'version': '', - 'who': '', 'on_behalf_of_name': '', 'on_behalf_of_email': '', }, [ - 'This field is required.', + 'Not a valid choice', 'Must be a number', ], ), From 2a76fd9ee88164fd52b0298eaca1a8f7c33c3a7d Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 19 Mar 2020 10:49:40 +0000 Subject: [PATCH 30/63] Reduce usage of the platform admin index page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This page is slow to load which means: - it’s annoying for us - it’s potentially causing load on the database This commit does two things to reduce the amount we’re unnecessarily looking at this page: 1. Avoid redirecting to it when signing in as a platform admin user 2. Don’t go directly to it when clicking ‘platform admin’ at the top, but instead show a holding page (there’s a fair chance you’ve clicked that link in order to go and manage some email branding or find a user, not wait for stats to load) --- app/main/views/platform_admin.py | 8 ++++++++ app/main/views/two_factor.py | 2 -- app/navigation.py | 4 ++++ app/templates/admin_template.html | 2 +- .../views/platform-admin/splash-page.html | 17 +++++++++++++++++ tests/app/main/views/test_platform_admin.py | 15 +++++++++++++++ tests/app/main/views/test_two_factor.py | 6 ++++++ 7 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 app/templates/views/platform-admin/splash-page.html diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 25b727630..50917c819 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -42,6 +42,14 @@ ZERO_FAILURE_THRESHOLD = 0 @main.route("/platform-admin") @user_is_platform_admin +def platform_admin_splash_page(): + return render_template( + 'views/platform-admin/splash-page.html', + ) + + +@main.route("/platform-admin/summary") +@user_is_platform_admin def platform_admin(): form = DateFilterForm(request.args, meta={'csrf': False}) api_args = {} diff --git a/app/main/views/two_factor.py b/app/main/views/two_factor.py index 0cd8b8444..00e2a3b53 100644 --- a/app/main/views/two_factor.py +++ b/app/main/views/two_factor.py @@ -111,7 +111,5 @@ def redirect_when_logged_in(platform_admin): next_url = request.args.get('next') if next_url and _is_safe_redirect_url(next_url): return redirect(next_url) - if platform_admin: - return redirect(url_for('main.platform_admin')) return redirect(url_for('main.show_accounts_or_dashboard')) diff --git a/app/navigation.py b/app/navigation.py index 4969af085..2342429a2 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -101,6 +101,7 @@ class HeaderNavigation(Navigation): 'platform_admin_list_complaints', 'platform_admin_reports', 'platform_admin_returned_letters', + 'platform_admin_splash_page', 'suspend_service', 'trial_services', 'update_email_branding', @@ -580,6 +581,7 @@ class MainNavigation(Navigation): 'platform_admin_list_complaints', 'platform_admin_reports', 'platform_admin_returned_letters', + 'platform_admin_splash_page', 'pricing', 'privacy', 'public_agreement', @@ -828,6 +830,7 @@ class CaseworkNavigation(Navigation): 'platform_admin_reports', 'platform_admin_returned_letters', 'platform_admin', + 'platform_admin_splash_page', 'pricing', 'privacy', 'public_agreement', @@ -1117,6 +1120,7 @@ class OrgNavigation(Navigation): 'platform_admin_list_complaints', 'platform_admin_reports', 'platform_admin_returned_letters', + 'platform_admin_splash_page', 'pricing', 'privacy', 'public_agreement', diff --git a/app/templates/admin_template.html b/app/templates/admin_template.html index 91eec3123..96e3d0cf4 100644 --- a/app/templates/admin_template.html +++ b/app/templates/admin_template.html @@ -55,7 +55,7 @@ "active": header_navigation.is_selected('user-profile') }, { - "href": url_for('main.platform_admin'), + "href": url_for('main.platform_admin_splash_page'), "text": "Platform admin", "active": header_navigation.is_selected('platform-admin') }, diff --git a/app/templates/views/platform-admin/splash-page.html b/app/templates/views/platform-admin/splash-page.html new file mode 100644 index 000000000..d048ef002 --- /dev/null +++ b/app/templates/views/platform-admin/splash-page.html @@ -0,0 +1,17 @@ +{% extends "views/platform-admin/_base_template.html" %} + +{% block per_page_title %} + Summary +{% endblock %} + +{% block platform_admin_content %} + +

    + Summary +

    + +

    + Load summary +

    + +{% endblock %} diff --git a/tests/app/main/views/test_platform_admin.py b/tests/app/main/views/test_platform_admin.py index 948f2a241..a157e3e46 100644 --- a/tests/app/main/views/test_platform_admin.py +++ b/tests/app/main/views/test_platform_admin.py @@ -37,6 +37,7 @@ def test_should_redirect_if_not_logged_in( @pytest.mark.parametrize('endpoint', [ 'main.platform_admin', + 'main.platform_admin_splash_page', 'main.live_services', 'main.trial_services', ]) @@ -589,6 +590,20 @@ def test_get_tech_failure_status_box_data_removes_percentage_data(): assert 'percentage' not in tech_failure_data +def test_platform_admin_splash_doesnt_talk_to_api( + client_request, + platform_admin_user, +): + + client_request.login(platform_admin_user) + + page = client_request.get('main.platform_admin_splash_page') + + assert page.select_one('main .govuk-body a')['href'] == url_for( + 'main.platform_admin', + ) + + def test_platform_admin_with_start_and_end_dates_provided(mocker, platform_admin_client): start_date = '2018-01-01' end_date = '2018-06-01' diff --git a/tests/app/main/views/test_two_factor.py b/tests/app/main/views/test_two_factor.py index f8333c5c8..6b0e4d9d3 100644 --- a/tests/app/main/views/test_two_factor.py +++ b/tests/app/main/views/test_two_factor.py @@ -1,3 +1,4 @@ +import pytest from bs4 import BeautifulSoup from flask import url_for from freezegun import freeze_time @@ -107,6 +108,9 @@ def test_should_login_user_and_not_redirect_to_external_url( assert response.location == url_for('main.show_accounts_or_dashboard', _external=True) +@pytest.mark.parametrize('platform_admin', ( + True, False, +)) @freeze_time('2020-01-27T12:00:00') def test_should_login_user_and_redirect_to_show_accounts( client, @@ -115,12 +119,14 @@ def test_should_login_user_and_redirect_to_show_accounts( mock_get_user_by_email, mock_check_verify_code, mock_create_event, + platform_admin, ): with client.session_transaction() as session: session['user_details'] = { 'id': api_user_active['id'], 'email': api_user_active['email_address']} api_user_active['email_access_validated_at'] = '2020-01-23T11:35:21.726132Z' + api_user_active['platform_admin'] = platform_admin response = client.post(url_for('main.two_factor'), data={'sms_code': '12345'}) From d0f3875d296f391a9d4605cde6d3646b31e5a616 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 19 Mar 2020 12:30:29 +0000 Subject: [PATCH 31/63] Slow down the update interval on the dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dashboard is the most intensive page we AJAX, and also the highest traffic one. We’ve already slowed it from 2 to 5 seconds, this slows it further to 20 seconds to reduce the load. This leaves other pages (for example looking at a single job) at the platform-level default of 5 seconds, because we think they cause less load and the real-timelyness isn’t critical to people’s business processes. For looking at a single notification we know from research that someone sending these one-at-a-time often waits to see if they’re delivered, so let’s bring this back down to the previous value of 2 seconds. --- app/templates/views/dashboard/dashboard.html | 10 +++++----- app/templates/views/notifications/notification.html | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/templates/views/dashboard/dashboard.html b/app/templates/views/dashboard/dashboard.html index fa8405690..e7332e9f5 100644 --- a/app/templates/views/dashboard/dashboard.html +++ b/app/templates/views/dashboard/dashboard.html @@ -19,25 +19,25 @@ {% include 'views/dashboard/write-first-messages.html' %} {% endif %} - {{ ajax_block(partials, updates_url, 'upcoming') }} + {{ ajax_block(partials, updates_url, 'upcoming', interval=20) }}

    In the last 7 days

    - {{ ajax_block(partials, updates_url, 'inbox') }} + {{ ajax_block(partials, updates_url, 'inbox', interval=20) }} - {{ ajax_block(partials, updates_url, 'totals') }} + {{ ajax_block(partials, updates_url, 'totals', interval=20) }} {{ show_more( url_for('.monthly', service_id=current_service.id), 'See messages sent per month' ) }} - {{ ajax_block(partials, updates_url, 'template-statistics') }} + {{ ajax_block(partials, updates_url, 'template-statistics', interval=20) }} {% if current_user.has_permissions('manage_service') %}

    This year

    - {{ ajax_block(partials, updates_url, 'usage') }} + {{ ajax_block(partials, updates_url, 'usage', interval=20) }} {{ show_more( url_for(".usage", service_id=current_service['id']), 'See usage' diff --git a/app/templates/views/notifications/notification.html b/app/templates/views/notifications/notification.html index 288cb7ee4..63bf4ceed 100644 --- a/app/templates/views/notifications/notification.html +++ b/app/templates/views/notifications/notification.html @@ -96,10 +96,10 @@
    {% elif template.template_type == 'email' %}
    - {{ ajax_block(partials, updates_url, 'status', finished=finished) }} + {{ ajax_block(partials, updates_url, 'status', interval=2, finished=finished) }}
    {% elif template.template_type == 'sms' %} - {{ ajax_block(partials, updates_url, 'status', finished=finished) }} + {{ ajax_block(partials, updates_url, 'status', interval=2, finished=finished) }} {% endif %} {% if current_user.has_permissions('send_messages') and current_user.has_permissions('view_activity') and template.template_type == 'sms' and can_receive_inbound %} From 8f15eee5e0aad7691035473b221cb44ca4c312aa Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 18 Mar 2020 13:37:39 +0000 Subject: [PATCH 32/63] Let users delete a contact list Follows our standard pattern of confirming with a red banner for actions that are destructive. --- app/main/views/uploads.py | 23 +++++++++ app/models/contact_list.py | 6 +++ app/navigation.py | 4 ++ app/notify_client/contact_list_api_client.py | 3 ++ .../uploads/contact-list/contact-list.html | 10 ++++ tests/app/main/views/test_uploads.py | 49 +++++++++++++++++++ 6 files changed, 95 insertions(+) diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index 4398152eb..20cc47f64 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -443,6 +443,29 @@ def contact_list(service_id, contact_list_id): ) +@main.route("/services//contact-list//delete", methods=['GET', 'POST']) +@user_has_permissions('manage_templates') +def delete_contact_list(service_id, contact_list_id): + contact_list = ContactList.from_id(contact_list_id, service_id=service_id) + + if request.method == 'POST': + contact_list.delete() + return redirect(url_for( + '.uploads', + service_id=service_id, + )) + + flash([ + f"Are you sure you want to delete ‘{contact_list.original_file_name}’?", + ], 'delete') + + return render_template( + 'views/uploads/contact-list/contact-list.html', + contact_list=contact_list, + confirm_delete_banner=True, + ) + + @main.route("/services//contact-list/.csv", methods=['GET']) @user_has_permissions('send_messages') def download_contact_list(service_id, contact_list_id): diff --git a/app/models/contact_list.py b/app/models/contact_list.py index 70378cb12..95cfdfdf2 100644 --- a/app/models/contact_list.py +++ b/app/models/contact_list.py @@ -105,6 +105,12 @@ class ContactList(JSONModel): template_type=metadata['template_type'], )) + def delete(self): + contact_list_api_client.delete_contact_list( + service_id=self.service_id, + contact_list_id=self.id, + ) + @property def contents(self): return self.download(self.service_id, self.id) diff --git a/app/navigation.py b/app/navigation.py index 2342429a2..6975793cb 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -325,6 +325,7 @@ class HeaderNavigation(Navigation): 'check_contact_list', 'save_contact_list', 'contact_list', + 'delete_contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -398,6 +399,7 @@ class MainNavigation(Navigation): 'check_contact_list', 'save_contact_list', 'contact_list', + 'delete_contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -682,6 +684,7 @@ class CaseworkNavigation(Navigation): 'check_contact_list', 'save_contact_list', 'contact_list', + 'delete_contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -1227,6 +1230,7 @@ class OrgNavigation(Navigation): 'check_contact_list', 'save_contact_list', 'contact_list', + 'delete_contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', diff --git a/app/notify_client/contact_list_api_client.py b/app/notify_client/contact_list_api_client.py index 9315d57d8..9f7c04dc7 100644 --- a/app/notify_client/contact_list_api_client.py +++ b/app/notify_client/contact_list_api_client.py @@ -30,5 +30,8 @@ class ContactListApiClient(NotifyAdminAPIClient): def get_contact_list(self, *, service_id, contact_list_id): return self.get(f'/service/{service_id}/contact-list/{contact_list_id}') + def delete_contact_list(self, *, service_id, contact_list_id): + return self.delete(f'/service/{service_id}/contact-list/{contact_list_id}') + contact_list_api_client = ContactListApiClient() diff --git a/app/templates/views/uploads/contact-list/contact-list.html b/app/templates/views/uploads/contact-list/contact-list.html index 50c0cef79..eee292712 100644 --- a/app/templates/views/uploads/contact-list/contact-list.html +++ b/app/templates/views/uploads/contact-list/contact-list.html @@ -45,4 +45,14 @@

    {% endif %} + {% if not confirm_delete_banner %} +
    +
    + + Delete this contact list + +
    +
    + {% endif %} + {% endblock %} diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 1491b45d6..1e2300e7b 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -1261,3 +1261,52 @@ def test_download_contact_list( 'phone number\n' '07900900321' ) + + +def test_confirm_delete_contact_list( + mocker, + client_request, + fake_uuid, + mock_get_contact_list, +): + mocker.patch( + 'app.models.contact_list.s3download', + return_value='phone number\n07900900321' + ) + page = client_request.get( + 'main.delete_contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + ) + assert normalize_spaces(page.select_one('.banner-dangerous').text) == ( + 'Are you sure you want to delete ‘EmergencyContactList.xls’? ' + 'Yes, delete' + ) + assert 'action' not in page.select_one('form') + assert page.select_one('form')['method'] == 'post' + assert page.select_one('form button')['type'] == 'submit' + + +def test_delete_contact_list( + mocker, + client_request, + fake_uuid, + mock_get_contact_list, +): + mock_delete = mocker.patch( + 'app.models.contact_list.contact_list_api_client.delete_contact_list' + ) + client_request.post( + 'main.delete_contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + _expected_redirect=url_for( + 'main.uploads', + service_id=SERVICE_ONE_ID, + _external=True, + ) + ) + mock_delete.assert_called_once_with( + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + ) From 40ef8af39d1b0efd856edc8708351eb6e412c3f5 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 18 Mar 2020 13:45:51 +0000 Subject: [PATCH 33/63] Fix misaligned upload button when no letter permission We should add spacing to the right, so if the first button is missing the second one still sits flush-left. --- app/templates/views/jobs/jobs.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/templates/views/jobs/jobs.html b/app/templates/views/jobs/jobs.html index 0b88d0c34..30507f21d 100644 --- a/app/templates/views/jobs/jobs.html +++ b/app/templates/views/jobs/jobs.html @@ -38,14 +38,14 @@ "element": "a", "text": "Upload a letter", "href": url_for('.upload_letter', service_id=current_service.id), - "classes": "govuk-button--secondary" + "classes": "govuk-button--secondary govuk-!-margin-right-3" }) }} {% endif %} {{ govukButton({ "element": "a", "text": "Upload an emergency contact list", "href": url_for('.upload_contact_list', service_id=current_service.id), - "classes": "govuk-button--secondary govuk-!-margin-left-3" + "classes": "govuk-button--secondary" }) }}
    {% endif %} From 317135736b5ab6bd0d87d27370a9c5796a415cc9 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Thu, 19 Mar 2020 15:57:15 +0000 Subject: [PATCH 34/63] Add status page link to failwhale page --- paas-failwhale/index.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/paas-failwhale/index.html b/paas-failwhale/index.html index 9d1bf2036..5fae76688 100644 --- a/paas-failwhale/index.html +++ b/paas-failwhale/index.html @@ -10,7 +10,7 @@ - + @@ -46,7 +46,7 @@ fill="currentColor" fill-rule="evenodd" d="M25 30.2c3.5 1.5 7.7-.2 9.1-3.7 1.5-3.6-.2-7.8-3.9-9.2-3.6-1.4-7.6.3-9.1 3.9-1.4 3.5.3 7.5 3.9 9zM9 39.5c3.6 1.5 7.8-.2 9.2-3.7 1.5-3.6-.2-7.8-3.9-9.1-3.6-1.5-7.6.2-9.1 3.8-1.4 3.5.3 7.5 3.8 9zM4.4 57.2c3.5 1.5 7.7-.2 9.1-3.8 1.5-3.6-.2-7.7-3.9-9.1-3.5-1.5-7.6.3-9.1 3.8-1.4 3.5.3 7.6 3.9 9.1zm38.3-21.4c3.5 1.5 7.7-.2 9.1-3.8 1.5-3.6-.2-7.7-3.9-9.1-3.6-1.5-7.6.3-9.1 3.8-1.3 3.6.4 7.7 3.9 9.1zm64.4-5.6c-3.6 1.5-7.8-.2-9.1-3.7-1.5-3.6.2-7.8 3.8-9.2 3.6-1.4 7.7.3 9.2 3.9 1.3 3.5-.4 7.5-3.9 9zm15.9 9.3c-3.6 1.5-7.7-.2-9.1-3.7-1.5-3.6.2-7.8 3.7-9.1 3.6-1.5 7.7.2 9.2 3.8 1.5 3.5-.3 7.5-3.8 9zm4.7 17.7c-3.6 1.5-7.8-.2-9.2-3.8-1.5-3.6.2-7.7 3.9-9.1 3.6-1.5 7.7.3 9.2 3.8 1.3 3.5-.4 7.6-3.9 9.1zM89.3 35.8c-3.6 1.5-7.8-.2-9.2-3.8-1.4-3.6.2-7.7 3.9-9.1 3.6-1.5 7.7.3 9.2 3.8 1.4 3.6-.3 7.7-3.9 9.1zM69.7 17.7l8.9 4.7V9.3l-8.9 2.8c-.2-.3-.5-.6-.9-.9L72.4 0H59.6l3.5 11.2c-.3.3-.6.5-.9.9l-8.8-2.8v13.1l8.8-4.7c.3.3.6.7.9.9l-5 15.4v.1c-.2.8-.4 1.6-.4 2.4 0 4.1 3.1 7.5 7 8.1h.2c.3 0 .7.1 1 .1.4 0 .7 0 1-.1h.2c4-.6 7.1-4.1 7.1-8.1 0-.8-.1-1.7-.4-2.4V34l-5.1-15.4c.4-.2.7-.6 1-.9zM66 92.8c16.9 0 32.8 1.1 47.1 3.2 4-16.9 8.9-26.7 14-33.5l-9.6-3.4c1 4.9 1.1 7.2 0 10.2-1.5-1.4-3-4.3-4.2-8.7L108.6 76c2.8-2 5-3.2 7.5-3.3-4.4 9.4-10 11.9-13.6 11.2-4.3-.8-6.3-4.6-5.6-7.9 1-4.7 5.7-5.9 8-.5 4.3-8.7-3-11.4-7.6-8.8 7.1-7.2 7.9-13.5 2.1-21.1-8 6.1-8.1 12.3-4.5 20.8-4.7-5.4-12.1-2.5-9.5 6.2 3.4-5.2 7.9-2 7.2 3.1-.6 4.3-6.4 7.8-13.5 7.2-10.3-.9-10.9-8-11.2-13.8 2.5-.5 7.1 1.8 11 7.3L80.2 60c-4.1 4.4-8 5.3-12.3 5.4 1.4-4.4 8-11.6 8-11.6H55.5s6.4 7.2 7.9 11.6c-4.2-.1-8-1-12.3-5.4l1.4 16.4c3.9-5.5 8.5-7.7 10.9-7.3-.3 5.8-.9 12.8-11.1 13.8-7.2.6-12.9-2.9-13.5-7.2-.7-5 3.8-8.3 7.1-3.1 2.7-8.7-4.6-11.6-9.4-6.2 3.7-8.5 3.6-14.7-4.6-20.8-5.8 7.6-5 13.9 2.2 21.1-4.7-2.6-11.9.1-7.7 8.8 2.3-5.5 7.1-4.2 8.1.5.7 3.3-1.3 7.1-5.7 7.9-3.5.7-9-1.8-13.5-11.2 2.5.1 4.7 1.3 7.5 3.3l-4.7-15.4c-1.2 4.4-2.7 7.2-4.3 8.7-1.1-3-.9-5.3 0-10.2l-9.5 3.4c5 6.9 9.9 16.7 14 33.5 14.8-2.1 30.8-3.2 47.7-3.2z" > - + @@ -74,6 +74,9 @@

    We’re performing some essential updates. Notify will be back shortly.

    +

    + Please check the status page for updates. +

    From d7385da0df7c513c8dae01fe5a81e7421ebebcb5 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 23 Mar 2020 11:04:03 +0000 Subject: [PATCH 35/63] Require email address for all support tickets We are seeing little benefit of allowing users to not put in their email address. This will mean that you must provide it for feedback, not just problems with the site. There could maybe be some more refactoring of the support templates as this is now very similar to the report a problem page but this is a quick fix so haven't gone too in depth. --- app/main/forms.py | 2 +- .../support/ask-question-give-feedback.html | 2 -- tests/app/main/views/test_feedback.py | 32 ++++--------------- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/app/main/forms.py b/app/main/forms.py index 84017130a..bbec39080 100644 --- a/app/main/forms.py +++ b/app/main/forms.py @@ -894,7 +894,7 @@ class SupportType(StripWhitespaceForm): class Feedback(StripWhitespaceForm): name = StringField('Name') - email_address = email_address(label='Email address', gov_user=False, required=False) + email_address = email_address(label='Email address', gov_user=False, required=True) feedback = TextAreaField('Your message', validators=[DataRequired(message="Cannot be empty")]) diff --git a/app/templates/views/support/ask-question-give-feedback.html b/app/templates/views/support/ask-question-give-feedback.html index 3a236dd40..77e9ec8f9 100644 --- a/app/templates/views/support/ask-question-give-feedback.html +++ b/app/templates/views/support/ask-question-give-feedback.html @@ -19,8 +19,6 @@ {% call form_wrapper() %} {{ textbox(form.feedback, width='1-1', hint='', rows=10, autosize=True) }} {% if not current_user.is_authenticated %} -

    Do you want a reply?

    -

    Leave your details below if you’d like a response.

    {{ textbox(form.name, width='1-1') }} {{ textbox(form.email_address, width='1-1') }} {% else %} diff --git a/tests/app/main/views/test_feedback.py b/tests/app/main/views/test_feedback.py index 12fd61ded..684fc2b12 100644 --- a/tests/app/main/views/test_feedback.py +++ b/tests/app/main/views/test_feedback.py @@ -204,34 +204,15 @@ def test_passes_user_details_through_flow( {'feedback': 'blah', 'name': 'Fred'}, {'feedback': 'blah'}, ]) -@pytest.mark.parametrize('ticket_type, expected_response, expected_redirect, expected_error', [ - ( - PROBLEM_TICKET_TYPE, - 200, - lambda: None, - element.Tag, - ), - ( - QUESTION_TICKET_TYPE, - 302, - partial( - url_for, - '.thanks', - email_address_provided=False, - out_of_hours_emergency=False, - _external=True, - ), - type(None), - ), +@pytest.mark.parametrize('ticket_type', [ + PROBLEM_TICKET_TYPE, + QUESTION_TICKET_TYPE, ]) -def test_email_address_required_for_problems( +def test_email_address_required_for_problems_and_questions( client_request, mocker, data, ticket_type, - expected_response, - expected_redirect, - expected_error ): mocker.patch('app.main.views.feedback.zendesk_client') client_request.logout() @@ -239,10 +220,9 @@ def test_email_address_required_for_problems( 'main.feedback', ticket_type=ticket_type, _data=data, - _expected_status=expected_response, - _expected_redirect=expected_redirect(), + _expected_status=200 ) - assert isinstance(page.find('span', {'class': 'error-message'}), expected_error) + assert isinstance(page.find('span', {'class': 'error-message'}), element.Tag) @freeze_time('2016-12-12 12:00:00.000000') From 9a3e53f57d1f4b141c911ba466a40ab7ceddafe2 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Mon, 23 Mar 2020 11:36:06 +0000 Subject: [PATCH 36/63] Mark field as optional As suggested by the design system --- app/main/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/forms.py b/app/main/forms.py index bbec39080..34e7c2918 100644 --- a/app/main/forms.py +++ b/app/main/forms.py @@ -893,7 +893,7 @@ class SupportType(StripWhitespaceForm): class Feedback(StripWhitespaceForm): - name = StringField('Name') + name = StringField('Name (optional)') email_address = email_address(label='Email address', gov_user=False, required=True) feedback = TextAreaField('Your message', validators=[DataRequired(message="Cannot be empty")]) From aba5e198057a744bece4355568a2a49939848e07 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 23 Mar 2020 12:03:01 +0000 Subject: [PATCH 37/63] Drop frequency of AJAX updates on notifications page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is another page that puts load on the database, so let’s treat it the same as the dashboard. --- app/templates/views/notifications.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/templates/views/notifications.html b/app/templates/views/notifications.html index 7c9bbabc0..d8bfa4bdd 100644 --- a/app/templates/views/notifications.html +++ b/app/templates/views/notifications.html @@ -31,7 +31,8 @@ {{ ajax_block( partials, url_for('.get_notifications_as_json', service_id=current_service.id, message_type=message_type, status=status), - 'counts' + 'counts', + interval=20 ) }} {% call form_wrapper( @@ -82,7 +83,8 @@ partials, url_for('.get_notifications_as_json', service_id=current_service.id, message_type=message_type, status=status, page=page), 'notifications', - form='search-form' + form='search-form', + interval=20 ) }} {% endblock %} From 7df45d6cce09e12ec3db35a390290cb1b2609c2a Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 23 Mar 2020 14:38:01 +0000 Subject: [PATCH 38/63] bump deployment timeout 5 minutes isn't long enough to deploy ten instances of the admin app - it turns out it takes marginally longer than 5 minutes to roll each instance one after the next. this can lead to confusion as the build fails, functional tests don't run, but the code may have deployed fine and be running on production. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0fccc7932..3e5415b44 100644 --- a/Makefile +++ b/Makefile @@ -183,7 +183,7 @@ cf-deploy: ## Deploys the app to Cloud Foundry cf v3-cancel-zdt-push ${CF_APP} || true cf v3-apply-manifest ${CF_APP} -f <(make -s generate-manifest) - cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete # fails after 5 mins if deploy doesn't work + CF_STARTUP_TIMEOUT=10 cf v3-zdt-push ${CF_APP} --wait-for-deploy-complete # fails after 5 mins if deploy doesn't work .PHONY: cf-deploy-prototype cf-deploy-prototype: cf-target ## Deploys the first prototype to Cloud Foundry From 61ab9661fb6f360d19df41b297c08c58544b1db9 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 24 Mar 2020 15:01:42 +0000 Subject: [PATCH 39/63] Direct members of the public somewhere more useful MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can’t give advice to members of the public, but increasingly we’re seeing them try to use our support form to ask. It would be better for them if we can direct them straight to somewhere more useful, before they have the chance to raise a support ticket. This commit replaces the report a problem/ask a question triaging for users who aren’t signed in. It’s not possible for non-signed-in users to raise an priority 1 ticket, so we never need to triage the tickets in this way. Instead we can triage people based on whether they work in the public sector or not. If they do then we send them on to the feedback form. If not then they go to a new page which contains some useful links. We’ve chosen these links based on some analysis of the support tickets we’ve received recently[1] 1. https://docs.google.com/spreadsheets/d/1uBQn-ZnCYfz6ltFaUKZpytgvBF0-MeshCLZ1cD74R0c/edit?usp=sharing --- app/main/forms.py | 11 +++ app/main/views/feedback.py | 40 ++++++-- app/navigation.py | 4 + app/templates/views/support/index.html | 28 +++--- app/templates/views/support/public.html | 48 ++++++++++ tests/app/main/views/test_feedback.py | 118 +++++++++++++++++------- 6 files changed, 199 insertions(+), 50 deletions(-) create mode 100644 app/templates/views/support/public.html diff --git a/app/main/forms.py b/app/main/forms.py index 34e7c2918..c0e9e1b24 100644 --- a/app/main/forms.py +++ b/app/main/forms.py @@ -892,6 +892,17 @@ class SupportType(StripWhitespaceForm): ) +class SupportRedirect(StripWhitespaceForm): + who = RadioField( + 'What do you need help with?', + choices=[ + ('public-sector', 'I work in the public sector and need to send emails, text messages or letters'), + ('public', 'I’m a member of the public with a question for the government'), + ], + validators=[DataRequired()] + ) + + class Feedback(StripWhitespaceForm): name = StringField('Name (optional)') email_address = email_address(label='Email address', gov_user=False, required=True) diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index 9cf0cd6a5..0f386d2b6 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -7,7 +7,13 @@ from flask_login import current_user from app import convert_to_boolean, current_service, service_api_client from app.extensions import zendesk_client from app.main import main -from app.main.forms import Feedback, Problem, SupportType, Triage +from app.main.forms import ( + Feedback, + Problem, + SupportRedirect, + SupportType, + Triage, +) QUESTION_TICKET_TYPE = 'ask-question-give-feedback' PROBLEM_TICKET_TYPE = "report-problem" @@ -29,15 +35,35 @@ def get_prefilled_message(): @main.route('/support', methods=['GET', 'POST']) def support(): - form = SupportType() - if form.validate_on_submit(): - return redirect(url_for( - '.feedback', - ticket_type=form.support_type.data, - )) + + if current_user.is_authenticated: + form = SupportType() + if form.validate_on_submit(): + return redirect(url_for( + '.feedback', + ticket_type=form.support_type.data, + )) + else: + form = SupportRedirect() + if form.validate_on_submit(): + if form.who.data == 'public': + return redirect(url_for( + '.support_public' + )) + else: + return redirect(url_for( + '.feedback', + ticket_type=QUESTION_TICKET_TYPE, + )) + return render_template('views/support/index.html', form=form) +@main.route('/support/public') +def support_public(): + return render_template('views/support/public.html') + + @main.route('/support/triage', methods=['GET', 'POST']) def triage(): form = Triage() diff --git a/app/navigation.py b/app/navigation.py index 2342429a2..53f01483c 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -47,6 +47,7 @@ class HeaderNavigation(Navigation): 'bat_phone', 'feedback', 'support', + 'support_public', 'thanks', 'triage', }, @@ -619,6 +620,7 @@ class MainNavigation(Navigation): 'start_tour', 'styleguide', 'support', + 'support_public', 'suspend_service', 'template_history', 'terms', @@ -916,6 +918,7 @@ class CaseworkNavigation(Navigation): 'styleguide', 'submit_request_to_go_live', 'support', + 'support_public', 'suspend_service', 'template_history', 'template_usage', @@ -1209,6 +1212,7 @@ class OrgNavigation(Navigation): 'styleguide', 'submit_request_to_go_live', 'support', + 'support_public', 'suspend_service', 'template_history', 'template_usage', diff --git a/app/templates/views/support/index.html b/app/templates/views/support/index.html index fcf32bf7e..09dd8caba 100644 --- a/app/templates/views/support/index.html +++ b/app/templates/views/support/index.html @@ -9,19 +9,25 @@ {% block maincolumn_content %} +

    Support

    +

    We provide 24-hour online support for teams with a live service on GOV.UK Notify.

    + + {% call form_wrapper() %} + {% if current_user.is_authenticated %} + {{ radios(form.support_type) }} + {% else %} +

    + What do you need help with? +

    + {{ radios(form.who, hide_legend=True) }} + {% endif %} + {{ page_footer('Continue') }} + {% endcall %} + +

    You can also contact us on Slack.

    +
    - -

    Support

    -

    We provide 24-hour online support for teams with a live service on GOV.UK Notify.

    - - {% call form_wrapper(class="bottom-gutter-2") %} - {{ radios(form.support_type) }} - {{ page_footer('Continue') }} - {% endcall %} - -

    You can also contact us on Slack.

    -

    Office hours

    Our office hours are 9:30am to 5:30pm, Monday to Friday.

    When you report a problem in office hours, we’ll aim to read it within 30 minutes and reply within one working day.

    diff --git a/app/templates/views/support/public.html b/app/templates/views/support/public.html new file mode 100644 index 000000000..711a08ccb --- /dev/null +++ b/app/templates/views/support/public.html @@ -0,0 +1,48 @@ +{% extends "withoutnav_template.html" %} + +{% from "components/page-header.html" import page_header %} + +{% block per_page_title %} + The GOV.UK Notify team can’t give advice to members of the public +{% endblock %} + +{% block maincolumn_content %} + +
    +
    + + {{ page_header( + 'The GOV.UK Notify service is for people who work in the government', + back_link=url_for('.support') + ) }} + +

    + We can’t give advice to the public. We don’t have access to information about you held by government departments. +

    + +

    + There are other pages on GOV.UK where you can get help: +

    + +

    + Coronavirus (COVID-19) +

    +

    + What you need to do +

    +

    + Contact the government +

    +

    + Ask about benefits, driving, transport, tax, and more +

    +

    + Avoid and report internet scams and phishing +

    +

    + Advice on suspicious emails and text messages +

    +
    +
    + +{% endblock %} diff --git a/tests/app/main/views/test_feedback.py b/tests/app/main/views/test_feedback.py index 684fc2b12..0a2cd3a58 100644 --- a/tests/app/main/views/test_feedback.py +++ b/tests/app/main/views/test_feedback.py @@ -19,18 +19,48 @@ def no_redirect(): return lambda _external=True: None -@pytest.mark.parametrize('endpoint', [ - 'main.old_feedback', - 'main.support', -]) def test_get_support_index_page( - client, - endpoint, + client_request, ): - response = client.get(url_for('main.support'), follow_redirects=True) - assert response.status_code == 200 - page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser') - assert page.h1.string.strip() == 'Support' + page = client_request.get('.support') + assert page.select_one('form')['method'] == 'post' + assert 'action' not in page.select_one('form') + assert normalize_spaces(page.select_one('h1').text) == 'Support' + assert normalize_spaces( + page.select_one('form label[for=support_type-0]').text + ) == 'Report a problem' + assert page.select_one('form input#support_type-0')['value'] == 'report-problem' + assert normalize_spaces( + page.select_one('form label[for=support_type-1]').text + ) == 'Ask a question or give feedback' + assert page.select_one('form input#support_type-1')['value'] == 'ask-question-give-feedback' + assert normalize_spaces( + page.select_one('form button[type=submit]').text + ) == 'Continue' + + +def test_get_support_index_page_when_signed_out( + client_request, +): + client_request.logout() + page = client_request.get('.support') + assert page.select_one('form')['method'] == 'post' + assert 'action' not in page.select_one('form') + assert normalize_spaces( + page.select_one('form label[for=who-0]').text + ) == ( + 'I work in the public sector and need to send emails, text messages or letters' + ) + assert page.select_one('form input#who-0')['value'] == 'public-sector' + assert normalize_spaces( + page.select_one('form label[for=who-1]').text + ) == ( + 'I’m a member of the public with a question for the government' + ) + assert page.select_one('form input#who-1')['value'] == 'public' + assert normalize_spaces( + page.select_one('form button[type=submit]').text + ) == 'Continue' @freeze_time('2016-12-12 12:00:00.000000') @@ -38,34 +68,58 @@ def test_get_support_index_page( (PROBLEM_TICKET_TYPE, 'Report a problem'), (QUESTION_TICKET_TYPE, 'Ask a question or give feedback'), ]) -@pytest.mark.parametrize('logged_in, expected_form_field, expected_contact_details', [ - (True, type(None), 'We’ll reply to test@user.gov.uk'), - (False, element.Tag, None), -]) def test_choose_support_type( - client, - api_user_active, - mock_get_user, - mock_get_services, - logged_in, - expected_form_field, - expected_contact_details, + client_request, support_type, expected_h1 ): - if logged_in: - client.login(api_user_active) - response = client.post( - url_for('main.support'), - data={'support_type': support_type}, follow_redirects=True + page = client_request.post( + 'main.support', + _data={'support_type': support_type}, + _follow_redirects=True, ) - assert response.status_code == 200 - page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser') assert page.h1.string.strip() == expected_h1 - assert isinstance(page.find('input', {'name': 'name'}), expected_form_field) - assert isinstance(page.find('input', {'name': 'email_address'}), expected_form_field) - if expected_contact_details: - assert page.find('form').find('p').text.strip() == expected_contact_details + assert not page.select_one('input[name=name]') + assert not page.select_one('input[name=email_address]') + assert page.find('form').find('p').text.strip() == ( + 'We’ll reply to test@user.gov.uk' + ) + + +def test_get_support_as_someone_in_the_public_sector( + client_request, +): + client_request.logout() + page = client_request.post( + 'main.support', + _data={'who': 'public-sector'}, + _follow_redirects=True, + ) + assert normalize_spaces(page.select('h1')) == ( + 'Ask a question or give feedback' + ) + assert page.select_one('form textarea[name=feedback]') + assert page.select_one('form input[name=name]') + assert page.select_one('form input[name=email_address]') + assert page.select_one('form button[type=submit]') + + +def test_get_support_as_member_of_public( + client_request, +): + client_request.logout() + page = client_request.post( + 'main.support', + _data={'who': 'public'}, + _follow_redirects=True, + ) + assert normalize_spaces(page.select('h1')) == ( + 'The GOV.UK Notify service is for people who work in the government' + ) + assert len(page.select('h2 a')) == 3 + assert not page.select('form') + assert not page.select('input') + assert not page.select('form [type=submit]') @freeze_time('2016-12-12 12:00:00.000000') From 11cdf582564ac72685a098db936ed00343e64c75 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 24 Mar 2020 15:13:53 +0000 Subject: [PATCH 40/63] Use the same form for problems and questions --- app/__init__.py | 2 ++ app/main/forms.py | 6 +----- app/main/views/feedback.py | 15 ++++----------- app/url_converters.py | 5 +++++ 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 79c315225..759b82b11 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -92,6 +92,7 @@ from app.url_converters import ( LetterFileExtensionConverter, SimpleDateTypeConverter, TemplateTypeConverter, + TicketTypeConverter, ) from app.utils import format_thousands, get_logo_cdn_domain, id_safe @@ -227,6 +228,7 @@ def init_app(application): application.url_map.converters['uuid'].to_python = lambda self, value: value application.url_map.converters['template_type'] = TemplateTypeConverter + application.url_map.converters['ticket_type'] = TicketTypeConverter application.url_map.converters['letter_file_extension'] = LetterFileExtensionConverter application.url_map.converters['simple_date'] = SimpleDateTypeConverter diff --git a/app/main/forms.py b/app/main/forms.py index c0e9e1b24..cb2eabcb7 100644 --- a/app/main/forms.py +++ b/app/main/forms.py @@ -903,16 +903,12 @@ class SupportRedirect(StripWhitespaceForm): ) -class Feedback(StripWhitespaceForm): +class FeedbackOrProblem(StripWhitespaceForm): name = StringField('Name (optional)') email_address = email_address(label='Email address', gov_user=False, required=True) feedback = TextAreaField('Your message', validators=[DataRequired(message="Cannot be empty")]) -class Problem(Feedback): - email_address = email_address(label='Email address', gov_user=False) - - class Triage(StripWhitespaceForm): severe = RadioField( 'Is it an emergency?', diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index 0f386d2b6..e2b4316fb 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -1,15 +1,14 @@ from datetime import datetime import pytz -from flask import abort, redirect, render_template, request, session, url_for +from flask import redirect, render_template, request, session, url_for from flask_login import current_user from app import convert_to_boolean, current_service, service_api_client from app.extensions import zendesk_client from app.main import main from app.main.forms import ( - Feedback, - Problem, + FeedbackOrProblem, SupportRedirect, SupportType, Triage, @@ -79,15 +78,9 @@ def triage(): ) -@main.route('/support/', methods=['GET', 'POST']) +@main.route('/support/', methods=['GET', 'POST']) def feedback(ticket_type): - try: - form = { - QUESTION_TICKET_TYPE: Feedback, - PROBLEM_TICKET_TYPE: Problem, - }[ticket_type]() - except KeyError: - abort(404) + form = FeedbackOrProblem() if not form.feedback.data: form.feedback.data = session.pop('feedback_message', '') diff --git a/app/url_converters.py b/app/url_converters.py index 9dc649b5a..bee034727 100644 --- a/app/url_converters.py +++ b/app/url_converters.py @@ -6,6 +6,11 @@ class TemplateTypeConverter(BaseConverter): regex = '(?:email|sms|letter)' +class TicketTypeConverter(BaseConverter): + + regex = '(?:ask-question-give-feedback|report-problem)' + + class LetterFileExtensionConverter(BaseConverter): regex = '(?:pdf|png)' From 17a3ee920c9e9117b19c6931d2592d6883f56124 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 24 Mar 2020 15:22:00 +0000 Subject: [PATCH 41/63] Use the same template for problems and questions --- app/main/views/feedback.py | 6 +++--- .../views/support/report-problem.html | 20 +++++++++++-------- tests/app/main/views/test_feedback.py | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index e2b4316fb..f0eab1908 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -52,7 +52,7 @@ def support(): else: return redirect(url_for( '.feedback', - ticket_type=QUESTION_TICKET_TYPE, + ticket_type=PROBLEM_TICKET_TYPE, )) return render_template('views/support/index.html', form=form) @@ -144,9 +144,9 @@ def feedback(ticket_type): form.feedback.data = get_prefilled_message() return render_template( - 'views/support/{}.html'.format(ticket_type), + 'views/support/{}.html'.format(PROBLEM_TICKET_TYPE), form=form, - ticket_type=ticket_type, + is_problem=(ticket_type == PROBLEM_TICKET_TYPE), ) diff --git a/app/templates/views/support/report-problem.html b/app/templates/views/support/report-problem.html index e2e9b4660..342d5cd5d 100644 --- a/app/templates/views/support/report-problem.html +++ b/app/templates/views/support/report-problem.html @@ -5,24 +5,28 @@ {% from "components/page-header.html" import page_header %} {% from "components/form.html" import form_wrapper %} +{% set page_title = 'Report a problem' if is_problem else 'Ask a question or give feedback' %} + {% block per_page_title %} - Report a problem + {{ page_title }} {% endblock %} {% block maincolumn_content %} {{ page_header( - 'Report a problem', + page_title, back_link=url_for('.support') ) }}
    -
    -

    - Check our system status - page to see if there are any known issues with GOV.UK Notify. -

    -
    + {% if is_problem %} +
    +

    + Check our system status + page to see if there are any known issues with GOV.UK Notify. +

    +
    + {% endif %} {% call form_wrapper() %} {{ textbox(form.feedback, width='1-1', hint='', rows=10, autosize=True) }} {% if not current_user.is_authenticated %} diff --git a/tests/app/main/views/test_feedback.py b/tests/app/main/views/test_feedback.py index 0a2cd3a58..b7250be42 100644 --- a/tests/app/main/views/test_feedback.py +++ b/tests/app/main/views/test_feedback.py @@ -96,7 +96,7 @@ def test_get_support_as_someone_in_the_public_sector( _follow_redirects=True, ) assert normalize_spaces(page.select('h1')) == ( - 'Ask a question or give feedback' + 'Report a problem' ) assert page.select_one('form textarea[name=feedback]') assert page.select_one('form input[name=name]') From 55e377baf629877a33e426f95caeaac73f372b47 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 24 Mar 2020 15:30:43 +0000 Subject: [PATCH 42/63] Rename and delete form templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `report-a-problem’ becomes `form` because it’s not just being used for problems. 'ask-a-question' gets deleted because it’s not being used. --- app/main/views/feedback.py | 2 +- .../support/ask-question-give-feedback.html | 32 ------------------- .../{report-problem.html => form.html} | 0 3 files changed, 1 insertion(+), 33 deletions(-) delete mode 100644 app/templates/views/support/ask-question-give-feedback.html rename app/templates/views/support/{report-problem.html => form.html} (100%) diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index f0eab1908..843d3f802 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -144,7 +144,7 @@ def feedback(ticket_type): form.feedback.data = get_prefilled_message() return render_template( - 'views/support/{}.html'.format(PROBLEM_TICKET_TYPE), + 'views/support/form.html', form=form, is_problem=(ticket_type == PROBLEM_TICKET_TYPE), ) diff --git a/app/templates/views/support/ask-question-give-feedback.html b/app/templates/views/support/ask-question-give-feedback.html deleted file mode 100644 index 77e9ec8f9..000000000 --- a/app/templates/views/support/ask-question-give-feedback.html +++ /dev/null @@ -1,32 +0,0 @@ -{% extends "withoutnav_template.html" %} -{% from "components/textbox.html" import textbox %} -{% from "components/page-footer.html" import sticky_page_footer %} -{% from "components/page-header.html" import page_header %} -{% from "components/form.html" import form_wrapper %} - -{% block per_page_title %} - Ask a question or give feedback -{% endblock %} - -{% block maincolumn_content %} - - {{ page_header( - 'Ask a question or give feedback', - back_link=url_for('.support') - ) }} -
    -
    - {% call form_wrapper() %} - {{ textbox(form.feedback, width='1-1', hint='', rows=10, autosize=True) }} - {% if not current_user.is_authenticated %} - {{ textbox(form.name, width='1-1') }} - {{ textbox(form.email_address, width='1-1') }} - {% else %} -

    We’ll reply to {{ current_user.email_address }}

    - {% endif %} - {{ sticky_page_footer('Send') }} - {% endcall %} -
    -
    - -{% endblock %} diff --git a/app/templates/views/support/report-problem.html b/app/templates/views/support/form.html similarity index 100% rename from app/templates/views/support/report-problem.html rename to app/templates/views/support/form.html From 30a341847399cc0bba1dcf8aaf4027fa2b46dd43 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 24 Mar 2020 15:36:39 +0000 Subject: [PATCH 43/63] Refactor so constants are used everywhere --- app/main/forms.py | 5 +++-- app/main/views/feedback.py | 4 +--- app/main/views/index.py | 2 +- app/models/feedback.py | 2 ++ app/url_converters.py | 4 +++- tests/app/main/views/test_feedback.py | 8 ++------ 6 files changed, 12 insertions(+), 13 deletions(-) create mode 100644 app/models/feedback.py diff --git a/app/main/forms.py b/app/main/forms.py index cb2eabcb7..85639dfa7 100644 --- a/app/main/forms.py +++ b/app/main/forms.py @@ -48,6 +48,7 @@ from app.main.validators import ( ValidEmail, ValidGovEmail, ) +from app.models.feedback import PROBLEM_TICKET_TYPE, QUESTION_TICKET_TYPE from app.models.organisation import Organisation from app.models.roles_and_permissions import permissions, roles from app.utils import guess_name_from_email_address @@ -885,8 +886,8 @@ class SupportType(StripWhitespaceForm): support_type = RadioField( 'How can we help you?', choices=[ - ('report-problem', 'Report a problem'), - ('ask-question-give-feedback', 'Ask a question or give feedback'), + (PROBLEM_TICKET_TYPE, 'Report a problem'), + (QUESTION_TICKET_TYPE, 'Ask a question or give feedback'), ], validators=[DataRequired()] ) diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index 843d3f802..f28fefaee 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -13,9 +13,7 @@ from app.main.forms import ( SupportType, Triage, ) - -QUESTION_TICKET_TYPE = 'ask-question-give-feedback' -PROBLEM_TICKET_TYPE = "report-problem" +from app.models.feedback import PROBLEM_TICKET_TYPE def get_prefilled_message(): diff --git a/app/main/views/index.py b/app/main/views/index.py index 13effe361..86c244d6f 100644 --- a/app/main/views/index.py +++ b/app/main/views/index.py @@ -15,12 +15,12 @@ from notifications_utils.template import HTMLEmailTemplate, LetterImageTemplate from app import email_branding_client, letter_branding_client, status_api_client from app.main import main from app.main.forms import FieldWithNoneOption, SearchByNameForm -from app.main.views.feedback import QUESTION_TICKET_TYPE from app.main.views.sub_navigation_dictionaries import ( features_nav, pricing_nav, using_notify_nav, ) +from app.models.feedback import QUESTION_TICKET_TYPE from app.utils import get_logo_cdn_domain diff --git a/app/models/feedback.py b/app/models/feedback.py new file mode 100644 index 000000000..fffbf0a29 --- /dev/null +++ b/app/models/feedback.py @@ -0,0 +1,2 @@ +QUESTION_TICKET_TYPE = 'ask-question-give-feedback' +PROBLEM_TICKET_TYPE = 'report-problem' diff --git a/app/url_converters.py b/app/url_converters.py index bee034727..aa2b5dabe 100644 --- a/app/url_converters.py +++ b/app/url_converters.py @@ -1,5 +1,7 @@ from werkzeug.routing import BaseConverter +from app.models.feedback import PROBLEM_TICKET_TYPE, QUESTION_TICKET_TYPE + class TemplateTypeConverter(BaseConverter): @@ -8,7 +10,7 @@ class TemplateTypeConverter(BaseConverter): class TicketTypeConverter(BaseConverter): - regex = '(?:ask-question-give-feedback|report-problem)' + regex = f'(?:{PROBLEM_TICKET_TYPE}|{QUESTION_TICKET_TYPE})' class LetterFileExtensionConverter(BaseConverter): diff --git a/tests/app/main/views/test_feedback.py b/tests/app/main/views/test_feedback.py index b7250be42..cfe649ab7 100644 --- a/tests/app/main/views/test_feedback.py +++ b/tests/app/main/views/test_feedback.py @@ -6,12 +6,8 @@ from bs4 import BeautifulSoup, element from flask import url_for from freezegun import freeze_time -from app.main.views.feedback import ( - PROBLEM_TICKET_TYPE, - QUESTION_TICKET_TYPE, - has_live_services, - in_business_hours, -) +from app.main.views.feedback import has_live_services, in_business_hours +from app.models.feedback import PROBLEM_TICKET_TYPE, QUESTION_TICKET_TYPE from tests.conftest import normalize_spaces From 697803a10c1c4edc722a91d31677d662e209c46d Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 24 Mar 2020 15:45:13 +0000 Subject: [PATCH 44/63] =?UTF-8?q?Add=20a=20=E2=80=98general=E2=80=99=20tic?= =?UTF-8?q?ket=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is for tickets coming from non-logged-in users. It’s effectively the same as reporting a problem, but doesn’t have the banner about the status page (because we can’t tell if they’re actually reporting a problem now we’re not asking). It also gives a more generic page title. --- app/main/views/feedback.py | 34 +++++-- app/models/feedback.py | 1 + app/templates/views/support/form.html | 4 +- app/templates/views/support/triage.html | 4 +- app/url_converters.py | 8 +- tests/app/main/views/test_feedback.py | 120 ++++++++++++++++++++---- 6 files changed, 136 insertions(+), 35 deletions(-) diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index f28fefaee..4b09e40f3 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -13,7 +13,11 @@ from app.main.forms import ( SupportType, Triage, ) -from app.models.feedback import PROBLEM_TICKET_TYPE +from app.models.feedback import ( + GENERAL_TICKET_TYPE, + PROBLEM_TICKET_TYPE, + QUESTION_TICKET_TYPE, +) def get_prefilled_message(): @@ -50,7 +54,7 @@ def support(): else: return redirect(url_for( '.feedback', - ticket_type=PROBLEM_TICKET_TYPE, + ticket_type=GENERAL_TICKET_TYPE, )) return render_template('views/support/index.html', form=form) @@ -62,17 +66,22 @@ def support_public(): @main.route('/support/triage', methods=['GET', 'POST']) -def triage(): +@main.route('/support/triage/', methods=['GET', 'POST']) +def triage(ticket_type=PROBLEM_TICKET_TYPE): form = Triage() if form.validate_on_submit(): return redirect(url_for( '.feedback', - ticket_type=PROBLEM_TICKET_TYPE, + ticket_type=ticket_type, severe=form.severe.data )) return render_template( 'views/support/triage.html', - form=form + form=form, + page_title={ + PROBLEM_TICKET_TYPE: 'Report a problem', + GENERAL_TICKET_TYPE: 'Contact GOV.UK Notify support', + }.get(ticket_type) ) @@ -89,14 +98,14 @@ def feedback(ticket_type): severe = None out_of_hours_emergency = all(( - ticket_type == PROBLEM_TICKET_TYPE, + ticket_type != QUESTION_TICKET_TYPE, not in_business_hours(), severe, )) if needs_triage(ticket_type, severe): session['feedback_message'] = form.feedback.data - return redirect(url_for('.triage')) + return redirect(url_for('.triage', ticket_type=ticket_type)) if needs_escalation(ticket_type, severe): return redirect(url_for('.bat_phone')) @@ -144,7 +153,12 @@ def feedback(ticket_type): return render_template( 'views/support/form.html', form=form, - is_problem=(ticket_type == PROBLEM_TICKET_TYPE), + show_status_page_banner=(ticket_type == PROBLEM_TICKET_TYPE), + page_title={ + GENERAL_TICKET_TYPE: 'Contact GOV.UK Notify support', + PROBLEM_TICKET_TYPE: 'Report a problem', + QUESTION_TICKET_TYPE: 'Ask a question or give feedback', + }.get(ticket_type), ) @@ -254,7 +268,7 @@ def has_live_services(user_id): def needs_triage(ticket_type, severe): return all(( - ticket_type == PROBLEM_TICKET_TYPE, + ticket_type != QUESTION_TICKET_TYPE, severe is None, ( not current_user.is_authenticated or has_live_services(current_user.id) @@ -265,7 +279,7 @@ def needs_triage(ticket_type, severe): def needs_escalation(ticket_type, severe): return all(( - ticket_type == PROBLEM_TICKET_TYPE, + ticket_type != QUESTION_TICKET_TYPE, severe, not current_user.is_authenticated, not in_business_hours(), diff --git a/app/models/feedback.py b/app/models/feedback.py index fffbf0a29..345be5e6a 100644 --- a/app/models/feedback.py +++ b/app/models/feedback.py @@ -1,2 +1,3 @@ QUESTION_TICKET_TYPE = 'ask-question-give-feedback' PROBLEM_TICKET_TYPE = 'report-problem' +GENERAL_TICKET_TYPE = 'general' diff --git a/app/templates/views/support/form.html b/app/templates/views/support/form.html index 342d5cd5d..62a67e403 100644 --- a/app/templates/views/support/form.html +++ b/app/templates/views/support/form.html @@ -5,8 +5,6 @@ {% from "components/page-header.html" import page_header %} {% from "components/form.html" import form_wrapper %} -{% set page_title = 'Report a problem' if is_problem else 'Ask a question or give feedback' %} - {% block per_page_title %} {{ page_title }} {% endblock %} @@ -19,7 +17,7 @@ ) }}
    - {% if is_problem %} + {% if show_status_page_banner %}

    Check our system status diff --git a/app/templates/views/support/triage.html b/app/templates/views/support/triage.html index 8124ce8e4..9b32dd13c 100644 --- a/app/templates/views/support/triage.html +++ b/app/templates/views/support/triage.html @@ -5,7 +5,7 @@ {% from "components/form.html" import form_wrapper %} {% block per_page_title %} - Feedback + {{ page_title }} {% endblock %} {% block maincolumn_content %} @@ -13,7 +13,7 @@

    {{ page_header( - 'Report a problem', + page_title, back_link=url_for('.support') ) }} {% call form_wrapper() %} diff --git a/app/url_converters.py b/app/url_converters.py index aa2b5dabe..847543b96 100644 --- a/app/url_converters.py +++ b/app/url_converters.py @@ -1,6 +1,10 @@ from werkzeug.routing import BaseConverter -from app.models.feedback import PROBLEM_TICKET_TYPE, QUESTION_TICKET_TYPE +from app.models.feedback import ( + GENERAL_TICKET_TYPE, + PROBLEM_TICKET_TYPE, + QUESTION_TICKET_TYPE, +) class TemplateTypeConverter(BaseConverter): @@ -10,7 +14,7 @@ class TemplateTypeConverter(BaseConverter): class TicketTypeConverter(BaseConverter): - regex = f'(?:{PROBLEM_TICKET_TYPE}|{QUESTION_TICKET_TYPE})' + regex = f'(?:{PROBLEM_TICKET_TYPE}|{QUESTION_TICKET_TYPE}|{GENERAL_TICKET_TYPE})' class LetterFileExtensionConverter(BaseConverter): diff --git a/tests/app/main/views/test_feedback.py b/tests/app/main/views/test_feedback.py index cfe649ab7..138bfa536 100644 --- a/tests/app/main/views/test_feedback.py +++ b/tests/app/main/views/test_feedback.py @@ -7,7 +7,11 @@ from flask import url_for from freezegun import freeze_time from app.main.views.feedback import has_live_services, in_business_hours -from app.models.feedback import PROBLEM_TICKET_TYPE, QUESTION_TICKET_TYPE +from app.models.feedback import ( + GENERAL_TICKET_TYPE, + PROBLEM_TICKET_TYPE, + QUESTION_TICKET_TYPE, +) from tests.conftest import normalize_spaces @@ -82,6 +86,7 @@ def test_choose_support_type( ) +@freeze_time('2016-12-12 12:00:00.000000') def test_get_support_as_someone_in_the_public_sector( client_request, ): @@ -92,7 +97,7 @@ def test_get_support_as_someone_in_the_public_sector( _follow_redirects=True, ) assert normalize_spaces(page.select('h1')) == ( - 'Report a problem' + 'Contact GOV.UK Notify support' ) assert page.select_one('form textarea[name=feedback]') assert page.select_one('form input[name=name]') @@ -344,8 +349,8 @@ def test_urgency( ids, params = zip(*[ ('non-logged in users always have to triage', ( - PROBLEM_TICKET_TYPE, False, False, True, - 302, partial(url_for, 'main.triage') + GENERAL_TICKET_TYPE, False, False, True, + 302, partial(url_for, 'main.triage', ticket_type=GENERAL_TICKET_TYPE) )), ('trial services are never high priority', ( PROBLEM_TICKET_TYPE, False, True, False, @@ -361,7 +366,7 @@ ids, params = zip(*[ )), ('should triage out of hours', ( PROBLEM_TICKET_TYPE, False, True, True, - 302, partial(url_for, 'main.triage') + 302, partial(url_for, 'main.triage', ticket_type=PROBLEM_TICKET_TYPE) )) ]) @@ -395,6 +400,21 @@ def test_redirects_to_triage( assert response.location == expected_redirect(_external=True) +@pytest.mark.parametrize('ticket_type, expected_h1', ( + (PROBLEM_TICKET_TYPE, 'Report a problem'), + (GENERAL_TICKET_TYPE, 'Contact GOV.UK Notify support'), +)) +def test_options_on_triage_page( + client_request, + ticket_type, + expected_h1, +): + page = client_request.get('main.triage', ticket_type=ticket_type) + assert normalize_spaces(page.select_one('h1').text) == expected_h1 + assert page.select('form input[type=radio]')[0]['value'] == 'yes' + assert page.select('form input[type=radio]')[1]['value'] == 'no' + + def test_doesnt_lose_message_if_post_across_closing( client_request, mocker, @@ -408,7 +428,7 @@ def test_doesnt_lose_message_if_post_across_closing( ticket_type=PROBLEM_TICKET_TYPE, _data={'feedback': 'foo'}, _expected_status=302, - _expected_redirect=url_for('.triage', _external=True), + _expected_redirect=url_for('.triage', ticket_type=PROBLEM_TICKET_TYPE, _external=True), ) with client_request.session_transaction() as session: assert session['feedback_message'] == 'foo' @@ -458,18 +478,31 @@ def test_in_business_hours(when, is_in_business_hours): assert in_business_hours() == is_in_business_hours +@pytest.mark.parametrize('ticket_type', ( + GENERAL_TICKET_TYPE, + PROBLEM_TICKET_TYPE, +)) @pytest.mark.parametrize('choice, expected_redirect_param', [ ('yes', 'yes'), ('no', 'no'), ]) -def test_triage_redirects_to_correct_url(client, choice, expected_redirect_param): - response = client.post(url_for('main.triage'), data={'severe': choice}) - assert response.status_code == 302 - assert response.location == url_for( - 'main.feedback', - ticket_type=PROBLEM_TICKET_TYPE, - severe=expected_redirect_param, - _external=True, +def test_triage_redirects_to_correct_url( + client_request, + ticket_type, + choice, + expected_redirect_param, +): + client_request.post( + 'main.triage', + ticket_type=ticket_type, + _data={'severe': choice}, + _expected_status=302, + _expected_redirect=url_for( + 'main.feedback', + ticket_type=ticket_type, + severe=expected_redirect_param, + _external=True, + ), ) @@ -499,15 +532,15 @@ def test_triage_redirects_to_correct_url(client, choice, expected_redirect_param # Treat empty query param as mangled URL – ask question again ( False, '', - 302, partial(url_for, 'main.triage'), - 302, partial(url_for, 'main.triage'), + 302, partial(url_for, 'main.triage', ticket_type=PROBLEM_TICKET_TYPE), + 302, partial(url_for, 'main.triage', ticket_type=PROBLEM_TICKET_TYPE), ), # User hasn’t answered the triage question ( False, None, - 302, partial(url_for, 'main.triage'), - 302, partial(url_for, 'main.triage'), + 302, partial(url_for, 'main.triage', ticket_type=PROBLEM_TICKET_TYPE), + 302, partial(url_for, 'main.triage', ticket_type=PROBLEM_TICKET_TYPE), ), # Escalation is needed for non-logged-in users @@ -548,6 +581,57 @@ def test_should_be_shown_the_bat_email( assert logged_in_response.location == expected_redirect_when_logged_in(_external=True) +@pytest.mark.parametrize( + ( + 'severe,' + 'expected_status_code, expected_redirect,' + 'expected_status_code_when_logged_in, expected_redirect_when_logged_in' + ), + [ + # User hasn’t answered the triage question + ( + None, + 302, partial(url_for, 'main.triage', ticket_type=GENERAL_TICKET_TYPE), + 302, partial(url_for, 'main.triage', ticket_type=GENERAL_TICKET_TYPE), + ), + + # Escalation is needed for non-logged-in users + ( + 'yes', + 302, partial(url_for, 'main.bat_phone'), + 200, no_redirect(), + ), + ] +) +def test_should_be_shown_the_bat_email_for_general_questions( + client, + active_user_with_permissions, + mocker, + service_one, + mock_get_services, + severe, + expected_status_code, + expected_redirect, + expected_status_code_when_logged_in, + expected_redirect_when_logged_in, +): + + mocker.patch('app.main.views.feedback.in_business_hours', return_value=False) + + feedback_page = url_for('main.feedback', ticket_type=GENERAL_TICKET_TYPE, severe=severe) + + response = client.get(feedback_page) + + assert response.status_code == expected_status_code + assert response.location == expected_redirect(_external=True) + + # logged in users should never be redirected to the bat email page + client.login(active_user_with_permissions, mocker, service_one) + logged_in_response = client.get(feedback_page) + assert logged_in_response.status_code == expected_status_code_when_logged_in + assert logged_in_response.location == expected_redirect_when_logged_in(_external=True) + + def test_bat_email_page( client, active_user_with_permissions, From 8e055a43752786d3b11725b37074ded8b1acefa1 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 24 Mar 2020 15:51:47 +0000 Subject: [PATCH 45/63] Remove prefilled message thing --- app/main/views/feedback.py | 17 ----------- tests/app/main/views/test_feedback.py | 44 --------------------------- 2 files changed, 61 deletions(-) diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index 4b09e40f3..5ed99554b 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -20,20 +20,6 @@ from app.models.feedback import ( ) -def get_prefilled_message(): - return { - 'agreement': ( - 'Please can you tell me if there’s an agreement in place ' - 'between GOV.UK Notify and my organisation?' - ), - 'letter-branding': ( - 'I would like my own logo on my letter templates.' - ), - }.get( - request.args.get('body'), '' - ) - - @main.route('/support', methods=['GET', 'POST']) def support(): @@ -147,9 +133,6 @@ def feedback(ticket_type): ), )) - if not form.feedback.data: - form.feedback.data = get_prefilled_message() - return render_template( 'views/support/form.html', form=form, diff --git a/tests/app/main/views/test_feedback.py b/tests/app/main/views/test_feedback.py index 138bfa536..47792b35a 100644 --- a/tests/app/main/views/test_feedback.py +++ b/tests/app/main/views/test_feedback.py @@ -134,50 +134,6 @@ def test_get_feedback_page(client, ticket_type, expected_status_code): assert response.status_code == expected_status_code -@pytest.mark.parametrize('prefilled_body, expected_textarea', [ - ( - 'agreement', - ( - 'Please can you tell me if there’s an agreement in place ' - 'between GOV.UK Notify and my organisation?' - ) - ), - ( - 'foo', - '' - ), -]) -@freeze_time('2016-12-12 12:00:00.000000') -def test_get_feedback_page_with_prefilled_body( - client_request, - mocker, - fake_uuid, - prefilled_body, - expected_textarea, - active_user_with_permissions, -): - active_user_with_permissions['email_address'] = 'test@marinemanagement.org.uk' - mocker.patch('app.user_api_client.get_user', return_value=active_user_with_permissions) - mock_post = mocker.patch('app.main.views.feedback.zendesk_client.create_ticket') - page = client_request.get( - 'main.feedback', - ticket_type=QUESTION_TICKET_TYPE, - body=prefilled_body, - ) - assert page.select_one('textarea').text == ( - expected_textarea - ) - client_request.post( - 'main.feedback', - ticket_type=QUESTION_TICKET_TYPE, - body='agreement', - _data={'feedback': 'blah', 'name': 'Example', 'email_address': 'test@example.com'} - ) - message = mock_post.call_args[1]['message'] - assert message.startswith('blah') - assert 'Please send' not in message - - @freeze_time('2016-12-12 12:00:00.000000') @pytest.mark.parametrize('ticket_type', [PROBLEM_TICKET_TYPE, QUESTION_TICKET_TYPE]) def test_passed_non_logged_in_user_details_through_flow(client, mocker, ticket_type): From 34a5ddaf2cae5db0a38a5920751eefbb73f49175 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 24 Mar 2020 15:54:44 +0000 Subject: [PATCH 46/63] Assume email always provided MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There’s no way for a user to submit the form without it now. --- app/main/views/feedback.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index 5ed99554b..0f1189579 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -111,10 +111,9 @@ def feedback(ticket_type): else: service_string = '' - feedback_msg = '{}\n{}{}'.format( + feedback_msg = '{}\n{}'.format( form.feedback.data, service_string, - '' if user_email else '{} (no email address supplied)'.format(form.name.data) ) zendesk_client.create_ticket( From 8335de14321c2adc6b30cd05bbbdd5ec6a312a50 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 25 Mar 2020 11:56:47 +0000 Subject: [PATCH 47/63] Use correct back link pattern on escalation page --- app/templates/views/support/bat-phone.html | 11 +++++------ tests/app/main/views/test_feedback.py | 8 +++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/templates/views/support/bat-phone.html b/app/templates/views/support/bat-phone.html index 596a3302f..41408a6d6 100644 --- a/app/templates/views/support/bat-phone.html +++ b/app/templates/views/support/bat-phone.html @@ -1,6 +1,7 @@ {% extends "withoutnav_template.html" %} {% from "components/textbox.html" import textbox %} {% from "components/page-footer.html" import page_footer %} +{% from "components/page-header.html" import page_header %} {% block per_page_title %} Out of hours emergencies @@ -8,9 +9,10 @@ {% block maincolumn_content %} -

    - Out of hours emergencies -

    + {{ page_header( + 'Out of hours emergencies', + url_for('.support') + )}}

    @@ -36,9 +38,6 @@ Fill in this form and we’ll get back to you by the next working day.

    -

    - Back to support -

    diff --git a/tests/app/main/views/test_feedback.py b/tests/app/main/views/test_feedback.py index 47792b35a..276e25619 100644 --- a/tests/app/main/views/test_feedback.py +++ b/tests/app/main/views/test_feedback.py @@ -600,9 +600,11 @@ def test_bat_email_page( assert response.status_code == 200 page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser') - assert page.select('main a')[1].text == 'Fill in this form' - assert page.select('main a')[1]['href'] == url_for('main.feedback', ticket_type=PROBLEM_TICKET_TYPE, severe='no') - next_page_response = client.get(page.select('main a')[1]['href']) + assert page.select('main a')[0].text == 'Back' + assert page.select('main a')[0]['href'] == url_for('main.support') + assert page.select('main a')[2].text == 'Fill in this form' + assert page.select('main a')[2]['href'] == url_for('main.feedback', ticket_type=PROBLEM_TICKET_TYPE, severe='no') + next_page_response = client.get(page.select('main a')[2]['href']) next_page = BeautifulSoup(next_page_response.data.decode('utf-8'), 'html.parser') assert next_page.h1.text.strip() == 'Report a problem' From 2ca32019e4b8e6adb9c16ccb8b19f41fc4d356ab Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 25 Mar 2020 12:17:39 +0000 Subject: [PATCH 48/63] =?UTF-8?q?Go=20back=20to=20triage=20page=20if=20you?= =?UTF-8?q?=E2=80=99ve=20come=20from=20there?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/main/views/feedback.py | 5 +++++ app/templates/views/support/form.html | 2 +- tests/app/main/views/test_feedback.py | 32 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/app/main/views/feedback.py b/app/main/views/feedback.py index 0f1189579..00896c1ce 100644 --- a/app/main/views/feedback.py +++ b/app/main/views/feedback.py @@ -135,6 +135,11 @@ def feedback(ticket_type): return render_template( 'views/support/form.html', form=form, + back_link=( + url_for('.support') + if severe is None else + url_for('.triage', ticket_type=ticket_type) + ), show_status_page_banner=(ticket_type == PROBLEM_TICKET_TYPE), page_title={ GENERAL_TICKET_TYPE: 'Contact GOV.UK Notify support', diff --git a/app/templates/views/support/form.html b/app/templates/views/support/form.html index 62a67e403..bf5e2c36a 100644 --- a/app/templates/views/support/form.html +++ b/app/templates/views/support/form.html @@ -13,7 +13,7 @@ {{ page_header( page_title, - back_link=url_for('.support') + back_link=back_link ) }}
    diff --git a/tests/app/main/views/test_feedback.py b/tests/app/main/views/test_feedback.py index 276e25619..736406859 100644 --- a/tests/app/main/views/test_feedback.py +++ b/tests/app/main/views/test_feedback.py @@ -462,6 +462,38 @@ def test_triage_redirects_to_correct_url( ) +@pytest.mark.parametrize('extra_args, expected_back_link', [ + ( + {'severe': 'yes'}, + partial(url_for, 'main.triage', ticket_type=PROBLEM_TICKET_TYPE) + ), + ( + {'severe': 'no'}, + partial(url_for, 'main.triage', ticket_type=PROBLEM_TICKET_TYPE) + ), + ( + {'severe': 'foo'}, # hacking the URL + partial(url_for, 'main.support') + ), + ( + {}, + partial(url_for, 'main.support') + ), +]) +def test_back_link_from_form( + client_request, + extra_args, + expected_back_link, +): + page = client_request.get( + 'main.feedback', + ticket_type=PROBLEM_TICKET_TYPE, + **extra_args + ) + assert page.select_one('.govuk-back-link')['href'] == expected_back_link() + assert normalize_spaces(page.select_one('h1').text) == 'Report a problem' + + @pytest.mark.parametrize( ( 'is_in_business_hours, severe,' From 7d57a4d9b86e5bf93422ca2d93b12d2bbf02055c Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 2 Mar 2020 13:48:32 +0000 Subject: [PATCH 49/63] Redirect the old jobs URL --- app/main/views/jobs.py | 24 +----- tests/app/main/views/test_jobs.py | 134 +++--------------------------- 2 files changed, 15 insertions(+), 143 deletions(-) diff --git a/app/main/views/jobs.py b/app/main/views/jobs.py index 375bc3523..25bf8a884 100644 --- a/app/main/views/jobs.py +++ b/app/main/views/jobs.py @@ -42,26 +42,10 @@ from app.utils import ( @main.route("/services//jobs") @user_has_permissions() def view_jobs(service_id): - jobs = current_service.get_page_of_jobs(page=request.args.get('page')) - - prev_page = None - if jobs.prev_page: - prev_page = generate_previous_dict('main.view_jobs', service_id, jobs.current_page) - next_page = None - if jobs.next_page: - next_page = generate_next_dict('main.view_jobs', service_id, jobs.current_page) - - return render_template( - 'views/jobs/jobs.html', - jobs=jobs, - prev_page=prev_page, - next_page=next_page, - show_scheduled_jobs=( - jobs.current_page == 1 - and not current_user.has_permissions('view_activity') - and current_service.scheduled_jobs - ), - ) + return redirect(url_for( + 'main.uploads', + service_id=current_service.id, + )) @main.route("/services//jobs/") diff --git a/tests/app/main/views/test_jobs.py b/tests/app/main/views/test_jobs.py index c4ea99aab..8a81153b4 100644 --- a/tests/app/main/views/test_jobs.py +++ b/tests/app/main/views/test_jobs.py @@ -18,131 +18,19 @@ from tests.conftest import ( ) -@pytest.mark.parametrize('user, expected_rows', [ - (create_active_user_with_permissions(), ( - ( - 'File Status' - ), - ( - 'export 1/1/2016.xls ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - ( - 'all email addresses.xlsx ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - ( - 'applicants.ods ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - ( - 'thisisatest.csv ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - )), - (create_active_caseworking_user(), ( - ( - 'File Messages to be sent' - ), - ( - 'even_later.csv ' - 'Sending 1 January 2016 at 11:09pm 1' - ), - ( - 'send_me_later.csv ' - 'Sending 1 January 2016 at 11:09am 1' - ), - ( - 'File Status' - ), - ( - 'export 1/1/2016.xls ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - ( - 'all email addresses.xlsx ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - ( - 'applicants.ods ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - ( - 'thisisatest.csv ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - )), -]) -@freeze_time("2012-12-12 12:12") -def test_jobs_page_shows_scheduled_jobs_if_user_doesnt_have_dashboard( +def test_old_jobs_hub_redirects( client_request, - service_one, - active_user_with_permissions, - mock_get_jobs, - user, - expected_rows, ): - client_request.login(user) - page = client_request.get('main.view_jobs', service_id=service_one['id']) - - for index, row in enumerate(expected_rows): - assert normalize_spaces(page.select('tr')[index].text) == row - - -@pytest.mark.parametrize('user', [ - create_active_user_with_permissions(), - create_active_caseworking_user(), -]) -def test_get_jobs_shows_page_links( - client_request, - active_user_with_permissions, - mock_get_jobs, - user, -): - client_request.login(user) - page = client_request.get('main.view_jobs', service_id=SERVICE_ONE_ID) - - assert 'Next page' in page.find('li', {'class': 'next-page'}).text - assert 'Previous page' in page.find('li', {'class': 'previous-page'}).text - - -@pytest.mark.parametrize('user', [ - create_active_user_with_permissions(), - create_active_caseworking_user(), -]) -@freeze_time("2012-12-12 12:12") -def test_jobs_page_doesnt_show_scheduled_on_page_2( - client_request, - service_one, - active_user_with_permissions, - mock_get_jobs, - user, -): - client_request.login(user) - page = client_request.get('main.view_jobs', service_id=service_one['id'], page=2) - - for index, row in enumerate(( - ( - 'File Status' - ), - ( - 'export 1/1/2016.xls ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - ( - 'all email addresses.xlsx ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - ( - 'applicants.ods ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - ( - 'thisisatest.csv ' - 'Sent today at 12:12pm 1 sending 0 delivered 0 failed' - ), - )): - assert normalize_spaces(page.select('tr')[index].text) == row + client_request.get( + 'main.view_jobs', + service_id=SERVICE_ONE_ID, + _expected_status=302, + _expected_redirect=url_for( + 'main.uploads', + service_id=SERVICE_ONE_ID, + _external=True, + ) + ) @pytest.mark.parametrize('user', [ From 481c9dba178bc51ad9fb3d847f9118bcd735816b Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 2 Mar 2020 14:07:25 +0000 Subject: [PATCH 50/63] Remove unused model property --- app/models/service.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/models/service.py b/app/models/service.py index 041ef5e5a..20c841c33 100644 --- a/app/models/service.py +++ b/app/models/service.py @@ -498,10 +498,6 @@ class Service(JSONModel): key=lambda folder: folder['name'].lower(), ) - @property - def can_upload_letters(self): - return self.has_permission('letter') and self.has_permission('upload_letters') - @cached_property def all_template_folder_ids(self): return {folder['id'] for folder in self.all_template_folders} From d83e7a8e9f277c7b6d78b9ed6ad2052302bed353 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 2 Mar 2020 14:26:04 +0000 Subject: [PATCH 51/63] Tidy up the jobs template --- app/templates/views/jobs/jobs.html | 20 ++++++++------------ tests/app/main/views/test_uploads.py | 4 ++-- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/app/templates/views/jobs/jobs.html b/app/templates/views/jobs/jobs.html index 0b88d0c34..4f4533dad 100644 --- a/app/templates/views/jobs/jobs.html +++ b/app/templates/views/jobs/jobs.html @@ -9,26 +9,22 @@ {% block maincolumn_content %}

    Uploads

    - {% if show_scheduled_jobs %} - {% with hide_heading = True %} - {% include 'views/jobs/_scheduled.html' %} - {% endwith %} - {% endif %} {% if jobs %} {% include 'views/dashboard/_jobs.html' %} - {% endif %} - {% if not jobs and not show_scheduled_jobs %} + {% else %}

    You have not uploaded any files recently.

    - {% if current_service.has_permission('upload_letters') %} + {% if current_user.has_permissions('send_messages') %} + {% if current_service.has_permission('letter') %} +

    + Upload a letter and Notify will print, pack and post it for you. +

    + {% endif %}

    - Upload a letter and Notify will print, pack and post it for you. + To upload a list of contact details, first choose a template.

    {% endif %} -

    - To upload a list of contact details, first choose a template. -

    {% endif %} {{ previous_next_navigation(prev_page, next_page) }} {% if current_user.has_permissions('send_messages') %} diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 1491b45d6..2c63839e8 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -65,11 +65,11 @@ def test_all_users_have_upload_contact_list( @pytest.mark.parametrize('extra_permissions, expected_empty_message', ( - (['letter'], ( + ([], ( 'You have not uploaded any files recently. ' 'To upload a list of contact details, first choose a template.' )), - (['letter', 'upload_letters'], ( + (['letter'], ( 'You have not uploaded any files recently. ' 'Upload a letter and Notify will print, pack and post it for you. ' 'To upload a list of contact details, first choose a template.' From 38b4b4d994d39a5b13c85b307522474ca34d50a6 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 18 Mar 2020 13:45:51 +0000 Subject: [PATCH 52/63] Fix misaligned upload button when no letter permission We should add spacing to the right, so if the first button is missing the second one still sits flush-left. --- app/templates/views/jobs/jobs.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/templates/views/jobs/jobs.html b/app/templates/views/jobs/jobs.html index 4f4533dad..27b5fb4b3 100644 --- a/app/templates/views/jobs/jobs.html +++ b/app/templates/views/jobs/jobs.html @@ -34,14 +34,14 @@ "element": "a", "text": "Upload a letter", "href": url_for('.upload_letter', service_id=current_service.id), - "classes": "govuk-button--secondary" + "classes": "govuk-button--secondary govuk-!-margin-right-3" }) }} {% endif %} {{ govukButton({ "element": "a", "text": "Upload an emergency contact list", "href": url_for('.upload_contact_list', service_id=current_service.id), - "classes": "govuk-button--secondary govuk-!-margin-left-3" + "classes": "govuk-button--secondary" }) }}
    {% endif %} From 41dd7a883b2c617a282d84d5b1777f518d2740d6 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 25 Mar 2020 11:52:30 +0000 Subject: [PATCH 53/63] =?UTF-8?q?Add=20who=E2=80=99s=20its=20for=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have a policy about how suppliers are allowed to use Notify. But we don’t explain it anywhere. Which drives contact to our support form. This commit that adds a new page that explains the policy. I’ve moved the related content about who else can use Notify from the get started page to this page as well, where it doesn’t need to sit in a details element. --- app/main/views/index.py | 8 ++++ app/main/views/sub_navigation_dictionaries.py | 4 ++ app/navigation.py | 4 ++ app/templates/admin_template.html | 4 ++ app/templates/views/get-started.html | 18 ++----- app/templates/views/guidance/who-its-for.html | 48 +++++++++++++++++++ tests/app/main/views/test_index.py | 2 +- 7 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 app/templates/views/guidance/who-its-for.html diff --git a/app/main/views/index.py b/app/main/views/index.py index 86c244d6f..062230eb2 100644 --- a/app/main/views/index.py +++ b/app/main/views/index.py @@ -320,6 +320,14 @@ def get_started(): ) +@main.route('/using-notify/who-its-for') +def who_its_for(): + return render_template( + 'views/guidance/who-its-for.html', + navigation_links=using_notify_nav(), + ) + + @main.route('/trial-mode') @main.route('/features/trial-mode') def trial_mode(): diff --git a/app/main/views/sub_navigation_dictionaries.py b/app/main/views/sub_navigation_dictionaries.py index b3acbb468..3acd67136 100644 --- a/app/main/views/sub_navigation_dictionaries.py +++ b/app/main/views/sub_navigation_dictionaries.py @@ -52,6 +52,10 @@ def using_notify_nav(): "name": "Get started", "link": "main.get_started", }, + { + "name": "Who it’s for", + "link": "main.who_its_for", + }, { "name": "Trial mode", "link": "main.trial_mode_new", diff --git a/app/navigation.py b/app/navigation.py index 53f01483c..b4bbd4429 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -346,6 +346,7 @@ class HeaderNavigation(Navigation): 'no_cookie.view_template_version_preview', 'view_template_versions', 'whitelist', + 'who_its_for', } # header HTML now comes from GOVUK Frontend so requires a boolean, not an attribute @@ -659,6 +660,7 @@ class MainNavigation(Navigation): 'view_provider', 'view_providers', 'no_cookie.view_template_version_preview', + 'who_its_for', } @@ -963,6 +965,7 @@ class CaseworkNavigation(Navigation): 'no_cookie.view_template_version_preview', 'view_template_versions', 'whitelist', + 'who_its_for', } @@ -1268,4 +1271,5 @@ class OrgNavigation(Navigation): 'no_cookie.view_template_version_preview', 'view_template_versions', 'whitelist', + 'who_its_for', } diff --git a/app/templates/admin_template.html b/app/templates/admin_template.html index 96e3d0cf4..804f65809 100644 --- a/app/templates/admin_template.html +++ b/app/templates/admin_template.html @@ -217,6 +217,10 @@ "href": url_for("main.get_started"), "text": "Get started" }, + { + "href": url_for("main.who_its_for"), + "text": "Who it’s for", + }, { "href": url_for("main.trial_mode_new"), "text": "Trial mode" diff --git a/app/templates/views/get-started.html b/app/templates/views/get-started.html index 489791f17..bbbe0969e 100644 --- a/app/templates/views/get-started.html +++ b/app/templates/views/get-started.html @@ -15,21 +15,9 @@

  • Check if GOV.UK Notify is right for you

    Read about our features, pricing and roadmap.

    - {{ govukDetails({ - "summaryText": "Organisations that can use Notify", - "html": ''' -
    -

    Notify is available to:

    -
      -
    • central government departments
      • -
    • local authorities
      • -
    • state-funded schools
      • -
    • the NHS
      • -
    • companies running a service on behalf of a public sector organisation
      • -
    -

    Notify is not currently available to charities.

    -
    ''' - }) }} +

    + Check whether your organisation can use Notify. +

  • diff --git a/app/templates/views/guidance/who-its-for.html b/app/templates/views/guidance/who-its-for.html new file mode 100644 index 000000000..1bbc9e15c --- /dev/null +++ b/app/templates/views/guidance/who-its-for.html @@ -0,0 +1,48 @@ +{% extends "content_template.html" %} +{% from "components/page-header.html" import page_header %} + +{% block per_page_title %} + Who it’s for +{% endblock %} + +{% block content_column_content %} + + {{ page_header( + 'Who it’s for' + ) }} + +

    + GOV.UK Notify is available to: +

    +
      +
    • central government departments
      • +
    • local authorities
      • +
    • the NHS
      • +
    • police forces and fire services
      • +
    • state-funded schools
      • +
    +

    + Notify is not currently available to charities. +

    +

    + If you work for one of these organisations but get an error when you try to create an account, contact support. +

    + +

    Suppliers

    +

    + If you’re doing work for a public sector organisation you can use GOV.UK Notify. +

    +

    + Someone from the public sector organisation you’re working with needs to set up the account. Then they can invite you as a team member. +

    + +

    Members of the public

    +

    + The GOV.UK Notify service is only for people who work in the government + or other public sector organisations. +

    +

    + Find government services and information on GOV.UK. +

    + +{% endblock %} diff --git a/tests/app/main/views/test_index.py b/tests/app/main/views/test_index.py index 246cdfdd3..5b708ab61 100644 --- a/tests/app/main/views/test_index.py +++ b/tests/app/main/views/test_index.py @@ -83,7 +83,7 @@ def test_robots(client): 'features_letters', 'how_to_pay', 'get_started', 'guidance_index', 'branding_and_customisation', 'create_and_send_messages', 'edit_and_format_messages', - 'send_files_by_email', 'upload_a_letter', + 'send_files_by_email', 'upload_a_letter', 'who_its_for', ]) def test_static_pages( client_request, From db2d312ba753713675a12efd39de95e8513f91e7 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 26 Mar 2020 12:41:46 +0000 Subject: [PATCH 54/63] =?UTF-8?q?Link=20to=20=E2=80=98who=20it=E2=80=99s?= =?UTF-8?q?=20for=E2=80=99=20when=20error=20creating=20account?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If someone enters an email address from a domain we don’t recognise we direct them straight to our support channel. This is causing increased contact from suppliers and members of the public. Now that we have a page which explains who can use Notify, let’s direct people there first. Then if they really do need to contact support (because we don’t recognise their organisation) then they can do so from that page. --- app/main/validators.py | 9 +++---- app/templates/views/register.html | 2 +- tests/app/main/test_validators.py | 2 +- tests/app/main/views/test_manage_users.py | 2 +- tests/app/main/views/test_register.py | 30 ++++++++++++++--------- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/app/main/validators.py b/app/main/validators.py index 39bf2fb0b..f35d6208a 100644 --- a/app/main/validators.py +++ b/app/main/validators.py @@ -43,11 +43,10 @@ class ValidGovEmail: return from flask import url_for - message = ( - 'Enter a government email address.' - ' If you think you should have access' - ' contact us' - ).format(url_for('main.support')) + message = ''' + Enter a public sector email address or + find out who can use Notify + '''.format(url_for('main.who_its_for')) if not is_gov_user(field.data.lower()): raise ValidationError(message) diff --git a/app/templates/views/register.html b/app/templates/views/register.html index 71611a4ae..dfcf141bf 100644 --- a/app/templates/views/register.html +++ b/app/templates/views/register.html @@ -14,7 +14,7 @@ Create an account

    Create an account

    {% call form_wrapper(autocomplete=True) %} {{ textbox(form.name, width='3-4') }} - {{ textbox(form.email_address, hint="Must be from a government organisation", width='3-4', safe_error_message=True, autocomplete='email') }} + {{ textbox(form.email_address, hint="Must be from a public sector organisation", width='3-4', safe_error_message=True, autocomplete='email') }}
    {{ textbox(form.mobile_number, width='3-4', hint='We’ll send you a security code by text message') }}
    diff --git a/tests/app/main/test_validators.py b/tests/app/main/test_validators.py index cb3a0b375..1aaeaa01d 100644 --- a/tests/app/main/test_validators.py +++ b/tests/app/main/test_validators.py @@ -36,7 +36,7 @@ def test_valid_email_not_in_valid_domains( ): form = RegisterUserForm(email_address="test@test.com", mobile_number='441231231231') assert not form.validate() - assert "Enter a government email address" in form.errors['email_address'][0] + assert "Enter a public sector email address" in form.errors['email_address'][0] def test_valid_email_in_valid_domains( diff --git a/tests/app/main/views/test_manage_users.py b/tests/app/main/views/test_manage_users.py index 9832d4c12..36095e59b 100644 --- a/tests/app/main/views/test_manage_users.py +++ b/tests/app/main/views/test_manage_users.py @@ -1192,7 +1192,7 @@ def test_edit_user_email_cannot_change_a_gov_email_address_to_a_non_gov_email_ad }, _expected_status=200, ) - assert 'Enter a government email address.' in page.find('span', class_='error-message').text + assert 'Enter a public sector email address' in page.select_one('.error-message').text with client_request.session_transaction() as session: assert 'team_member_email_change-'.format(active_user_with_permissions['id']) not in session diff --git a/tests/app/main/views/test_register.py b/tests/app/main/views/test_register.py index fdb4a4e6f..6cf7a8c16 100644 --- a/tests/app/main/views/test_register.py +++ b/tests/app/main/views/test_register.py @@ -7,6 +7,7 @@ from flask import session, url_for from flask_login import current_user from app.models.user import InvitedUser +from tests.conftest import normalize_spaces def test_render_register_returns_template_with_form(client): @@ -97,20 +98,27 @@ def test_process_register_returns_200_when_mobile_number_is_invalid( def test_should_return_200_when_email_is_not_gov_uk( - client, - mock_send_verify_code, - mock_get_user_by_email, + client_request, mock_get_organisations, - mock_login, ): - response = client.post(url_for('main.register'), - data={'name': 'Bad Mobile', - 'email_address': 'bad_mobile@example.not.right', - 'mobile_number': '+44123412345', - 'password': 'validPassword!'}) + client_request.logout() + page = client_request.post( + 'main.register', + _data={ + 'name': 'Firstname Lastname', + 'email_address': 'bad_mobile@example.not.right', + 'mobile_number': '07900900123', + 'password': 'validPassword!' + }, + _expected_status=200, + ) - assert response.status_code == 200 - assert 'Enter a government email address' in response.get_data(as_text=True) + assert normalize_spaces(page.select_one('.error-message').text) == ( + 'Enter a public sector email address or find out who can use Notify' + ) + assert page.select_one('.error-message a')['href'] == url_for( + 'main.who_its_for' + ) @pytest.mark.parametrize('email_address', ( From 971ecde37a0126d735de41b0b823f393080701b2 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 26 Mar 2020 13:47:25 +0000 Subject: [PATCH 55/63] Update list of organisation types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds emergency services to cover more than just fire and police (eg coastguard). Adds armed forces and GP practices for people who might not be sure that they’re covered under MOD and the NHS respectively. --- app/templates/views/guidance/who-its-for.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/templates/views/guidance/who-its-for.html b/app/templates/views/guidance/who-its-for.html index 1bbc9e15c..5f5055637 100644 --- a/app/templates/views/guidance/who-its-for.html +++ b/app/templates/views/guidance/who-its-for.html @@ -16,9 +16,10 @@

    • central government departments
      • +
    • emergency services
    • local authorities
      • -
    • the NHS
      • -
    • police forces and fire services
      • +
    • the armed forces
      • +
    • the NHS and GP practices
    • state-funded schools

    From 6e78541008c07434aa208c8ae1de62463ede31b8 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Thu, 26 Mar 2020 17:44:16 +0000 Subject: [PATCH 56/63] Fix bug where adding same reply-to-email twice for the same service caused a 5xx This was due to changed quotataion marks in error message. --- app/main/views/service_settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/main/views/service_settings.py b/app/main/views/service_settings.py index 614f861d9..34081cdbb 100644 --- a/app/main/views/service_settings.py +++ b/app/main/views/service_settings.py @@ -391,7 +391,7 @@ def service_add_email_reply_to(service_id): service_id, form.email_address.data )["data"]["id"] except HTTPError as e: - error_msg = "Your service already uses '{}' as an email reply-to address.".format(form.email_address.data) + error_msg = "Your service already uses ‘{}’ as an email reply-to address.".format(form.email_address.data) if e.status_code == 400 and error_msg == e.message: flash(error_msg, 'error') return redirect(url_for('.service_email_reply_to', service_id=service_id)) @@ -519,7 +519,7 @@ def service_edit_email_reply_to(service_id, reply_to_email_id): service_id, form.email_address.data )["data"]["id"] except HTTPError as e: - error_msg = "Your service already uses ‘{}’ as a reply-to email address.".format(form.email_address.data) + error_msg = "Your service already uses ‘{}’ as an email reply-to address.".format(form.email_address.data) if e.status_code == 400 and error_msg == e.message: flash(error_msg, 'error') return redirect(url_for('.service_email_reply_to', service_id=service_id)) From dd6944e8ef337d5cfa87cdced6ec4adde80b802c Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 27 Mar 2020 09:30:28 +0000 Subject: [PATCH 57/63] Use tabular figures for template statistics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We use tabular (not lining) figures everywhere else that we display counts, so that they don’t shift around as the AJAX updates the numbers. There’s a good explanation of the difference here: https://www.fonts.com/content/learning/fontology/level-3/numbers/proportional-vs-tabular-figures --- app/assets/stylesheets/views/dashboard.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/stylesheets/views/dashboard.scss b/app/assets/stylesheets/views/dashboard.scss index 1db274e5f..f694415d6 100644 --- a/app/assets/stylesheets/views/dashboard.scss +++ b/app/assets/stylesheets/views/dashboard.scss @@ -27,7 +27,7 @@ text-align: left; &-bar { - @include bold-27; + @include bold-27($tabular-numbers: true); box-sizing: border-box; display: inline-block; overflow: visible; From ebe2c1c4aeef7533e804c4fb64da6bb60f16cab6 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Mon, 30 Mar 2020 17:11:31 +0100 Subject: [PATCH 58/63] Accept 409 conflict response code when reply-to address already exists Also stop checking error message. --- app/main/views/service_settings.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/main/views/service_settings.py b/app/main/views/service_settings.py index 34081cdbb..78e371363 100644 --- a/app/main/views/service_settings.py +++ b/app/main/views/service_settings.py @@ -391,9 +391,8 @@ def service_add_email_reply_to(service_id): service_id, form.email_address.data )["data"]["id"] except HTTPError as e: - error_msg = "Your service already uses ‘{}’ as an email reply-to address.".format(form.email_address.data) - if e.status_code == 400 and error_msg == e.message: - flash(error_msg, 'error') + if e.status_code == 409 or e.status_code == 400: + flash(e.message, 'error') return redirect(url_for('.service_email_reply_to', service_id=service_id)) else: raise e @@ -519,9 +518,8 @@ def service_edit_email_reply_to(service_id, reply_to_email_id): service_id, form.email_address.data )["data"]["id"] except HTTPError as e: - error_msg = "Your service already uses ‘{}’ as an email reply-to address.".format(form.email_address.data) - if e.status_code == 400 and error_msg == e.message: - flash(error_msg, 'error') + if e.status_code == 400 or e.status_code == 409: + flash(e.message, 'error') return redirect(url_for('.service_email_reply_to', service_id=service_id)) else: raise e From df51bf6f5faabe243297548fa18411349226e355 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Thu, 12 Mar 2020 19:14:17 +0000 Subject: [PATCH 59/63] small QOL changes in conftest if we are asserting on a redirect we expect the result to be 302 (you could still override this to 301 if you want). also give some function calls kwargs to make them easier to read --- tests/conftest.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a5dc9eeb3..78d83225f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -847,10 +847,10 @@ def mock_get_service_email_template_without_placeholders(mocker): template = template_json( service_id, template_id, - "Two week reminder", - "email", - "Your vehicle tax expires soon", - "Your thing is due soon", + name="Two week reminder", + type_="email", + content="Your vehicle tax expires soon", + subject="Your thing is due soon", redact_personalisation=False, ) return {'data': template} @@ -865,10 +865,10 @@ def mock_get_service_letter_template(mocker): template = template_json( service_id, template_id, - "Two week reminder", - "letter", - "Template content with & entity", - "Subject", + name="Two week reminder", + type_="letter", + content="Template content with & entity", + subject="Subject", postage=postage, ) return {'data': template} @@ -2802,7 +2802,7 @@ def os_environ(): os.environ[k] = v -@pytest.fixture +@pytest.fixture # noqa (C901 too complex) def client_request( logged_in_client, mocker, @@ -2855,6 +2855,10 @@ def client_request( url, follow_redirects=_follow_redirects, ) + + if _expected_redirect and _expected_status == 200: + _expected_status = 302 + assert resp.status_code == _expected_status, resp.location if _expected_redirect: assert resp.location == _expected_redirect From c4d839d4f55d0db6c8c510c8e92839f354460c4a Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 10 Mar 2020 15:12:19 +0000 Subject: [PATCH 60/63] input letter address data in a single block rather than in multiple placeholders - this is the first step towards making postcodes non-required, which is the first step towards international letters. they still populate address_line_# and postcode fields under the hood - to keep validation working the same, the last line always goes into `postcode`. the form normalises whitespace, removes extra new lines, and enforces that you have between three and seven lines. if the letter repeats address placeholders further down (eg "Dear ((address_line_1))"), then it'll fill those in as well. It'll still prompt you to fill them in, but they'll be pre-filled. --- app/main/forms.py | 62 ++++- app/main/views/send.py | 72 ++++++ app/navigation.py | 4 + .../views/send-one-off-letter-address.html | 45 ++++ app/templates/views/templates/_template.html | 10 +- tests/app/main/views/test_send.py | 218 +++++++++++++++++- tests/conftest.py | 13 ++ 7 files changed, 411 insertions(+), 13 deletions(-) create mode 100644 app/templates/views/send-one-off-letter-address.html diff --git a/app/main/forms.py b/app/main/forms.py index 85639dfa7..a665db984 100644 --- a/app/main/forms.py +++ b/app/main/forms.py @@ -1,3 +1,4 @@ +import re import weakref from datetime import datetime, timedelta from itertools import chain @@ -9,12 +10,17 @@ from flask_wtf import FlaskForm as Form from flask_wtf.file import FileAllowed from flask_wtf.file import FileField as FileField_wtf from notifications_utils.columns import Columns -from notifications_utils.formatters import strip_whitespace +from notifications_utils.formatters import ( + normalise_whitespace_and_newlines, + remove_whitespace_before_punctuation, + strip_whitespace, +) from notifications_utils.recipients import ( InvalidPhoneError, normalise_phone_number, validate_phone_number, ) +from notifications_utils.take import Take from wtforms import ( BooleanField, DateField, @@ -362,6 +368,23 @@ class StripWhitespaceStringField(StringField): super(StringField, self).__init__(label, **kwargs) +class StripWhitespaceTextAreaField(TextAreaField): + def process_formdata(self, valuelist): + if valuelist: + self.data = Take( + valuelist[0] + ).then( + remove_whitespace_before_punctuation + ).then( + normalise_whitespace_and_newlines + ).then( + # similar to normalise_multiple_newlines but taking everything down to one `\n` instead of two + lambda value: re.compile(r'\n{2,}').sub('\n', value) + ).then( + str.strip + ) + + class OnOffField(RadioField): def __init__(self, label, choices=None, *args, **kwargs): @@ -741,6 +764,43 @@ class SMSTemplateForm(BaseTemplateForm): OnlySMSCharacters()(None, field) +class LetterAddressForm(StripWhitespaceForm): + MIN_ADDRESS_LINES = 3 + MAX_ADDRESS_LINES = 7 + + address = StripWhitespaceTextAreaField( + 'Address', + validators=[DataRequired(message="Cannot be empty")] + ) + + def validate_address(self, field): + lines = field.data.splitlines() + if len(lines) < self.MIN_ADDRESS_LINES: + raise ValidationError('Address must be at least 3 lines long') + if len(lines) > self.MAX_ADDRESS_LINES: + raise ValidationError('Address must be no more than 7 lines long') + + @property + def as_address_lines_1_to_7_with_postcode(self): + lines = self.address.data.splitlines() + placeholders = {} + + # set all placeholders to empty strings, or all_placeholders_in_session will always return false. + # note that it must be `address line #` with spaces, not underscores or dashes + for i in range(1, 7): + placeholders[f'address line {i}'] = '' + + # unroll the address into lines, and place into the session in the underlying placeholder names + # postcode is required so make sure we put the last value in that + # TODO: When postcode is no longer a required field, remove this special case and just use `address line #` + address_lines, last_address_line = lines[:-1], lines[-1] + for i, line in enumerate(address_lines, start=1): + placeholders[f'address line {i}'] = line + placeholders['postcode'] = last_address_line + + return placeholders + + class EmailTemplateForm(BaseTemplateForm): subject = TextAreaField( u'Subject', diff --git a/app/main/views/send.py b/app/main/views/send.py index 7784dd902..8db4d931e 100644 --- a/app/main/views/send.py +++ b/app/main/views/send.py @@ -39,6 +39,7 @@ from app.main import main, no_cookie from app.main.forms import ( ChooseTimeForm, CsvUploadForm, + LetterAddressForm, SetSenderForm, get_placeholder_form_instance, ) @@ -307,6 +308,11 @@ def send_test(service_id, template_id): return_to='view_template', template_id=template_id)) + if db_template['template_type'] == 'letter': + return redirect( + url_for('.send_one_off_letter_address', service_id=service_id, template_id=template_id) + ) + return redirect(url_for( { 'main.send_test': '.send_test_step', @@ -329,6 +335,72 @@ def get_notification_check_endpoint(service_id, template): )) +@main.route( + "/services//send//one-off/address", + methods=['GET', 'POST'] +) +@user_has_permissions('send_messages', restrict_admin_usage=True) +def send_one_off_letter_address(service_id, template_id): + if {'recipient', 'placeholders'} - set(session.keys()): + # if someone has come here via a bookmark or back button they might have some stuff still in their session + return redirect(url_for('.send_one_off', service_id=service_id, template_id=template_id)) + + db_template = current_service.get_template_with_user_permission_or_403(template_id, current_user) + + session['send_test_letter_page_count'] = get_page_count_for_letter(db_template) + + template = get_template( + db_template, + current_service, + show_recipient=True, + letter_preview_url=url_for( + 'no_cookie.send_test_preview', + service_id=service_id, + template_id=template_id, + filetype='png', + ), + page_count=session['send_test_letter_page_count'], + email_reply_to=None, + sms_sender=None + ) + + form = LetterAddressForm() + + if form.validate_on_submit(): + session['placeholders'].update(form.as_address_lines_1_to_7_with_postcode) + + placeholders = fields_to_fill_in( + template, + prefill_current_user=(request.endpoint == 'main.send_test_step'), + ) + if all_placeholders_in_session(placeholders): + return get_notification_check_endpoint(service_id, template) + + first_non_address_placeholder_index = len(first_column_headings['letter']) + return redirect(url_for( + 'main.send_one_off_step', + service_id=service_id, + template_id=template_id, + step_index=first_non_address_placeholder_index, + )) + + return render_template( + 'views/send-one-off-letter-address.html', + page_title=get_send_test_page_title( + template_type='letter', + help_argument=None, + entering_recipient=True, + name=template.name, + ), + template=template, + form=form, + optional_placeholder=False, + back_link=get_back_link(service_id, template, 0), + help=False, + link_to_upload=True, + ) + + @main.route( "/services//send//test/step-", methods=['GET', 'POST'], diff --git a/app/navigation.py b/app/navigation.py index 0f11b3df1..7343559d6 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -259,6 +259,7 @@ class HeaderNavigation(Navigation): 'send_messages', 'send_notification', 'send_one_off', + 'send_one_off_letter_address', 'send_one_off_step', 'send_test', 'no_cookie.send_test_preview', @@ -386,6 +387,7 @@ class MainNavigation(Navigation): 'manage_template_folder', 'send_messages', 'send_one_off', + 'send_one_off_letter_address', 'send_one_off_step', 'send_test', 'no_cookie.send_test_preview', @@ -673,6 +675,7 @@ class CaseworkNavigation(Navigation): 'choose_from_contact_list', 'choose_template', 'send_one_off', + 'send_one_off_letter_address', 'send_one_off_step', 'send_test', 'send_test_step', @@ -1156,6 +1159,7 @@ class OrgNavigation(Navigation): 'send_messages', 'send_notification', 'send_one_off', + 'send_one_off_letter_address', 'send_one_off_step', 'send_test', 'no_cookie.send_test_preview', diff --git a/app/templates/views/send-one-off-letter-address.html b/app/templates/views/send-one-off-letter-address.html new file mode 100644 index 000000000..2bed9c15a --- /dev/null +++ b/app/templates/views/send-one-off-letter-address.html @@ -0,0 +1,45 @@ +{% extends "withnav_template.html" %} +{% from "components/page-header.html" import page_header %} +{% from "components/page-footer.html" import page_footer %} +{% from "components/message-count-label.html" import recipient_count_label %} +{% from "components/textbox.html" import textbox %} +{% from "components/form.html" import form_wrapper %} + +{% block service_page_title %} + {{ page_title }} +{% endblock %} + +{% block maincolumn_content %} + + {{ page_header( + page_title, + back_link=back_link + ) }} + + {% call form_wrapper( + class='send-one-off-form', + module="autofocus", + data_kwargs={'force-focus': True} + ) %} +

    +
    + {{ textbox( + form.address, + rows=4, + width='1-1', + autofocus=True, + autosize=True, + ) }} +
    +
    +

    + + Upload a list of {{ recipient_count_label(999, template.template_type) }} + +

    + {{ page_footer('Continue') }} + {% endcall %} + + {{ template|string }} + +{% endblock %} diff --git a/app/templates/views/templates/_template.html b/app/templates/views/templates/_template.html index 97de4182f..0b1c39e77 100644 --- a/app/templates/views/templates/_template.html +++ b/app/templates/views/templates/_template.html @@ -16,11 +16,11 @@
    {% if template.template_type == 'letter' %} - {% if letter_too_long %} - {% call banner_wrapper(type='dangerous') %} - {% include "partials/check/letter-too-long.html" %} - {% endcall %} - {% endif %} + {% if letter_too_long %} + {% call banner_wrapper(type='dangerous') %} + {% include "partials/check/letter-too-long.html" %} + {% endcall %} + {% endif %} {% if current_user.has_permissions('send_messages', restrict_admin_usage=True) and not letter_too_long %}
    diff --git a/tests/app/main/views/test_send.py b/tests/app/main/views/test_send.py index 9d0665a5c..9a3352184 100644 --- a/tests/app/main/views/test_send.py +++ b/tests/app/main/views/test_send.py @@ -779,6 +779,7 @@ def test_upload_valid_csv_only_sets_meta_if_filename_known( mock_get_job_doesnt_exist, mock_get_jobs, mock_s3_set_metadata, + mock_template_preview, fake_uuid, ): @@ -790,10 +791,6 @@ def test_upload_valid_csv_only_sets_meta_if_filename_known( 'app.main.views.send.get_page_count_for_letter', return_value=5, ) - mocker.patch( - 'app.main.views.send.TemplatePreview.from_utils_template', - return_value='foo' - ) client_request.get( 'no_cookie.check_messages_preview', @@ -1331,9 +1328,9 @@ def test_send_one_off_has_skip_link( @pytest.mark.parametrize('template_type, expected_sticky', [ ('sms', False), ('email', True), - ('letter', True), + ('letter', False), ]) -def test_send_one_off_has_sticky_header_for_email_and_letter( +def test_send_one_off_has_sticky_header_for_email( mocker, client_request, fake_uuid, @@ -1342,7 +1339,7 @@ def test_send_one_off_has_sticky_header_for_email_and_letter( template_type, expected_sticky, ): - template_data = create_template(template_type=template_type) + template_data = create_template(template_type=template_type, content='((body))') mocker.patch('app.service_api_client.get_service_template', return_value={'data': template_data}) mocker.patch('app.main.views.send.get_page_count_for_letter', return_value=9) @@ -1357,6 +1354,39 @@ def test_send_one_off_has_sticky_header_for_email_and_letter( assert bool(page.select('.js-stick-at-top-when-scrolling')) == expected_sticky +def test_send_one_off_has_sticky_header_for_letter_on_non_address_placeholders( + mocker, + client_request, + fake_uuid, + mock_get_live_service, +): + template_data = create_template(template_type='letter', content='((body))') + mocker.patch('app.service_api_client.get_service_template', return_value={'data': template_data}) + mocker.patch('app.main.views.send.get_page_count_for_letter', return_value=9) + + with client_request.session_transaction() as session: + session['send_test_letter_page_count'] = 1 + session['recipient'] = '' + session['placeholders'] = { + 'address line 1': 'foo', + 'address line 2': 'bar', + 'address line 3': '', + 'address line 4': '', + 'address line 5': '', + 'address line 6': '', + 'postcode': 'SW1 1AA', + } + + page = client_request.get( + 'main.send_one_off_step', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + step_index=7, # letter template has 7 placeholders – we’re at the end + _follow_redirects=True, + ) + assert page.select('.js-stick-at-top-when-scrolling') + + @pytest.mark.parametrize('user', ( create_active_user_with_permissions(), create_active_caseworking_user(), @@ -2146,6 +2176,180 @@ def test_send_test_clears_session( assert session['placeholders'] == {} +def test_send_one_off_redirects_to_letter_address(client_request, fake_uuid, mock_get_service_letter_template): + with client_request.session_transaction() as session: + session['placeholders'] = {'foo': 'some old data that we dont care about'} + + client_request.get( + 'main.send_one_off', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + _expected_redirect=url_for( + 'main.send_one_off_letter_address', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + _external=True, + ) + ) + # make sure it cleared session first + with client_request.session_transaction() as session: + assert session['recipient'] is None + assert session['placeholders'] == {} + + +def test_send_one_off_letter_address_shows_form( + client_request, + fake_uuid, + mock_get_service_letter_template, + mock_template_preview, +): + with client_request.session_transaction() as session: + session['recipient'] = None + session['placeholders'] = {} + + page = client_request.get( + 'main.send_one_off_letter_address', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid + ) + + assert page.select_one('h1').text.strip() == 'Send ‘Two week reminder’' + + form = page.select_one('form') + + assert form.select_one('label').text.strip() == 'Address' + assert form.select_one('textarea').attrs['name'] == 'address' + + upload_link = form.select_one('a') + + assert upload_link.text.strip() == 'Upload a list of addresses' + assert upload_link['href'] == url_for( + 'main.send_messages', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + ) + + assert ( + page.find_all('a', {'class': 'govuk-back-link'})[0]['href'] + ) == url_for('main.view_template', service_id=SERVICE_ONE_ID, template_id=fake_uuid) + + +@pytest.mark.parametrize(['form_data', 'expected_placeholders'], [ + # minimal + ('\n'.join(['a', 'b', 'c']), { + 'address line 1': 'a', + 'address line 2': 'b', + 'address line 3': '', + 'address line 4': '', + 'address line 5': '', + 'address line 6': '', + 'postcode': 'c', + }), + # maximal + ('\n'.join(['a', 'b', 'c', 'd', 'e', 'f', 'g']), { + 'address line 1': 'a', + 'address line 2': 'b', + 'address line 3': 'c', + 'address line 4': 'd', + 'address line 5': 'e', + 'address line 6': 'f', + 'postcode': 'g', + }), + # it ignores empty lines and strips whitespace from each line. + # It also strips extra whitespace from the middle of lines. + ('\n a\ta \n\n\n \n\n\n\nb b \r\nc', { + 'address line 1': 'a\ta', + 'address line 2': 'b b', + 'address line 3': '', + 'address line 4': '', + 'address line 5': '', + 'address line 6': '', + 'postcode': 'c', + }), +]) +def test_send_one_off_letter_address_populates_address_fields_in_session( + client_request, + fake_uuid, + mock_get_service_letter_template, + mock_template_preview, + form_data, + expected_placeholders +): + with client_request.session_transaction() as session: + session['recipient'] = None + session['placeholders'] = {} + + client_request.post( + 'main.send_one_off_letter_address', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + _data={'address': form_data}, + # there are no additional placeholders so go straight to the check page + _expected_redirect=url_for( + 'main.check_notification', + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + _external=True, + ), + ) + with client_request.session_transaction() as session: + assert session['placeholders'] == expected_placeholders + + +@pytest.mark.parametrize(['form_data', 'expected_error_message'], [ + ('', 'Cannot be empty'), + ('a\n\n\n\nb', 'Address must be at least 3 lines long'), + ('\n'.join(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']), 'Address must be no more than 7 lines long'), +]) +def test_send_one_off_letter_address_rejects_bad_addresses( + client_request, + fake_uuid, + mock_get_service_letter_template, + mock_template_preview, + form_data, + expected_error_message +): + with client_request.session_transaction() as session: + session['recipient'] = None + session['placeholders'] = {} + + page = client_request.post( + 'main.send_one_off_letter_address', + _data={'address': form_data}, + service_id=SERVICE_ONE_ID, + template_id=fake_uuid, + _expected_status=200 + ) + + error = page.select('form .error-message') + assert normalize_spaces(error[0].text) == expected_error_message + + +def test_send_one_off_letter_address_goes_to_next_placeholder(client_request, mock_template_preview, mocker): + with client_request.session_transaction() as session: + session['recipient'] = None + session['placeholders'] = {} + + template_data = create_template(template_type='letter', content='((foo))') + + mocker.patch('app.service_api_client.get_service_template', return_value={'data': template_data}) + + client_request.post( + 'main.send_one_off_letter_address', + service_id=SERVICE_ONE_ID, + template_id=template_data['id'], + _data={'address': 'a\nb\nc'}, + # step 0-6 represent address line 1-6 and postcode. step 7 is the first non address placeholder + _expected_redirect=url_for( + 'main.send_one_off_step', + service_id=SERVICE_ONE_ID, + template_id=template_data['id'], + step_index=7, + _external=True, + ) + ) + + def test_download_example_csv( logged_in_client, mocker, diff --git a/tests/conftest.py b/tests/conftest.py index 78d83225f..1510ccf1f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3483,6 +3483,19 @@ def mock_get_returned_letter_summary_with_no_returned_letters(mocker): ) +@pytest.fixture +def mock_template_preview(mocker): + content = b'{"count":1}' + status_code = 200 + headers = {} + example_response = (content, status_code, headers) + mocker.patch('app.template_previews.TemplatePreview.from_database_object', return_value=example_response) + mocker.patch('app.template_previews.TemplatePreview.from_valid_pdf_file', return_value=example_response) + mocker.patch('app.template_previews.TemplatePreview.from_invalid_pdf_file', return_value=example_response) + mocker.patch('app.template_previews.TemplatePreview.from_example_template', return_value=example_response) + mocker.patch('app.template_previews.TemplatePreview.from_utils_template', return_value=example_response) + + def create_api_user_active(with_unique_id=False): return { 'id': str(uuid4()) if with_unique_id else sample_uuid(), From 435081a2a4bfd1b31ce2c7b8c79fd7d6f9be44c8 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Tue, 31 Mar 2020 08:59:45 +0100 Subject: [PATCH 61/63] Bump utils to 36.9.3 to bring in new version of Bleach --- requirements-app.txt | 2 +- requirements.txt | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-app.txt b/requirements-app.txt index 95332f00a..d078e09bb 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -24,5 +24,5 @@ awscli-cwlogs>=1.4,<1.5 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default itsdangerous==0.24 # pyup: <1.0.0 -git+https://github.com/alphagov/notifications-utils.git@36.9.1#egg=notifications-utils==36.9.1 +git+https://github.com/alphagov/notifications-utils.git@36.9.3#egg=notifications-utils==36.9.3 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha diff --git a/requirements.txt b/requirements.txt index 49e7f5607..3ae10fb2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,14 +26,14 @@ awscli-cwlogs>=1.4,<1.5 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default itsdangerous==0.24 # pyup: <1.0.0 -git+https://github.com/alphagov/notifications-utils.git@36.9.1#egg=notifications-utils==36.9.1 +git+https://github.com/alphagov/notifications-utils.git@36.9.3#egg=notifications-utils==36.9.3 git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha ## The following requirements were added by pip freeze: -awscli==1.18.23 -bleach==3.1.3 +awscli==1.18.32 +bleach==3.1.4 boto3==1.10.38 -botocore==1.15.23 +botocore==1.15.32 certifi==2019.11.28 chardet==3.0.4 click==7.1.1 @@ -63,7 +63,7 @@ PyJWT==1.7.1 PyPDF2==1.26.0 python-dateutil==2.8.1 python-json-logger==0.1.11 -PyYAML==5.3 +PyYAML==5.3.1 redis==3.4.1 requests==2.23.0 rsa==3.4.2 From 7fb8e1de92f1d4b16de4bb8011b5a240270765dc Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 3 Mar 2020 17:40:50 +0000 Subject: [PATCH 62/63] Use statistics for returned letters on dashboard This should be faster and more accurate than querying all the reports. --- app/main/views/platform_admin.py | 3 + app/models/service.py | 33 +++---- app/notify_client/service_api_client.py | 4 + app/templates/views/dashboard/_inbox.html | 4 +- tests/app/main/views/test_accept_invite.py | 4 +- tests/app/main/views/test_dashboard.py | 98 +++++++++---------- tests/app/main/views/test_platform_admin.py | 7 +- tests/app/main/views/test_sign_out.py | 2 +- .../notify_client/test_service_api_client.py | 19 ++++ tests/conftest.py | 11 +++ 10 files changed, 102 insertions(+), 83 deletions(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 50917c819..eb7eb7647 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -368,6 +368,9 @@ def platform_admin_returned_letters(): try: letter_jobs_client.submit_returned_letters(references) + redis_client.delete_cache_keys_by_pattern( + 'service-????????-????-????-????-????????????-returned-letters-statistics' + ) redis_client.delete_cache_keys_by_pattern( 'service-????????-????-????-????-????????????-returned-letters-summary' ) diff --git a/app/models/service.py b/app/models/service.py index 20c841c33..c053438cc 100644 --- a/app/models/service.py +++ b/app/models/service.py @@ -1,8 +1,4 @@ -from datetime import datetime, timedelta - -from dateutil.parser import parse from flask import abort, current_app -from notifications_utils.timezones import local_timezone from werkzeug.utils import cached_property from app.models import JSONModel @@ -668,30 +664,25 @@ class Service(JSONModel): if test: yield BASE + '_incomplete' + tag + @cached_property + def returned_letter_statistics(self): + return service_api_client.get_returned_letter_statistics(self.id) + @cached_property def returned_letter_summary(self): return service_api_client.get_returned_letter_summary(self.id) @property - def most_recent_returned_letter_report(self): - if not self.returned_letter_summary: - return None - return parse( - self.returned_letter_summary[0]['reported_at'] + " 00:00:00" - ).replace(tzinfo=local_timezone) + def count_of_returned_letters_in_last_7_days(self): + return self.returned_letter_statistics['returned_letter_count'] @property - def count_of_returned_letters_in_last_7_days(self): - seven_days_ago = ( - datetime.now() - timedelta(days=7) - ).replace( - hour=0, minute=0, second=0 - ) - return sum( - report['returned_letter_count'] - for report in self.returned_letter_summary - if parse(report['reported_at'] + " 00:00:00") >= seven_days_ago - ) + def date_of_most_recent_returned_letter_report(self): + return self.returned_letter_statistics['most_recent_report'] + + @property + def has_returned_letters(self): + return bool(self.date_of_most_recent_returned_letter_report) @property def contact_lists(self): diff --git a/app/notify_client/service_api_client.py b/app/notify_client/service_api_client.py index b89e5c196..b0ad13904 100644 --- a/app/notify_client/service_api_client.py +++ b/app/notify_client/service_api_client.py @@ -569,6 +569,10 @@ class ServiceAPIClient(NotifyAdminAPIClient): def get_service_data_retention(self, service_id): return self.get("/service/{}/data-retention".format(service_id)) + @cache.set('service-{service_id}-returned-letters-statistics') + def get_returned_letter_statistics(self, service_id): + return self.get("service/{}/returned-letter-statistics".format(service_id)) + @cache.set('service-{service_id}-returned-letters-summary') def get_returned_letter_summary(self, service_id): return self.get("service/{}/returned-letter-summary".format(service_id)) diff --git a/app/templates/views/dashboard/_inbox.html b/app/templates/views/dashboard/_inbox.html index a73ee296c..901b23195 100644 --- a/app/templates/views/dashboard/_inbox.html +++ b/app/templates/views/dashboard/_inbox.html @@ -17,7 +17,7 @@ {% endif %} {% endif %} - {% if current_service.returned_letter_summary %} + {% if current_service.has_returned_letters %} {% endif %} diff --git a/tests/app/main/views/test_accept_invite.py b/tests/app/main/views/test_accept_invite.py index c5631ce2a..45ab72622 100644 --- a/tests/app/main/views/test_accept_invite.py +++ b/tests/app/main/views/test_accept_invite.py @@ -149,7 +149,7 @@ def test_accepting_invite_removes_invite_from_session( mock_get_billable_units, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, fake_uuid, user, landing_page_title, @@ -477,7 +477,7 @@ def test_new_invited_user_verifies_and_added_to_service( mock_get_service_statistics, mock_get_usage, mock_get_free_sms_fragment_limit, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, mock_create_event, mocker, ): diff --git a/tests/app/main/views/test_dashboard.py b/tests/app/main/views/test_dashboard.py index 208cd1e20..34d29dbd7 100644 --- a/tests/app/main/views/test_dashboard.py +++ b/tests/app/main/views/test_dashboard.py @@ -143,7 +143,7 @@ def test_get_started( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): mocker.patch( 'app.template_statistics_client.get_template_statistics_for_service', @@ -168,7 +168,7 @@ def test_get_started_is_hidden_once_templates_exist( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): mocker.patch( 'app.template_statistics_client.get_template_statistics_for_service', @@ -194,7 +194,7 @@ def test_inbound_messages_not_visible_to_service_without_permissions( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): service_one['permissions'] = [] @@ -219,7 +219,7 @@ def test_inbound_messages_shows_count_of_messages_when_there_are_messages( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): service_one['permissions'] = ['inbound_sms'] page = client_request.get( @@ -246,7 +246,7 @@ def test_inbound_messages_shows_count_of_messages_when_there_are_no_messages( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary_with_no_messages, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): service_one['permissions'] = ['inbound_sms'] page = client_request.get( @@ -488,7 +488,7 @@ def test_returned_letters_not_visible_if_service_has_no_returned_letters( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): page = client_request.get( 'main.service_dashboard', @@ -511,21 +511,11 @@ def test_returned_letters_shows_count_of_recently_returned_letters( mock_get_inbound_sms_summary, ): mocker.patch( - 'app.service_api_client.get_returned_letter_summary', - return_value=[ - # Today (should be counted) - { - 'returned_letter_count': 1000, 'reported_at': '2020-01-10' - }, - # Just within the last 7 days (should be counted) - { - 'returned_letter_count': 3000, 'reported_at': '2020-01-3' - }, - # Just after the last 7 days (should not be counted) - { - 'returned_letter_count': 2000, 'reported_at': '2020-01-2' - }, - ], + 'app.service_api_client.get_returned_letter_statistics', + return_value={ + 'returned_letter_count': 4000, + 'most_recent_report': '2020-01-10', + }, ) page = client_request.get( 'main.service_dashboard', @@ -540,29 +530,29 @@ def test_returned_letters_shows_count_of_recently_returned_letters( ) -@pytest.mark.parametrize('reporting_date, expected_message', ( - ('2020-02-02', ( +@pytest.mark.parametrize('reporting_date, count, expected_message', ( + ('2020-02-02', 1, ( '1 returned letter latest report today' )), - ('2020-02-01', ( + ('2020-02-01', 1, ( '1 returned letter latest report yesterday' )), - ('2020-01-31', ( + ('2020-01-31', 1, ( '1 returned letter latest report 2 days ago' )), - ('2020-01-26', ( + ('2020-01-26', 1, ( '1 returned letter latest report 7 days ago' )), - ('2020-01-25', ( + ('2020-01-25', 0, ( '0 returned letters latest report 8 days ago' )), - ('2020-01-01', ( + ('2020-01-01', 0, ( '0 returned letters latest report 1 month ago' )), - ('2019-09-09', ( + ('2019-09-09', 0, ( '0 returned letters latest report 4 months ago' )), - ('2010-10-10', ( + ('2010-10-10', 0, ( '0 returned letters latest report 9 years ago' )), )) @@ -579,15 +569,15 @@ def test_returned_letters_only_counts_recently_returned_letters( mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary_with_no_messages, reporting_date, + count, expected_message, ): mocker.patch( - 'app.service_api_client.get_returned_letter_summary', - return_value=[ - { - 'returned_letter_count': 1, 'reported_at': reporting_date - }, - ], + 'app.service_api_client.get_returned_letter_statistics', + return_value={ + 'returned_letter_count': count, + 'most_recent_report': reporting_date, + }, ) page = client_request.get( 'main.service_dashboard', @@ -609,7 +599,7 @@ def test_should_show_recent_templates_on_dashboard( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): mock_template_stats = mocker.patch('app.template_statistics_client.get_template_statistics_for_service', return_value=copy.deepcopy(stub_template_stats)) @@ -663,7 +653,7 @@ def test_should_not_show_recent_templates_on_dashboard_if_only_one_template_used mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, stats, ): mock_template_stats = mocker.patch( @@ -818,7 +808,7 @@ def test_should_show_upcoming_jobs_on_dashboard( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): page = client_request.get( 'main.service_dashboard', @@ -875,7 +865,7 @@ def test_correct_font_size_for_big_numbers( mock_get_jobs, mock_get_usage, mock_get_free_sms_fragment_limit, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, service_one, permissions, totals, @@ -912,7 +902,7 @@ def test_should_not_show_jobs_on_dashboard_for_users_with_uploads_page( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): page = client_request.get( 'main.service_dashboard', @@ -1107,7 +1097,7 @@ def test_menu_send_messages( mock_get_usage, mock_get_inbound_sms_summary, mock_get_free_sms_fragment_limit, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): service_one['permissions'] = ['email', 'sms', 'letter', 'upload_letters'] @@ -1143,7 +1133,7 @@ def test_menu_send_messages_when_service_does_not_have_upload_letters_permission mock_get_usage, mock_get_inbound_sms_summary, mock_get_free_sms_fragment_limit, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): with app_.test_request_context(): resp = _test_dashboard_menu( @@ -1168,7 +1158,7 @@ def test_menu_manage_service( mock_get_service_statistics, mock_get_usage, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, mock_get_free_sms_fragment_limit, ): with app_.test_request_context(): @@ -1200,7 +1190,7 @@ def test_menu_manage_api_keys( mock_get_service_statistics, mock_get_usage, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, mock_get_free_sms_fragment_limit, ): with app_.test_request_context(): @@ -1230,7 +1220,7 @@ def test_menu_all_services_for_platform_admin_user( mock_get_service_statistics, mock_get_usage, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, mock_get_free_sms_fragment_limit, ): with app_.test_request_context(): @@ -1263,7 +1253,7 @@ def test_route_for_service_permissions( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): with app_.test_request_context(): validate_route_permission( @@ -1317,7 +1307,7 @@ def test_service_dashboard_updates_gets_dashboard_totals( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): mocker.patch('app.main.views.dashboard.get_dashboard_totals', return_value={ 'email': {'requested': 123, 'delivered': 0, 'failed': 0}, @@ -1500,7 +1490,7 @@ def test_org_breadcrumbs_do_not_show_if_service_has_no_org( mock_get_jobs, mock_get_usage, mock_get_free_sms_fragment_limit, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): page = client_request.get('main.service_dashboard', service_id=SERVICE_ONE_ID) @@ -1515,7 +1505,7 @@ def test_org_breadcrumbs_do_not_show_if_user_is_not_an_org_member( active_caseworking_user, client_request, mock_get_template_folders, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, ): # active_caseworking_user is not an org member @@ -1538,7 +1528,7 @@ def test_org_breadcrumbs_show_if_user_is_a_member_of_the_services_org( mock_get_jobs, mock_get_usage, mock_get_free_sms_fragment_limit, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, active_user_with_permissions, client_request, ): @@ -1566,7 +1556,7 @@ def test_org_breadcrumbs_do_not_show_if_user_is_a_member_of_the_services_org_but mock_get_jobs, mock_get_usage, mock_get_free_sms_fragment_limit, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, active_user_with_permissions, client_request, ): @@ -1591,7 +1581,7 @@ def test_org_breadcrumbs_show_if_user_is_platform_admin( mock_get_jobs, mock_get_usage, mock_get_free_sms_fragment_limit, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, platform_admin_user, platform_admin_client, ): @@ -1623,7 +1613,7 @@ def test_should_show_usage_on_dashboard( mock_get_jobs, mock_get_usage, mock_get_free_sms_fragment_limit, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters, permissions, ): service_one['permissions'] = permissions diff --git a/tests/app/main/views/test_platform_admin.py b/tests/app/main/views/test_platform_admin.py index a157e3e46..84ce28a63 100644 --- a/tests/app/main/views/test_platform_admin.py +++ b/tests/app/main/views/test_platform_admin.py @@ -706,9 +706,10 @@ def test_platform_admin_submit_returned_letters(mocker, platform_admin_client): ) mock_client.assert_called_once_with(['REF1', 'REF2']) - redis.delete_cache_keys_by_pattern.assert_called_once_with( - 'service-????????-????-????-????-????????????-returned-letters-summary' - ) + assert redis.delete_cache_keys_by_pattern.call_args_list == [ + call('service-????????-????-????-????-????????????-returned-letters-statistics'), + call('service-????????-????-????-????-????????????-returned-letters-summary'), + ] assert response.status_code == 302 assert response.location == url_for('main.platform_admin_returned_letters', _external=True) diff --git a/tests/app/main/views/test_sign_out.py b/tests/app/main/views/test_sign_out.py index 2d1be4c2f..fdb604acf 100644 --- a/tests/app/main/views/test_sign_out.py +++ b/tests/app/main/views/test_sign_out.py @@ -31,7 +31,7 @@ def test_sign_out_user( mock_get_usage, mock_get_free_sms_fragment_limit, mock_get_inbound_sms_summary, - mock_get_returned_letter_summary_with_no_returned_letters, + mock_get_returned_letter_statistics_with_no_returned_letters ): with client_request.session_transaction() as session: assert session.get('user_id') is not None diff --git a/tests/app/notify_client/test_service_api_client.py b/tests/app/notify_client/test_service_api_client.py index de7e731be..ef8dfc33d 100644 --- a/tests/app/notify_client/test_service_api_client.py +++ b/tests/app/notify_client/test_service_api_client.py @@ -335,6 +335,25 @@ def test_client_returns_count_of_service_templates( ], {'data_from': 'api'}, ), + ( + service_api_client.get_returned_letter_statistics, + [SERVICE_ONE_ID], + [ + call('service-{}-returned-letters-statistics'.format(SERVICE_ONE_ID)) + ], + None, + [ + call('service/{}/returned-letter-statistics'.format(SERVICE_ONE_ID)) + ], + [ + call( + 'service-{}-returned-letters-statistics'.format(SERVICE_ONE_ID), + '{"data_from": "api"}', + ex=604800, + ) + ], + {'data_from': 'api'}, + ), ] ) def test_returns_value_from_cache( diff --git a/tests/conftest.py b/tests/conftest.py index 1510ccf1f..fa31dc42f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3496,6 +3496,17 @@ def mock_template_preview(mocker): mocker.patch('app.template_previews.TemplatePreview.from_utils_template', return_value=example_response) +@pytest.fixture(scope='function') +def mock_get_returned_letter_statistics_with_no_returned_letters(mocker): + return mocker.patch( + 'app.service_api_client.get_returned_letter_statistics', + return_value={ + 'returned_letter_count': 0, + 'most_recent_report': None, + }, + ) + + def create_api_user_active(with_unique_id=False): return { 'id': str(uuid4()) if with_unique_id else sample_uuid(), From 61d87bf6c0e676bc3f860af644cc3b8a1f0e6179 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 31 Mar 2020 14:01:28 +0100 Subject: [PATCH 63/63] Let platform admins delete returned letters cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just in case there’s ever a problem with something in these caches. We missed this before. --- app/main/views/platform_admin.py | 2 ++ tests/app/main/views/test_platform_admin.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index eb7eb7647..e2f9b27cd 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -408,6 +408,8 @@ def clear_cache(): 'service-????????-????-????-????-????????????-templates', 'service-????????-????-????-????-????????????-data-retention', 'service-????????-????-????-????-????????????-template-folders', + 'service-????????-????-????-????-????????????-returned-letters-statistics', + 'service-????????-????-????-????-????????????-returned-letters-summary', ]), ('template', [ 'service-????????-????-????-????-????????????-templates', diff --git a/tests/app/main/views/test_platform_admin.py b/tests/app/main/views/test_platform_admin.py index 84ce28a63..28effd4dc 100644 --- a/tests/app/main/views/test_platform_admin.py +++ b/tests/app/main/views/test_platform_admin.py @@ -750,6 +750,15 @@ def test_clear_cache_shows_form(client_request, platform_admin_user, mocker): call('template-????????-????-????-????-????????????-version-*'), call('template-????????-????-????-????-????????????-versions'), ], 'Removed 3 template objects from redis'), + ('service', [ + call('has_jobs-????????-????-????-????-????????????'), + call('service-????????-????-????-????-????????????'), + call('service-????????-????-????-????-????????????-templates'), + call('service-????????-????-????-????-????????????-data-retention'), + call('service-????????-????-????-????-????????????-template-folders'), + call('service-????????-????-????-????-????????????-returned-letters-statistics'), + call('service-????????-????-????-????-????????????-returned-letters-summary'), + ], 'Removed 3 service objects from redis'), ('organisation', [ call('organisations'), call('domains'), @@ -765,7 +774,7 @@ def test_clear_cache_submits_and_tells_you_how_many_things_were_deleted( expected_confirmation, ): redis = mocker.patch('app.main.views.platform_admin.redis_client') - redis.delete_cache_keys_by_pattern.side_effect = [0, 3, 1] + redis.delete_cache_keys_by_pattern.side_effect = [0, 3, 1, 0, 0, 0, 0, 0] client_request.login(platform_admin_user) page = client_request.post('main.clear_cache', _data={'model_type': model_type}, _expected_status=200)