From 1c02476ee7b3d66cf4db4f9b8616918084caeeda Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 12 Mar 2020 16:13:18 +0000 Subject: [PATCH] Let users upload a contact list to use later MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We increasingly have teams wanting to do business-continuity type messaging. They might be without access to their normal systems, which is where they would otherwise go to get the list of email addresses or phone numbers. So we want to give them a place in Notify where they can store their spreadsheets and use them at a later date. For the initial pass we’re going to scope this to only allowing spreadsheets with one column, ie just phone numbers/email addresses. This is because: - it minimises the amount of personal info we’re storing - it reduces the chance of getting a placeholder error when you go to send the message, which is probably a high-stress situation where you might not be able to re-generate the file The code for this is mostly copied from the existing upload CSV journey. It’s quite duplicative, but that’s what I needed to do to get this out quickly. There are opportunities for refactoring later. Similarly, I would have liked to split this up into better commit messages, but it really was a case of just bashing code out until it worked 😳 This commit does not: - implement the ‘view a contact list page’ (it just has a placeholder because the API isn’t ready at the moment) - link to this page (because it’s not ready to use yet) --- app/__init__.py | 2 + app/main/views/uploads.py | 159 ++++++- app/models/contact_list.py | 24 ++ app/models/service.py | 4 + app/navigation.py | 16 + app/notify_client/contact_list_api_client.py | 28 ++ app/s3_client/s3_csv_client.py | 10 + .../uploads/contact-list/column-errors.html | 91 ++++ .../views/uploads/contact-list/ok.html | 70 +++ .../uploads/contact-list/row-errors.html | 108 +++++ .../contact-list/too-many-columns.html | 112 +++++ .../views/uploads/contact-list/upload.html | 84 ++++ tests/app/main/views/test_uploads.py | 402 +++++++++++++++++- tests/conftest.py | 23 + 14 files changed, 1131 insertions(+), 2 deletions(-) create mode 100644 app/models/contact_list.py create mode 100644 app/notify_client/contact_list_api_client.py create mode 100644 app/templates/views/uploads/contact-list/column-errors.html create mode 100644 app/templates/views/uploads/contact-list/ok.html create mode 100644 app/templates/views/uploads/contact-list/row-errors.html create mode 100644 app/templates/views/uploads/contact-list/too-many-columns.html create mode 100644 app/templates/views/uploads/contact-list/upload.html diff --git a/app/__init__.py b/app/__init__.py index 625a697b0..79c315225 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -64,6 +64,7 @@ from app.notify_client import InviteTokenError from app.notify_client.api_key_api_client import api_key_api_client from app.notify_client.billing_api_client import billing_api_client from app.notify_client.complaint_api_client import complaint_api_client +from app.notify_client.contact_list_api_client import contact_list_api_client from app.notify_client.email_branding_client import email_branding_client from app.notify_client.events_api_client import events_api_client from app.notify_client.inbound_number_client import inbound_number_client @@ -140,6 +141,7 @@ def create_app(application): # API clients api_key_api_client, billing_api_client, + contact_list_api_client, complaint_api_client, email_branding_client, events_api_client, diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index a6b32f28d..ea9fc5770 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -1,25 +1,38 @@ import base64 +import itertools import json import urllib import uuid from io import BytesIO +from zipfile import BadZipFile from flask import ( abort, current_app, + flash, redirect, render_template, request, url_for, ) +from notifications_utils.columns import Columns from notifications_utils.pdf import pdf_page_count +from notifications_utils.recipients import RecipientCSV +from notifications_utils.sanitise_text import SanitiseASCII from PyPDF2.utils import PdfReadError from requests import RequestException +from xlrd.biffh import XLRDError +from xlrd.xldate import XLDateError from app import current_service, notification_api_client, service_api_client from app.extensions import antivirus_client from app.main import main -from app.main.forms import LetterUploadPostageForm, PDFUploadForm +from app.main.forms import CsvUploadForm, LetterUploadPostageForm, PDFUploadForm +from app.s3_client.s3_csv_client import ( + s3download, + s3upload, + set_metadata_on_csv_upload, +) from app.s3_client.s3_letter_upload_client import ( get_letter_metadata, get_letter_pdf_and_metadata, @@ -28,10 +41,13 @@ from app.s3_client.s3_letter_upload_client import ( ) from app.template_previews import TemplatePreview, sanitise_letter from app.utils import ( + Spreadsheet, generate_next_dict, generate_previous_dict, + get_errors_for_csv, get_letter_validation_error, get_template, + unicode_truncate, user_has_permissions, ) @@ -282,3 +298,144 @@ def send_uploaded_letter(service_id): service_id=service_id, notification_id=file_id, )) + + +@main.route("/services//upload-a-contact-list", methods=['GET', 'POST']) +@user_has_permissions('send_messages') +def upload_contact_list(service_id): + form = CsvUploadForm() + + if form.validate_on_submit(): + try: + upload_id = s3upload( + service_id, + Spreadsheet.from_file(form.file.data, filename=form.file.data.filename).as_dict, + current_app.config['AWS_REGION'], + ) + return redirect(url_for( + '.check_contact_list', + service_id=service_id, + upload_id=upload_id, + original_file_name=form.file.data.filename, + )) + except (UnicodeDecodeError, BadZipFile, XLRDError): + flash('Could not read {}. Try using a different file format.'.format( + form.file.data.filename + )) + except (XLDateError): + flash(( + '{} contains numbers or dates that Notify cannot understand. ' + 'Try formatting all columns as ‘text’ or export your file as CSV.' + ).format( + form.file.data.filename + )) + + return render_template( + 'views/uploads/contact-list/upload.html', + form=form, + ) + + +@main.route( + "/services//check-contact-list/", + methods=['GET', 'POST'], +) +@user_has_permissions('send_messages') +def check_contact_list(service_id, upload_id): + + form = CsvUploadForm() + + contents = s3download(service_id, upload_id).strip() + first_row = contents.splitlines()[0].strip().rstrip(',') if contents else '' + + template_type = { + 'emailaddress': 'email', + 'phonenumber': 'sms', + }.get(Columns.make_key(first_row)) + + original_file_name = SanitiseASCII.encode(request.args.get('original_file_name', '')) + + recipients = RecipientCSV( + contents, + template_type=template_type or 'sms', + whitelist=itertools.chain.from_iterable( + [user.name, user.mobile_number, user.email_address] + for user in current_service.active_users + ) if current_service.trial_mode else None, + international_sms=current_service.has_permission('international_sms'), + max_initial_rows_shown=50, + max_errors_shown=50, + ) + + non_empty_column_headers = list(filter(None, recipients.column_headers)) + + if len(non_empty_column_headers) > 1 or not template_type or not recipients: + return render_template( + 'views/uploads/contact-list/too-many-columns.html', + recipients=recipients, + original_file_name=original_file_name, + template_type=template_type, + form=form, + ) + + if recipients.too_many_rows or not len(recipients): + return render_template( + 'views/uploads/contact-list/column-errors.html', + recipients=recipients, + original_file_name=original_file_name, + form=form, + ) + + row_errors = get_errors_for_csv(recipients, template_type) + if row_errors: + return render_template( + 'views/uploads/contact-list/row-errors.html', + recipients=recipients, + original_file_name=original_file_name, + row_errors=row_errors, + form=form, + ) + + if recipients.has_errors: + return render_template( + 'views/uploads/contact-list/column-errors.html', + recipients=recipients, + original_file_name=original_file_name, + form=form, + ) + + metadata_kwargs = { + 'row_count': len(recipients), + 'valid': True, + 'original_file_name': unicode_truncate( + original_file_name, + 1600, + ), + 'template_type': template_type + } + + set_metadata_on_csv_upload(service_id, upload_id, **metadata_kwargs) + + return render_template( + 'views/uploads/contact-list/ok.html', + recipients=recipients, + original_file_name=original_file_name, + upload_id=upload_id, + ) + + +@main.route("/services//save-contact-list/", methods=['POST']) +@user_has_permissions('send_messages') +def save_contact_list(service_id, upload_id): + current_service.save_contact_list(upload_id) + return redirect(url_for( + '.contact_list', + service_id=current_service.id, + contact_list_id=upload_id, + )) + + +@main.route("/services//contact-list/", methods=['GET']) +@user_has_permissions('send_messages') +def contact_list(service_id, contact_list_id): + return 'page for contact list {}'.format(contact_list_id) diff --git a/app/models/contact_list.py b/app/models/contact_list.py new file mode 100644 index 000000000..67cabc6d9 --- /dev/null +++ b/app/models/contact_list.py @@ -0,0 +1,24 @@ +from flask import abort + +from app.models import JSONModel +from app.notify_client.contact_list_api_client import contact_list_api_client +from app.s3_client.s3_csv_client import get_csv_metadata + + +class ContactList(JSONModel): + + @classmethod + def create(cls, service_id, upload_id): + + metadata = get_csv_metadata(service_id, upload_id) + + if not metadata.get('valid'): + abort(403) + + return cls(contact_list_api_client.create_contact_list( + service_id=service_id, + upload_id=upload_id, + original_file_name=metadata['original_file_name'], + row_count=int(metadata['row_count']), + template_type=metadata['template_type'], + )) diff --git a/app/models/service.py b/app/models/service.py index 132ac157c..6b883724b 100644 --- a/app/models/service.py +++ b/app/models/service.py @@ -6,6 +6,7 @@ from notifications_utils.timezones import local_timezone from werkzeug.utils import cached_property from app.models import JSONModel +from app.models.contact_list import ContactList from app.models.job import ( ImmediateJobs, PaginatedJobs, @@ -132,6 +133,9 @@ class Service(JSONModel): return [] return ScheduledJobs(self.id) + def save_contact_list(self, upload_id): + return ContactList.create(self.id, upload_id) + @cached_property def invited_users(self): return InvitedUsers(self.id) diff --git a/app/navigation.py b/app/navigation.py index aae89b581..491d3989a 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -317,6 +317,10 @@ class HeaderNavigation(Navigation): 'template_history', 'template_usage', 'trial_mode', + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -385,6 +389,10 @@ class MainNavigation(Navigation): 'view_template_versions', }, 'uploads': { + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -661,6 +669,10 @@ class CaseworkNavigation(Navigation): 'uploads': { 'view_jobs', 'view_job', + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -1195,6 +1207,10 @@ class OrgNavigation(Navigation): 'two_factor_email_sent', 'update_email_branding', 'update_letter_branding', + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', diff --git a/app/notify_client/contact_list_api_client.py b/app/notify_client/contact_list_api_client.py new file mode 100644 index 000000000..67f55950f --- /dev/null +++ b/app/notify_client/contact_list_api_client.py @@ -0,0 +1,28 @@ +from app.notify_client import NotifyAdminAPIClient, _attach_current_user + + +class ContactListApiClient(NotifyAdminAPIClient): + + def create_contact_list( + self, + *, + service_id, + upload_id, + original_file_name, + row_count, + template_type, + ): + data = { + "id": upload_id, + "original_file_name": original_file_name, + "row_count": row_count, + "template_type": template_type, + } + + data = _attach_current_user(data) + job = self.post(url='/service/{}/contact-list'.format(service_id), data=data) + + return job + + +contact_list_api_client = ContactListApiClient() diff --git a/app/s3_client/s3_csv_client.py b/app/s3_client/s3_csv_client.py index f9d0b5b9f..7999426e1 100644 --- a/app/s3_client/s3_csv_client.py +++ b/app/s3_client/s3_csv_client.py @@ -55,3 +55,13 @@ def set_metadata_on_csv_upload(service_id, upload_id, **kwargs): }, MetadataDirective='REPLACE', ) + + +def get_csv_metadata(service_id, upload_id): + try: + key = get_csv_upload(service_id, upload_id) + return key.get()['Metadata'] + except botocore.exceptions.ClientError as e: + current_app.logger.error("Unable to download s3 file {}".format( + FILE_LOCATION_STRUCTURE.format(service_id, upload_id))) + raise e diff --git a/app/templates/views/uploads/contact-list/column-errors.html b/app/templates/views/uploads/contact-list/column-errors.html new file mode 100644 index 000000000..34dd9b13e --- /dev/null +++ b/app/templates/views/uploads/contact-list/column-errors.html @@ -0,0 +1,91 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/back-link/macro.njk" import govukBackLink %} +{% from "components/message-count-label.html" import message_count_label, recipient_count_label %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

+ Skip to file contents +

+{% endmacro %} + +{% block service_page_title %} + Error +{% endblock %} + +{% block maincolumn_content %} + + {{ govukBackLink({ "href": url_for('main.upload_contact_list', service_id=current_service.id) }) }} + +
+ {% call banner_wrapper(type='dangerous') %} + + {% if recipients.too_many_rows %} + +

+ Your file has too many rows +

+

+ Notify can store files up to + {{ "{:,}".format(recipients.max_rows) }} rows in size. Your + file has {{ "{:,}".format(recipients|length) }} rows. +

+ + {% elif not recipients.allowed_to_send_to %} + +

+ You cannot save + {{ 'this' if recipients|length == 1 else 'these' }} + {{ recipient_count_label(recipients|length, recipients.template_type) }} +

+

+ In trial mode you can only + send to yourself and members of your team +

+ + {% endif %} + + {{ skip_to_file_contents() }} + + {% endcall %} +
+ + +
+
+ {{ file_upload( + form.file, + action=url_for('.upload_contact_list', service_id=current_service.id), + button_text='Upload your file again' + ) }} +
+ Back to top +
+ +

{{ original_file_name }}

+ + {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients.column_headers + ) %} + {{ index_field(item.index + 2) }} + {% for column in recipients.column_headers %} + {{ text_field(item[column].data or '') }} + {% endfor %} + {% endcall %} + + {% if recipients.displayed_rows|list|length < recipients|length %} +

+ Only showing the first {{ recipients.displayed_rows|list|length }} rows +

+ {% endif %} + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/ok.html b/app/templates/views/uploads/contact-list/ok.html new file mode 100644 index 000000000..0dfa68a4a --- /dev/null +++ b/app/templates/views/uploads/contact-list/ok.html @@ -0,0 +1,70 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/page-header.html" import page_header %} +{% from "components/message-count-label.html" import message_count_label, recipient_count_label %} +{% from "components/button/macro.njk" import govukButton %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

+ Skip to file contents +

+{% endmacro %} + +{% block service_page_title %} + {{ original_file_name }} +{% endblock %} + +{% block maincolumn_content %} + + {{ page_header( + original_file_name, + back_link=url_for('main.upload_contact_list', service_id=current_service.id) + ) }} + +

+ {{ recipients|length|format_thousands }} {{ recipient_count_label(recipients|length, recipients.template_type) }} found +

+ +
+
+ + {{ govukButton({ "text": "Save contact list" }) }} +
+
+ +

+ File preview +

+ {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients.column_headers + ) %} + {{ index_field(item.index + 2) }} + {% for column in recipients.column_headers %} + {% if item[column].ignore %} + {{ text_field(item[column].data or '', status='default') }} + {% else %} + {{ text_field(item[column].data or '') }} + {% endif %} + {% endfor %} + {% if item[None].data %} + {% for column in item[None].data %} + {{ text_field(column, status='default') }} + {% endfor %} + {% endif %} + {% endcall %} + + {% if recipients.displayed_rows|list|length < recipients|length %} +

+ Only showing the first {{ recipients.displayed_rows|list|length }} rows +

+ {% endif %} + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/row-errors.html b/app/templates/views/uploads/contact-list/row-errors.html new file mode 100644 index 000000000..c164a5ad1 --- /dev/null +++ b/app/templates/views/uploads/contact-list/row-errors.html @@ -0,0 +1,108 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/back-link/macro.njk" import govukBackLink %} +{% from "components/message-count-label.html" import message_count_label %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

+ Skip to file contents +

+{% endmacro %} + +{% block service_page_title %} + Error +{% endblock %} + +{% block maincolumn_content %} + + {{ govukBackLink({ "href": url_for('main.upload_contact_list', service_id=current_service.id) }) }} + +
+ {% call banner_wrapper(type='dangerous') %} + {% if row_errors|length == 1 %} +

+ There’s a problem with {{ original_file_name }} +

+

+ You need to {{ row_errors[0] }}. +

+ {% else %} +

+ There are some problems with {{ original_file_name }} +

+

+ You need to: +

+
    + {% for error in row_errors %} +
  • {{ error }}
    • + {% endfor %} +
+ {% endif %} + {{ skip_to_file_contents() }} + {% endcall %} +
+ +
+
+ {{ file_upload( + form.file, + action=url_for('.upload_contact_list', service_id=current_service.id), + button_text='Upload your file again' + ) }} +
+ Back to top +
+ + {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients.column_headers + ) %} + {% call index_field() %} + + {{ item.index + 2 }} + + {% endcall %} + {% for column in recipients.column_headers %} + {% if item[column].error and not recipients.missing_column_headers %} + {% call field() %} + + {{ item[column].error }} + {{ item[column].data if item[column].data != None }} + + {% endcall %} + {% elif item[column].ignore %} + {{ text_field(item[column].data or '', status='default') }} + {% else %} + {{ text_field(item[column].data or '') }} + {% endif %} + {% endfor %} + {% if item[None].data %} + {% for column in item[None].data %} + {{ text_field(column, status='default') }} + {% endfor %} + {% endif %} + {% endcall %} + + {% if recipients.displayed_rows|list|length < recipients|length %} + {% if recipients.displayed_rows|list|length < recipients.rows_with_errors|list|length %} +

+ Only showing the first {{ recipients.displayed_rows|list|length }} rows with errors +

+ {% else %} +

+ Only showing rows with errors +

+ {% endif %} + {% endif %} + + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/too-many-columns.html b/app/templates/views/uploads/contact-list/too-many-columns.html new file mode 100644 index 000000000..41ea59a86 --- /dev/null +++ b/app/templates/views/uploads/contact-list/too-many-columns.html @@ -0,0 +1,112 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/back-link/macro.njk" import govukBackLink %} +{% from "components/message-count-label.html" import message_count_label %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

+ Skip to file contents +

+{% endmacro %} + +{% block service_page_title %} + Error +{% endblock %} + +{% block maincolumn_content %} + + {{ govukBackLink({ "href": url_for('main.upload_contact_list', service_id=current_service.id) }) }} + +
+ {% call banner_wrapper(type='dangerous') %} + + {% if not recipients|length %} + +

+ Your file is missing some rows +

+

+ It needs at least one row of data + {%- if template_type %}.{% else %}, in a column called ‘email address’ or ‘phone number’.{% endif %} +

+ + {% elif recipients.column_headers|length == 1 %} + +

+ Your file needs a column called ‘email address’ or ‘phone number’. +

+

+ Right now it has 1 column called ‘{{ recipients._raw_column_headers[0] }}’. +

+ + {% else %} + +

+ Your file has too many columns +

+

+ It needs to have 1 column, called ‘email address’ or ‘phone number’. +

+

+ Right now it has {{ recipients._raw_column_headers|length }} columns called {{ recipients._raw_column_headers | formatted_list }}. +

+ + {% endif %} + + {{ skip_to_file_contents() }} + + {% endcall %} +
+ + +
+
+ {{ file_upload( + form.file, + action=url_for('.upload_contact_list', service_id=current_service.id), + button_text='Upload your file again' + ) }} +
+ Back to top +
+ + {% set column_headers = recipients._raw_column_headers if recipients.duplicate_recipient_column_headers else recipients.column_headers %} + +

{{ original_file_name }}

+ +
+ {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients._raw_column_headers + ) %} + {{ index_field(item.index + 2) }} + {% for column in column_headers %} + {% if item[column].ignore %} + {{ text_field(item[column].data or '', status='default') }} + {% else %} + {{ text_field(item[column].data or '') }} + {% endif %} + {% endfor %} + {% if item[None].data %} + {% for column in item[None].data %} + {{ text_field(column, status='default') }} + {% endfor %} + {% endif %} + {% endcall %} +
+ + {% if recipients.displayed_rows|list|length < recipients|length %} +

+ Only showing the first {{ recipients.displayed_rows|list|length }} rows +

+ {% endif %} + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/upload.html b/app/templates/views/uploads/contact-list/upload.html new file mode 100644 index 000000000..7b6355a38 --- /dev/null +++ b/app/templates/views/uploads/contact-list/upload.html @@ -0,0 +1,84 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/page-header.html" import page_header %} +{% from "components/table.html" import list_table, text_field, index_field, index_field_heading %} + +{% block service_page_title %} + Upload an emergency contact list +{% endblock %} + +{% block maincolumn_content %} + {% if error %} + {% call banner_wrapper(type='dangerous') %} +

{{ error.title }}

+ {% if error.detail %} +

{{ error.detail | safe }}

+ {% endif %} + {% endcall %} + {% else %} + {{ page_header( + 'Upload an emergency contact list', + back_link=url_for('main.uploads', service_id=current_service.id) + ) }} +

Upload a list of phone numbers or email addresses.

+

Don’t put members of the public in here.

+ {% endif %} + +
+ {{ file_upload( + form.file, + button_text='Upload your file again' if error else 'Choose file', + show_errors=False + )}} +
+ +

Your file needs to look like one of these examples

+ +

+ Save your file as a + CSV, + TSV, + ODS, + or Microsoft Excel spreadsheet +

+ +
+
+
+ {% call(item, row_number) list_table( + [ + ['email address'], + ['test@example.gov.uk'], + ], + caption="Example", + caption_visible=False, + field_headings=['', 'A'] + ) %} + {{ index_field(row_number - 1) }} + {% for column in item %} + {{ text_field(column) }} + {% endfor %} + {% endcall %} +
+
+
+
+ {% call(item, row_number) list_table( + [ + ['phone number'], + ['07700 900123'], + ], + caption="Example", + caption_visible=False, + field_headings=['', 'A'] + ) %} + {{ index_field(row_number - 1) }} + {% for column in item %} + {{ text_field(column) }} + {% endfor %} + {% endcall %} +
+
+
+{% endblock %} diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 2033c5852..fa9b12d9d 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -1,6 +1,7 @@ import re import urllib -from unittest.mock import Mock +from io import BytesIO +from unittest.mock import ANY, Mock import pytest from flask import make_response, url_for @@ -697,3 +698,402 @@ def test_get_uploads_shows_pagination( 'Previous page ' 'page 0' ) + + +def test_upload_contact_list_page(client_request): + page = client_request.get( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + ) + assert 'action' not in page.select_one('form') + assert page.select_one('form input')['name'] == 'file' + assert page.select_one('form input')['type'] == 'file' + + assert normalize_spaces(page.select('.spreadsheet')[0].text) == ( + 'Example A ' + '1 email address ' + '2 test@example.gov.uk' + ) + assert normalize_spaces(page.select('.spreadsheet')[1].text) == ( + 'Example A ' + '1 phone number ' + '2 07700 900123' + ) + + +@pytest.mark.parametrize('file_contents, expected_error, expected_thead, expected_tbody,', [ + ( + """ + telephone,name + +447700900986 + """, + ( + 'Your file has too many columns ' + 'It needs to have 1 column, called ‘email address’ or ‘phone number’. ' + 'Right now it has 2 columns called ‘telephone’ and ‘name’. ' + 'Skip to file contents' + ), + 'Row in file 1 telephone name', + '2 +447700900986', + ), + ( + """ + phone number, email address + +447700900986, test@example.com + """, + ( + 'Your file has too many columns ' + 'It needs to have 1 column, called ‘email address’ or ‘phone number’. ' + 'Right now it has 2 columns called ‘phone number’ and ‘email address’. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number email address', + '2 +447700900986 test@example.com', + ), + ( + """ + email address + +447700900986 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 1 email address. ' + 'Skip to file contents' + ), + 'Row in file 1 email address', + '2 Not a valid email address +447700900986', + ), + ( + """ + phone number + test@example.com + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 1 phone number. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + '2 Must not contain letters or symbols test@example.com', + ), + ( + """ + phone number, phone number, PHONE_NUMBER + +447700900111,+447700900222,+447700900333, + """, + ( + 'Your file has too many columns ' + 'It needs to have 1 column, called ‘email address’ or ‘phone number’. ' + 'Right now it has 3 columns called ‘phone number’, ‘phone number’ and ‘PHONE_NUMBER’. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number phone number PHONE_NUMBER', + '2 +447700900333 +447700900333 +447700900333', + ), + ( + """ + phone number + """, + ( + 'Your file is missing some rows ' + 'It needs at least one row of data. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + '', + ), + ( + "+447700900986", + ( + 'Your file is missing some rows ' + 'It needs at least one row of data, in a column called ' + '‘email address’ or ‘phone number’. ' + 'Skip to file contents' + ), + 'Row in file 1 +447700900986', + '', + ), + ( + "", + ( + 'Your file is missing some rows ' + 'It needs at least one row of data, in a column called ' + '‘email address’ or ‘phone number’. ' + 'Skip to file contents' + ), + 'Row in file 1', + '', + ), + ( + """ + phone number + +447700900986 + + +447700900986 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to enter missing data in 1 row. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + ( + '3 Missing' + ) + ), + ( + """ + phone number + +447700900 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 1 phone number. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + '2 Not enough digits +447700900', + ), + ( + """ + email address + ok@example.com + bad@example1 + bad@example2 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 2 email addresses. ' + 'Skip to file contents' + ), + 'Row in file 1 email address', + ( + '3 Not a valid email address bad@example1 ' + '4 Not a valid email address bad@example2' + ), + ), +]) +def test_upload_csv_file_shows_error_banner( + client_request, + mocker, + mock_s3_upload, + mock_get_job_doesnt_exist, + mock_get_users_by_service, + fake_uuid, + file_contents, + expected_error, + expected_thead, + expected_tbody, +): + + mock_upload = mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) + mock_download = mocker.patch('app.main.views.uploads.s3download', return_value=file_contents) + + page = client_request.post( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + _data={'file': (BytesIO(''.encode('utf-8')), 'invalid.csv')}, + _follow_redirects=True, + ) + mock_upload.assert_called_once_with( + SERVICE_ONE_ID, {'data': '', 'file_name': 'invalid.csv'}, ANY, + ) + mock_download.assert_called_once_with(SERVICE_ONE_ID, fake_uuid) + + assert normalize_spaces(page.select_one('.banner-dangerous').text) == expected_error + + assert page.select_one('form')['action'] == url_for( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + ) + assert page.select_one('form input')['type'] == 'file' + + assert normalize_spaces(page.select_one('thead').text) == expected_thead + assert normalize_spaces(page.select_one('tbody').text) == expected_tbody + + +def test_upload_csv_file_shows_error_banner_for_too_many_rows( + client_request, + mocker, + mock_s3_upload, + mock_get_job_doesnt_exist, + mock_get_users_by_service, + fake_uuid, +): + + mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) + mocker.patch('app.main.views.uploads.s3download', return_value='\n'.join( + ['phone number'] + (['07700900986'] * 50001) + )) + + page = client_request.post( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + _data={'file': (BytesIO(''.encode('utf-8')), 'invalid.csv')}, + _follow_redirects=True, + ) + + assert normalize_spaces(page.select_one('.banner-dangerous').text) == ( + 'Your file has too many rows ' + 'Notify can store files up to 50,000 rows in size. ' + 'Your file has 50,001 rows. ' + 'Skip to file contents' + ) + assert len(page.select('tbody tr')) == 50 + assert normalize_spaces(page.select_one('.table-show-more-link').text) == ( + 'Only showing the first 50 rows' + ) + + +def test_upload_csv_shows_trial_mode_error( + client_request, + mock_get_users_by_service, + mock_get_job_doesnt_exist, + fake_uuid, + mocker +): + mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) + mocker.patch('app.main.views.uploads.s3download', return_value=( + 'phone number\n' + '07900900321' # Not in team + )) + + page = client_request.get( + 'main.check_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + _test_page_title=False, + ) + + assert normalize_spaces(page.select_one('.banner-dangerous').text) == ( + 'You cannot save this phone number ' + 'In trial mode you can only send to yourself and members of your team ' + 'Skip to file contents' + ) + assert page.select_one('.banner-dangerous a')['href'] == url_for( + 'main.trial_mode_new' + ) + + +def test_upload_csv_shows_ok_page( + client_request, + mock_get_live_service, + mock_get_users_by_service, + mock_get_job_doesnt_exist, + fake_uuid, + mocker +): + mocker.patch('app.main.views.uploads.s3download', return_value='\n'.join( + ['email address'] + ['test@example.com'] * 51 + )) + mock_metadata_set = mocker.patch('app.main.views.uploads.set_metadata_on_csv_upload') + + page = client_request.get( + 'main.check_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + original_file_name='good times.xlsx', + _test_page_title=False, + ) + + mock_metadata_set.assert_called_once_with( + SERVICE_ONE_ID, + fake_uuid, + row_count=51, + original_file_name='good times.xlsx', + template_type='email', + valid=True, + ) + + assert normalize_spaces(page.select_one('h1').text) == ( + 'good times.xlsx' + ) + assert normalize_spaces(page.select_one('main p').text) == ( + '51 email addresses found' + ) + assert page.select_one('form')['action'] == url_for( + 'main.save_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + ) + + assert normalize_spaces(page.select_one('form [type=submit]').text) == ( + 'Save contact list' + ) + assert normalize_spaces(page.select_one('thead').text) == ( + 'Row in file 1 email address' + ) + assert len(page.select('tbody tr')) == 50 + assert normalize_spaces(page.select_one('tbody tr').text) == ( + '2 test@example.com' + ) + assert normalize_spaces(page.select_one('.table-show-more-link').text) == ( + 'Only showing the first 50 rows' + ) + + +def test_save_contact_list( + mocker, + client_request, + fake_uuid, + mock_create_contact_list, +): + mocker.patch('app.models.contact_list.get_csv_metadata', return_value={ + 'row_count': 999, + 'valid': True, + 'original_file_name': 'example.csv', + 'template_type': 'email' + }) + client_request.post( + 'main.save_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + _expected_status=302, + _expected_redirect=url_for( + 'main.contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + _external=True, + ) + ) + mock_create_contact_list.assert_called_once_with( + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + original_file_name='example.csv', + row_count=999, + template_type='email', + ) + + +def test_cant_save_bad_contact_list( + mocker, + client_request, + fake_uuid, + mock_create_contact_list, +): + mocker.patch('app.models.contact_list.get_csv_metadata', return_value={ + 'row_count': 999, + 'valid': False, + 'original_file_name': 'example.csv', + 'template_type': 'email' + }) + client_request.post( + 'main.save_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + _expected_status=403, + ) + assert mock_create_contact_list.called is False + + +def test_view_contact_list( + client_request, + fake_uuid, +): + page = client_request.get( + 'main.contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + _test_page_title=False, + ) + assert page.text == 'page for contact list 6ce466d0-fd6a-11e5-82f5-e0accb9d11a6' diff --git a/tests/conftest.py b/tests/conftest.py index f01c9150c..d44aa12eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1819,6 +1819,29 @@ def mock_get_no_uploads(mocker, api_user_active): ) +@pytest.fixture(scope='function') +def mock_create_contact_list(mocker, api_user_active): + def _create( + service_id, + upload_id, + original_file_name, + row_count, + template_type, + ): + return { + 'service_id': service_id, + 'upload_id': upload_id, + 'original_file_name': original_file_name, + 'row_count': row_count, + 'template_type': template_type, + } + + return mocker.patch( + 'app.contact_list_api_client.create_contact_list', + side_effect=_create, + ) + + @pytest.fixture(scope='function') def mock_get_notifications( mocker,