diff --git a/app/__init__.py b/app/__init__.py index 625a697b0..79c315225 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -64,6 +64,7 @@ from app.notify_client import InviteTokenError from app.notify_client.api_key_api_client import api_key_api_client from app.notify_client.billing_api_client import billing_api_client from app.notify_client.complaint_api_client import complaint_api_client +from app.notify_client.contact_list_api_client import contact_list_api_client from app.notify_client.email_branding_client import email_branding_client from app.notify_client.events_api_client import events_api_client from app.notify_client.inbound_number_client import inbound_number_client @@ -140,6 +141,7 @@ def create_app(application): # API clients api_key_api_client, billing_api_client, + contact_list_api_client, complaint_api_client, email_branding_client, events_api_client, diff --git a/app/main/views/uploads.py b/app/main/views/uploads.py index a6b32f28d..ea9fc5770 100644 --- a/app/main/views/uploads.py +++ b/app/main/views/uploads.py @@ -1,25 +1,38 @@ import base64 +import itertools import json import urllib import uuid from io import BytesIO +from zipfile import BadZipFile from flask import ( abort, current_app, + flash, redirect, render_template, request, url_for, ) +from notifications_utils.columns import Columns from notifications_utils.pdf import pdf_page_count +from notifications_utils.recipients import RecipientCSV +from notifications_utils.sanitise_text import SanitiseASCII from PyPDF2.utils import PdfReadError from requests import RequestException +from xlrd.biffh import XLRDError +from xlrd.xldate import XLDateError from app import current_service, notification_api_client, service_api_client from app.extensions import antivirus_client from app.main import main -from app.main.forms import LetterUploadPostageForm, PDFUploadForm +from app.main.forms import CsvUploadForm, LetterUploadPostageForm, PDFUploadForm +from app.s3_client.s3_csv_client import ( + s3download, + s3upload, + set_metadata_on_csv_upload, +) from app.s3_client.s3_letter_upload_client import ( get_letter_metadata, get_letter_pdf_and_metadata, @@ -28,10 +41,13 @@ from app.s3_client.s3_letter_upload_client import ( ) from app.template_previews import TemplatePreview, sanitise_letter from app.utils import ( + Spreadsheet, generate_next_dict, generate_previous_dict, + get_errors_for_csv, get_letter_validation_error, get_template, + unicode_truncate, user_has_permissions, ) @@ -282,3 +298,144 @@ def send_uploaded_letter(service_id): service_id=service_id, notification_id=file_id, )) + + +@main.route("/services//upload-a-contact-list", methods=['GET', 'POST']) +@user_has_permissions('send_messages') +def upload_contact_list(service_id): + form = CsvUploadForm() + + if form.validate_on_submit(): + try: + upload_id = s3upload( + service_id, + Spreadsheet.from_file(form.file.data, filename=form.file.data.filename).as_dict, + current_app.config['AWS_REGION'], + ) + return redirect(url_for( + '.check_contact_list', + service_id=service_id, + upload_id=upload_id, + original_file_name=form.file.data.filename, + )) + except (UnicodeDecodeError, BadZipFile, XLRDError): + flash('Could not read {}. Try using a different file format.'.format( + form.file.data.filename + )) + except (XLDateError): + flash(( + '{} contains numbers or dates that Notify cannot understand. ' + 'Try formatting all columns as ‘text’ or export your file as CSV.' + ).format( + form.file.data.filename + )) + + return render_template( + 'views/uploads/contact-list/upload.html', + form=form, + ) + + +@main.route( + "/services//check-contact-list/", + methods=['GET', 'POST'], +) +@user_has_permissions('send_messages') +def check_contact_list(service_id, upload_id): + + form = CsvUploadForm() + + contents = s3download(service_id, upload_id).strip() + first_row = contents.splitlines()[0].strip().rstrip(',') if contents else '' + + template_type = { + 'emailaddress': 'email', + 'phonenumber': 'sms', + }.get(Columns.make_key(first_row)) + + original_file_name = SanitiseASCII.encode(request.args.get('original_file_name', '')) + + recipients = RecipientCSV( + contents, + template_type=template_type or 'sms', + whitelist=itertools.chain.from_iterable( + [user.name, user.mobile_number, user.email_address] + for user in current_service.active_users + ) if current_service.trial_mode else None, + international_sms=current_service.has_permission('international_sms'), + max_initial_rows_shown=50, + max_errors_shown=50, + ) + + non_empty_column_headers = list(filter(None, recipients.column_headers)) + + if len(non_empty_column_headers) > 1 or not template_type or not recipients: + return render_template( + 'views/uploads/contact-list/too-many-columns.html', + recipients=recipients, + original_file_name=original_file_name, + template_type=template_type, + form=form, + ) + + if recipients.too_many_rows or not len(recipients): + return render_template( + 'views/uploads/contact-list/column-errors.html', + recipients=recipients, + original_file_name=original_file_name, + form=form, + ) + + row_errors = get_errors_for_csv(recipients, template_type) + if row_errors: + return render_template( + 'views/uploads/contact-list/row-errors.html', + recipients=recipients, + original_file_name=original_file_name, + row_errors=row_errors, + form=form, + ) + + if recipients.has_errors: + return render_template( + 'views/uploads/contact-list/column-errors.html', + recipients=recipients, + original_file_name=original_file_name, + form=form, + ) + + metadata_kwargs = { + 'row_count': len(recipients), + 'valid': True, + 'original_file_name': unicode_truncate( + original_file_name, + 1600, + ), + 'template_type': template_type + } + + set_metadata_on_csv_upload(service_id, upload_id, **metadata_kwargs) + + return render_template( + 'views/uploads/contact-list/ok.html', + recipients=recipients, + original_file_name=original_file_name, + upload_id=upload_id, + ) + + +@main.route("/services//save-contact-list/", methods=['POST']) +@user_has_permissions('send_messages') +def save_contact_list(service_id, upload_id): + current_service.save_contact_list(upload_id) + return redirect(url_for( + '.contact_list', + service_id=current_service.id, + contact_list_id=upload_id, + )) + + +@main.route("/services//contact-list/", methods=['GET']) +@user_has_permissions('send_messages') +def contact_list(service_id, contact_list_id): + return 'page for contact list {}'.format(contact_list_id) diff --git a/app/models/contact_list.py b/app/models/contact_list.py new file mode 100644 index 000000000..67cabc6d9 --- /dev/null +++ b/app/models/contact_list.py @@ -0,0 +1,24 @@ +from flask import abort + +from app.models import JSONModel +from app.notify_client.contact_list_api_client import contact_list_api_client +from app.s3_client.s3_csv_client import get_csv_metadata + + +class ContactList(JSONModel): + + @classmethod + def create(cls, service_id, upload_id): + + metadata = get_csv_metadata(service_id, upload_id) + + if not metadata.get('valid'): + abort(403) + + return cls(contact_list_api_client.create_contact_list( + service_id=service_id, + upload_id=upload_id, + original_file_name=metadata['original_file_name'], + row_count=int(metadata['row_count']), + template_type=metadata['template_type'], + )) diff --git a/app/models/service.py b/app/models/service.py index 132ac157c..6b883724b 100644 --- a/app/models/service.py +++ b/app/models/service.py @@ -6,6 +6,7 @@ from notifications_utils.timezones import local_timezone from werkzeug.utils import cached_property from app.models import JSONModel +from app.models.contact_list import ContactList from app.models.job import ( ImmediateJobs, PaginatedJobs, @@ -132,6 +133,9 @@ class Service(JSONModel): return [] return ScheduledJobs(self.id) + def save_contact_list(self, upload_id): + return ContactList.create(self.id, upload_id) + @cached_property def invited_users(self): return InvitedUsers(self.id) diff --git a/app/navigation.py b/app/navigation.py index aae89b581..491d3989a 100644 --- a/app/navigation.py +++ b/app/navigation.py @@ -317,6 +317,10 @@ class HeaderNavigation(Navigation): 'template_history', 'template_usage', 'trial_mode', + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -385,6 +389,10 @@ class MainNavigation(Navigation): 'view_template_versions', }, 'uploads': { + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -661,6 +669,10 @@ class CaseworkNavigation(Navigation): 'uploads': { 'view_jobs', 'view_job', + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', @@ -1195,6 +1207,10 @@ class OrgNavigation(Navigation): 'two_factor_email_sent', 'update_email_branding', 'update_letter_branding', + 'upload_contact_list', + 'check_contact_list', + 'save_contact_list', + 'contact_list', 'upload_letter', 'uploaded_letter_preview', 'uploads', diff --git a/app/notify_client/contact_list_api_client.py b/app/notify_client/contact_list_api_client.py new file mode 100644 index 000000000..67f55950f --- /dev/null +++ b/app/notify_client/contact_list_api_client.py @@ -0,0 +1,28 @@ +from app.notify_client import NotifyAdminAPIClient, _attach_current_user + + +class ContactListApiClient(NotifyAdminAPIClient): + + def create_contact_list( + self, + *, + service_id, + upload_id, + original_file_name, + row_count, + template_type, + ): + data = { + "id": upload_id, + "original_file_name": original_file_name, + "row_count": row_count, + "template_type": template_type, + } + + data = _attach_current_user(data) + job = self.post(url='/service/{}/contact-list'.format(service_id), data=data) + + return job + + +contact_list_api_client = ContactListApiClient() diff --git a/app/s3_client/s3_csv_client.py b/app/s3_client/s3_csv_client.py index f9d0b5b9f..7999426e1 100644 --- a/app/s3_client/s3_csv_client.py +++ b/app/s3_client/s3_csv_client.py @@ -55,3 +55,13 @@ def set_metadata_on_csv_upload(service_id, upload_id, **kwargs): }, MetadataDirective='REPLACE', ) + + +def get_csv_metadata(service_id, upload_id): + try: + key = get_csv_upload(service_id, upload_id) + return key.get()['Metadata'] + except botocore.exceptions.ClientError as e: + current_app.logger.error("Unable to download s3 file {}".format( + FILE_LOCATION_STRUCTURE.format(service_id, upload_id))) + raise e diff --git a/app/templates/views/uploads/contact-list/column-errors.html b/app/templates/views/uploads/contact-list/column-errors.html new file mode 100644 index 000000000..34dd9b13e --- /dev/null +++ b/app/templates/views/uploads/contact-list/column-errors.html @@ -0,0 +1,91 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/back-link/macro.njk" import govukBackLink %} +{% from "components/message-count-label.html" import message_count_label, recipient_count_label %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

+ Skip to file contents +

+{% endmacro %} + +{% block service_page_title %} + Error +{% endblock %} + +{% block maincolumn_content %} + + {{ govukBackLink({ "href": url_for('main.upload_contact_list', service_id=current_service.id) }) }} + +
+ {% call banner_wrapper(type='dangerous') %} + + {% if recipients.too_many_rows %} + +

+ Your file has too many rows +

+

+ Notify can store files up to + {{ "{:,}".format(recipients.max_rows) }} rows in size. Your + file has {{ "{:,}".format(recipients|length) }} rows. +

+ + {% elif not recipients.allowed_to_send_to %} + +

+ You cannot save + {{ 'this' if recipients|length == 1 else 'these' }} + {{ recipient_count_label(recipients|length, recipients.template_type) }} +

+

+ In trial mode you can only + send to yourself and members of your team +

+ + {% endif %} + + {{ skip_to_file_contents() }} + + {% endcall %} +
+ + +
+
+ {{ file_upload( + form.file, + action=url_for('.upload_contact_list', service_id=current_service.id), + button_text='Upload your file again' + ) }} +
+ Back to top +
+ +

{{ original_file_name }}

+ + {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients.column_headers + ) %} + {{ index_field(item.index + 2) }} + {% for column in recipients.column_headers %} + {{ text_field(item[column].data or '') }} + {% endfor %} + {% endcall %} + + {% if recipients.displayed_rows|list|length < recipients|length %} + + {% endif %} + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/ok.html b/app/templates/views/uploads/contact-list/ok.html new file mode 100644 index 000000000..0dfa68a4a --- /dev/null +++ b/app/templates/views/uploads/contact-list/ok.html @@ -0,0 +1,70 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/page-header.html" import page_header %} +{% from "components/message-count-label.html" import message_count_label, recipient_count_label %} +{% from "components/button/macro.njk" import govukButton %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

+ Skip to file contents +

+{% endmacro %} + +{% block service_page_title %} + {{ original_file_name }} +{% endblock %} + +{% block maincolumn_content %} + + {{ page_header( + original_file_name, + back_link=url_for('main.upload_contact_list', service_id=current_service.id) + ) }} + +

+ {{ recipients|length|format_thousands }} {{ recipient_count_label(recipients|length, recipients.template_type) }} found +

+ +
+
+ + {{ govukButton({ "text": "Save contact list" }) }} +
+
+ +

+ File preview +

+ {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients.column_headers + ) %} + {{ index_field(item.index + 2) }} + {% for column in recipients.column_headers %} + {% if item[column].ignore %} + {{ text_field(item[column].data or '', status='default') }} + {% else %} + {{ text_field(item[column].data or '') }} + {% endif %} + {% endfor %} + {% if item[None].data %} + {% for column in item[None].data %} + {{ text_field(column, status='default') }} + {% endfor %} + {% endif %} + {% endcall %} + + {% if recipients.displayed_rows|list|length < recipients|length %} + + {% endif %} + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/row-errors.html b/app/templates/views/uploads/contact-list/row-errors.html new file mode 100644 index 000000000..c164a5ad1 --- /dev/null +++ b/app/templates/views/uploads/contact-list/row-errors.html @@ -0,0 +1,108 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/back-link/macro.njk" import govukBackLink %} +{% from "components/message-count-label.html" import message_count_label %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

+ Skip to file contents +

+{% endmacro %} + +{% block service_page_title %} + Error +{% endblock %} + +{% block maincolumn_content %} + + {{ govukBackLink({ "href": url_for('main.upload_contact_list', service_id=current_service.id) }) }} + +
+ {% call banner_wrapper(type='dangerous') %} + {% if row_errors|length == 1 %} +

+ There’s a problem with {{ original_file_name }} +

+

+ You need to {{ row_errors[0] }}. +

+ {% else %} +

+ There are some problems with {{ original_file_name }} +

+

+ You need to: +

+
    + {% for error in row_errors %} +
  • {{ error }}
  • + {% endfor %} +
+ {% endif %} + {{ skip_to_file_contents() }} + {% endcall %} +
+ +
+
+ {{ file_upload( + form.file, + action=url_for('.upload_contact_list', service_id=current_service.id), + button_text='Upload your file again' + ) }} +
+ Back to top +
+ + {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients.column_headers + ) %} + {% call index_field() %} + + {{ item.index + 2 }} + + {% endcall %} + {% for column in recipients.column_headers %} + {% if item[column].error and not recipients.missing_column_headers %} + {% call field() %} + + {{ item[column].error }} + {{ item[column].data if item[column].data != None }} + + {% endcall %} + {% elif item[column].ignore %} + {{ text_field(item[column].data or '', status='default') }} + {% else %} + {{ text_field(item[column].data or '') }} + {% endif %} + {% endfor %} + {% if item[None].data %} + {% for column in item[None].data %} + {{ text_field(column, status='default') }} + {% endfor %} + {% endif %} + {% endcall %} + + {% if recipients.displayed_rows|list|length < recipients|length %} + {% if recipients.displayed_rows|list|length < recipients.rows_with_errors|list|length %} + + {% else %} + + {% endif %} + {% endif %} + + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/too-many-columns.html b/app/templates/views/uploads/contact-list/too-many-columns.html new file mode 100644 index 000000000..41ea59a86 --- /dev/null +++ b/app/templates/views/uploads/contact-list/too-many-columns.html @@ -0,0 +1,112 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/radios.html" import radio_select %} +{% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/back-link/macro.njk" import govukBackLink %} +{% from "components/message-count-label.html" import message_count_label %} + +{% set file_contents_header_id = 'file-preview' %} +{% macro skip_to_file_contents() %} +

+ Skip to file contents +

+{% endmacro %} + +{% block service_page_title %} + Error +{% endblock %} + +{% block maincolumn_content %} + + {{ govukBackLink({ "href": url_for('main.upload_contact_list', service_id=current_service.id) }) }} + +
+ {% call banner_wrapper(type='dangerous') %} + + {% if not recipients|length %} + +

+ Your file is missing some rows +

+

+ It needs at least one row of data + {%- if template_type %}.{% else %}, in a column called ‘email address’ or ‘phone number’.{% endif %} +

+ + {% elif recipients.column_headers|length == 1 %} + +

+ Your file needs a column called ‘email address’ or ‘phone number’. +

+

+ Right now it has 1 column called ‘{{ recipients._raw_column_headers[0] }}’. +

+ + {% else %} + +

+ Your file has too many columns +

+

+ It needs to have 1 column, called ‘email address’ or ‘phone number’. +

+

+ Right now it has {{ recipients._raw_column_headers|length }} columns called {{ recipients._raw_column_headers | formatted_list }}. +

+ + {% endif %} + + {{ skip_to_file_contents() }} + + {% endcall %} +
+ + +
+
+ {{ file_upload( + form.file, + action=url_for('.upload_contact_list', service_id=current_service.id), + button_text='Upload your file again' + ) }} +
+ Back to top +
+ + {% set column_headers = recipients._raw_column_headers if recipients.duplicate_recipient_column_headers else recipients.column_headers %} + +

{{ original_file_name }}

+ +
+ {% call(item, row_number) list_table( + recipients.displayed_rows, + caption=original_file_name, + caption_visible=False, + field_headings=[ + 'Row in file '|safe + ] + recipients._raw_column_headers + ) %} + {{ index_field(item.index + 2) }} + {% for column in column_headers %} + {% if item[column].ignore %} + {{ text_field(item[column].data or '', status='default') }} + {% else %} + {{ text_field(item[column].data or '') }} + {% endif %} + {% endfor %} + {% if item[None].data %} + {% for column in item[None].data %} + {{ text_field(column, status='default') }} + {% endfor %} + {% endif %} + {% endcall %} +
+ + {% if recipients.displayed_rows|list|length < recipients|length %} + + {% endif %} + +{% endblock %} diff --git a/app/templates/views/uploads/contact-list/upload.html b/app/templates/views/uploads/contact-list/upload.html new file mode 100644 index 000000000..7b6355a38 --- /dev/null +++ b/app/templates/views/uploads/contact-list/upload.html @@ -0,0 +1,84 @@ +{% extends "withnav_template.html" %} +{% from "components/banner.html" import banner_wrapper %} +{% from "components/file-upload.html" import file_upload %} +{% from "components/page-header.html" import page_header %} +{% from "components/table.html" import list_table, text_field, index_field, index_field_heading %} + +{% block service_page_title %} + Upload an emergency contact list +{% endblock %} + +{% block maincolumn_content %} + {% if error %} + {% call banner_wrapper(type='dangerous') %} +

{{ error.title }}

+ {% if error.detail %} +

{{ error.detail | safe }}

+ {% endif %} + {% endcall %} + {% else %} + {{ page_header( + 'Upload an emergency contact list', + back_link=url_for('main.uploads', service_id=current_service.id) + ) }} +

Upload a list of phone numbers or email addresses.

+

Don’t put members of the public in here.

+ {% endif %} + +
+ {{ file_upload( + form.file, + button_text='Upload your file again' if error else 'Choose file', + show_errors=False + )}} +
+ +

Your file needs to look like one of these examples

+ +

+ Save your file as a + CSV, + TSV, + ODS, + or Microsoft Excel spreadsheet +

+ +
+
+
+ {% call(item, row_number) list_table( + [ + ['email address'], + ['test@example.gov.uk'], + ], + caption="Example", + caption_visible=False, + field_headings=['', 'A'] + ) %} + {{ index_field(row_number - 1) }} + {% for column in item %} + {{ text_field(column) }} + {% endfor %} + {% endcall %} +
+
+
+
+ {% call(item, row_number) list_table( + [ + ['phone number'], + ['07700 900123'], + ], + caption="Example", + caption_visible=False, + field_headings=['', 'A'] + ) %} + {{ index_field(row_number - 1) }} + {% for column in item %} + {{ text_field(column) }} + {% endfor %} + {% endcall %} +
+
+
+{% endblock %} diff --git a/tests/app/main/views/test_uploads.py b/tests/app/main/views/test_uploads.py index 2033c5852..fa9b12d9d 100644 --- a/tests/app/main/views/test_uploads.py +++ b/tests/app/main/views/test_uploads.py @@ -1,6 +1,7 @@ import re import urllib -from unittest.mock import Mock +from io import BytesIO +from unittest.mock import ANY, Mock import pytest from flask import make_response, url_for @@ -697,3 +698,402 @@ def test_get_uploads_shows_pagination( 'Previous page ' 'page 0' ) + + +def test_upload_contact_list_page(client_request): + page = client_request.get( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + ) + assert 'action' not in page.select_one('form') + assert page.select_one('form input')['name'] == 'file' + assert page.select_one('form input')['type'] == 'file' + + assert normalize_spaces(page.select('.spreadsheet')[0].text) == ( + 'Example A ' + '1 email address ' + '2 test@example.gov.uk' + ) + assert normalize_spaces(page.select('.spreadsheet')[1].text) == ( + 'Example A ' + '1 phone number ' + '2 07700 900123' + ) + + +@pytest.mark.parametrize('file_contents, expected_error, expected_thead, expected_tbody,', [ + ( + """ + telephone,name + +447700900986 + """, + ( + 'Your file has too many columns ' + 'It needs to have 1 column, called ‘email address’ or ‘phone number’. ' + 'Right now it has 2 columns called ‘telephone’ and ‘name’. ' + 'Skip to file contents' + ), + 'Row in file 1 telephone name', + '2 +447700900986', + ), + ( + """ + phone number, email address + +447700900986, test@example.com + """, + ( + 'Your file has too many columns ' + 'It needs to have 1 column, called ‘email address’ or ‘phone number’. ' + 'Right now it has 2 columns called ‘phone number’ and ‘email address’. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number email address', + '2 +447700900986 test@example.com', + ), + ( + """ + email address + +447700900986 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 1 email address. ' + 'Skip to file contents' + ), + 'Row in file 1 email address', + '2 Not a valid email address +447700900986', + ), + ( + """ + phone number + test@example.com + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 1 phone number. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + '2 Must not contain letters or symbols test@example.com', + ), + ( + """ + phone number, phone number, PHONE_NUMBER + +447700900111,+447700900222,+447700900333, + """, + ( + 'Your file has too many columns ' + 'It needs to have 1 column, called ‘email address’ or ‘phone number’. ' + 'Right now it has 3 columns called ‘phone number’, ‘phone number’ and ‘PHONE_NUMBER’. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number phone number PHONE_NUMBER', + '2 +447700900333 +447700900333 +447700900333', + ), + ( + """ + phone number + """, + ( + 'Your file is missing some rows ' + 'It needs at least one row of data. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + '', + ), + ( + "+447700900986", + ( + 'Your file is missing some rows ' + 'It needs at least one row of data, in a column called ' + '‘email address’ or ‘phone number’. ' + 'Skip to file contents' + ), + 'Row in file 1 +447700900986', + '', + ), + ( + "", + ( + 'Your file is missing some rows ' + 'It needs at least one row of data, in a column called ' + '‘email address’ or ‘phone number’. ' + 'Skip to file contents' + ), + 'Row in file 1', + '', + ), + ( + """ + phone number + +447700900986 + + +447700900986 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to enter missing data in 1 row. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + ( + '3 Missing' + ) + ), + ( + """ + phone number + +447700900 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 1 phone number. ' + 'Skip to file contents' + ), + 'Row in file 1 phone number', + '2 Not enough digits +447700900', + ), + ( + """ + email address + ok@example.com + bad@example1 + bad@example2 + """, + ( + 'There’s a problem with invalid.csv ' + 'You need to fix 2 email addresses. ' + 'Skip to file contents' + ), + 'Row in file 1 email address', + ( + '3 Not a valid email address bad@example1 ' + '4 Not a valid email address bad@example2' + ), + ), +]) +def test_upload_csv_file_shows_error_banner( + client_request, + mocker, + mock_s3_upload, + mock_get_job_doesnt_exist, + mock_get_users_by_service, + fake_uuid, + file_contents, + expected_error, + expected_thead, + expected_tbody, +): + + mock_upload = mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) + mock_download = mocker.patch('app.main.views.uploads.s3download', return_value=file_contents) + + page = client_request.post( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + _data={'file': (BytesIO(''.encode('utf-8')), 'invalid.csv')}, + _follow_redirects=True, + ) + mock_upload.assert_called_once_with( + SERVICE_ONE_ID, {'data': '', 'file_name': 'invalid.csv'}, ANY, + ) + mock_download.assert_called_once_with(SERVICE_ONE_ID, fake_uuid) + + assert normalize_spaces(page.select_one('.banner-dangerous').text) == expected_error + + assert page.select_one('form')['action'] == url_for( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + ) + assert page.select_one('form input')['type'] == 'file' + + assert normalize_spaces(page.select_one('thead').text) == expected_thead + assert normalize_spaces(page.select_one('tbody').text) == expected_tbody + + +def test_upload_csv_file_shows_error_banner_for_too_many_rows( + client_request, + mocker, + mock_s3_upload, + mock_get_job_doesnt_exist, + mock_get_users_by_service, + fake_uuid, +): + + mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) + mocker.patch('app.main.views.uploads.s3download', return_value='\n'.join( + ['phone number'] + (['07700900986'] * 50001) + )) + + page = client_request.post( + 'main.upload_contact_list', + service_id=SERVICE_ONE_ID, + _data={'file': (BytesIO(''.encode('utf-8')), 'invalid.csv')}, + _follow_redirects=True, + ) + + assert normalize_spaces(page.select_one('.banner-dangerous').text) == ( + 'Your file has too many rows ' + 'Notify can store files up to 50,000 rows in size. ' + 'Your file has 50,001 rows. ' + 'Skip to file contents' + ) + assert len(page.select('tbody tr')) == 50 + assert normalize_spaces(page.select_one('.table-show-more-link').text) == ( + 'Only showing the first 50 rows' + ) + + +def test_upload_csv_shows_trial_mode_error( + client_request, + mock_get_users_by_service, + mock_get_job_doesnt_exist, + fake_uuid, + mocker +): + mocker.patch('app.main.views.uploads.s3upload', return_value=fake_uuid) + mocker.patch('app.main.views.uploads.s3download', return_value=( + 'phone number\n' + '07900900321' # Not in team + )) + + page = client_request.get( + 'main.check_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + _test_page_title=False, + ) + + assert normalize_spaces(page.select_one('.banner-dangerous').text) == ( + 'You cannot save this phone number ' + 'In trial mode you can only send to yourself and members of your team ' + 'Skip to file contents' + ) + assert page.select_one('.banner-dangerous a')['href'] == url_for( + 'main.trial_mode_new' + ) + + +def test_upload_csv_shows_ok_page( + client_request, + mock_get_live_service, + mock_get_users_by_service, + mock_get_job_doesnt_exist, + fake_uuid, + mocker +): + mocker.patch('app.main.views.uploads.s3download', return_value='\n'.join( + ['email address'] + ['test@example.com'] * 51 + )) + mock_metadata_set = mocker.patch('app.main.views.uploads.set_metadata_on_csv_upload') + + page = client_request.get( + 'main.check_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + original_file_name='good times.xlsx', + _test_page_title=False, + ) + + mock_metadata_set.assert_called_once_with( + SERVICE_ONE_ID, + fake_uuid, + row_count=51, + original_file_name='good times.xlsx', + template_type='email', + valid=True, + ) + + assert normalize_spaces(page.select_one('h1').text) == ( + 'good times.xlsx' + ) + assert normalize_spaces(page.select_one('main p').text) == ( + '51 email addresses found' + ) + assert page.select_one('form')['action'] == url_for( + 'main.save_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + ) + + assert normalize_spaces(page.select_one('form [type=submit]').text) == ( + 'Save contact list' + ) + assert normalize_spaces(page.select_one('thead').text) == ( + 'Row in file 1 email address' + ) + assert len(page.select('tbody tr')) == 50 + assert normalize_spaces(page.select_one('tbody tr').text) == ( + '2 test@example.com' + ) + assert normalize_spaces(page.select_one('.table-show-more-link').text) == ( + 'Only showing the first 50 rows' + ) + + +def test_save_contact_list( + mocker, + client_request, + fake_uuid, + mock_create_contact_list, +): + mocker.patch('app.models.contact_list.get_csv_metadata', return_value={ + 'row_count': 999, + 'valid': True, + 'original_file_name': 'example.csv', + 'template_type': 'email' + }) + client_request.post( + 'main.save_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + _expected_status=302, + _expected_redirect=url_for( + 'main.contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + _external=True, + ) + ) + mock_create_contact_list.assert_called_once_with( + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + original_file_name='example.csv', + row_count=999, + template_type='email', + ) + + +def test_cant_save_bad_contact_list( + mocker, + client_request, + fake_uuid, + mock_create_contact_list, +): + mocker.patch('app.models.contact_list.get_csv_metadata', return_value={ + 'row_count': 999, + 'valid': False, + 'original_file_name': 'example.csv', + 'template_type': 'email' + }) + client_request.post( + 'main.save_contact_list', + service_id=SERVICE_ONE_ID, + upload_id=fake_uuid, + _expected_status=403, + ) + assert mock_create_contact_list.called is False + + +def test_view_contact_list( + client_request, + fake_uuid, +): + page = client_request.get( + 'main.contact_list', + service_id=SERVICE_ONE_ID, + contact_list_id=fake_uuid, + _test_page_title=False, + ) + assert page.text == 'page for contact list 6ce466d0-fd6a-11e5-82f5-e0accb9d11a6' diff --git a/tests/conftest.py b/tests/conftest.py index f01c9150c..d44aa12eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1819,6 +1819,29 @@ def mock_get_no_uploads(mocker, api_user_active): ) +@pytest.fixture(scope='function') +def mock_create_contact_list(mocker, api_user_active): + def _create( + service_id, + upload_id, + original_file_name, + row_count, + template_type, + ): + return { + 'service_id': service_id, + 'upload_id': upload_id, + 'original_file_name': original_file_name, + 'row_count': row_count, + 'template_type': template_type, + } + + return mocker.patch( + 'app.contact_list_api_client.create_contact_list', + side_effect=_create, + ) + + @pytest.fixture(scope='function') def mock_get_notifications( mocker,