mirror of
https://github.com/GSA/notifications-admin.git
synced 2026-08-19 14:09:20 -04:00
Give the user better error messages for CSV files
Makes uses of the additions to utils in https://github.com/alphagov/notifications-utils/pull/9 This commit strips out a lot of the complex stuff that the views and templates in this app were doing. There is now a cleaner separation of concerns: - utils returns the number and type of errors in the csv - `get_errors_for_csv` helper in this app maps the number and type of errors onto human-friendly error messages - the view and template just doing the glueing-together of all the pieces This is (hopefully) easier to understand, definitely makes the component parts easier to test in isolation, and makes it easier to give more specific error messages.
This commit is contained in:
Chris Hill-Scott
@@ -1,6 +1,7 @@
|
||||
import csv
|
||||
import io
|
||||
import uuid
|
||||
from contextlib import suppress
|
||||
|
||||
from flask import (
|
||||
request,
|
||||
@@ -15,7 +16,8 @@ from flask import (
|
||||
|
||||
from flask_login import login_required, current_user
|
||||
from notifications_python_client.errors import HTTPError
|
||||
from utils.template import Template, NeededByTemplateError, NoPlaceholderForDataError
|
||||
from utils.template import Template
|
||||
from utils.recipients import RecipientCSV, first_column_heading
|
||||
|
||||
from app.main import main
|
||||
from app.main.forms import CsvUploadForm
|
||||
@@ -26,10 +28,7 @@ from app.main.uploader import (
|
||||
from app.main.dao import templates_dao
|
||||
from app.main.dao import services_dao
|
||||
from app import job_api_client
|
||||
from app.utils import (
|
||||
validate_recipient, validate_header_row, InvalidPhoneError, InvalidEmailError, user_has_permissions,
|
||||
InvalidHeaderError)
|
||||
from utils.process_csv import first_column_heading
|
||||
from app.utils import user_has_permissions, get_errors_for_csv
|
||||
|
||||
|
||||
send_messages_page_headings = {
|
||||
@@ -100,16 +99,25 @@ def send_messages(service_id, template_id):
|
||||
form = CsvUploadForm()
|
||||
if form.validate_on_submit():
|
||||
try:
|
||||
csv_file = form.file
|
||||
filedata = _get_filedata(csv_file)
|
||||
upload_id = str(uuid.uuid4())
|
||||
s3upload(upload_id, service_id, filedata, current_app.config['AWS_REGION'])
|
||||
session['upload_data'] = {"template_id": template_id, "original_file_name": filedata['file_name']}
|
||||
s3upload(
|
||||
upload_id,
|
||||
service_id,
|
||||
{
|
||||
'file_name': form.file.data.filename,
|
||||
'data': form.file.data.getvalue().decode('utf-8')
|
||||
},
|
||||
current_app.config['AWS_REGION']
|
||||
)
|
||||
session['upload_data'] = {
|
||||
"template_id": template_id,
|
||||
"original_file_name": form.file.data.filename
|
||||
}
|
||||
return redirect(url_for('.check_messages',
|
||||
service_id=service_id,
|
||||
upload_id=upload_id))
|
||||
except ValueError as e:
|
||||
flash('There was a problem uploading: {}'.format(csv_file.data.filename))
|
||||
flash('There was a problem uploading: {}'.format(form.file.data.filename))
|
||||
flash(str(e))
|
||||
return redirect(url_for('.send_messages', service_id=service_id, template_id=template_id))
|
||||
|
||||
@@ -118,7 +126,6 @@ def send_messages(service_id, template_id):
|
||||
templates_dao.get_service_template_or_404(service_id, template_id)['data'],
|
||||
prefix=service['name']
|
||||
)
|
||||
recipient_column = first_column_heading[template.template_type]
|
||||
|
||||
return render_template(
|
||||
'views/send.html',
|
||||
@@ -174,7 +181,7 @@ def send_message_to_self(service_id, template_id):
|
||||
|
||||
filedata = {
|
||||
'file_name': 'Test run',
|
||||
'data': output.getvalue().splitlines()
|
||||
'data': output.getvalue()
|
||||
}
|
||||
upload_id = str(uuid.uuid4())
|
||||
s3upload(upload_id, service_id, filedata, current_app.config['AWS_REGION'])
|
||||
@@ -185,107 +192,81 @@ def send_message_to_self(service_id, template_id):
|
||||
upload_id=upload_id))
|
||||
|
||||
|
||||
@main.route("/services/<service_id>/check/<upload_id>",
|
||||
methods=['GET', 'POST'])
|
||||
@main.route("/services/<service_id>/check/<upload_id>", methods=['GET'])
|
||||
@login_required
|
||||
@user_has_permissions('send_texts', 'send_emails', 'send_letters')
|
||||
def check_messages(service_id, upload_id):
|
||||
|
||||
upload_data = session['upload_data']
|
||||
template_id = upload_data.get('template_id')
|
||||
service = services_dao.get_service_by_id_or_404(service_id)
|
||||
|
||||
if request.method == 'GET':
|
||||
contents = s3download(service_id, upload_id)
|
||||
if not contents:
|
||||
flash('There was a problem reading your upload file')
|
||||
raw_template = templates_dao.get_service_template_or_404(service_id, template_id)['data']
|
||||
upload_result = _get_rows(contents, raw_template)
|
||||
session['upload_data']['notification_count'] = len(upload_result['rows'])
|
||||
template = Template(
|
||||
raw_template,
|
||||
values=upload_result['rows'][0] if upload_result['valid'] else {},
|
||||
drop_values={first_column_heading[raw_template['template_type']]},
|
||||
prefix=service['name']
|
||||
)
|
||||
return render_template(
|
||||
'views/check.html',
|
||||
upload_result=upload_result,
|
||||
template=template,
|
||||
page_heading=get_page_headings(template.template_type),
|
||||
column_headers=[first_column_heading[template.template_type]] + list(template.placeholders_as_markup),
|
||||
original_file_name=upload_data.get('original_file_name'),
|
||||
service_id=service_id,
|
||||
service=service,
|
||||
form=CsvUploadForm()
|
||||
)
|
||||
elif request.method == 'POST':
|
||||
if request.files:
|
||||
# The csv was invalid, validate the csv again
|
||||
return send_messages(service_id, template_id)
|
||||
contents = s3download(service_id, upload_id)
|
||||
if not contents:
|
||||
flash('There was a problem reading your upload file')
|
||||
|
||||
original_file_name = upload_data.get('original_file_name')
|
||||
notification_count = upload_data.get('notification_count')
|
||||
session.pop('upload_data')
|
||||
try:
|
||||
job_api_client.create_job(upload_id, service_id, template_id, original_file_name, notification_count)
|
||||
except HTTPError as e:
|
||||
if e.status_code == 404:
|
||||
abort(404)
|
||||
else:
|
||||
raise e
|
||||
|
||||
return redirect(
|
||||
url_for('main.view_job', service_id=service_id, job_id=upload_id)
|
||||
)
|
||||
|
||||
|
||||
def _get_filedata(file):
|
||||
import itertools
|
||||
reader = csv.reader(
|
||||
file.data.getvalue().decode('utf-8').splitlines(),
|
||||
quoting=csv.QUOTE_NONE,
|
||||
skipinitialspace=True
|
||||
template = Template(
|
||||
templates_dao.get_service_template_or_404(
|
||||
service_id,
|
||||
session['upload_data'].get('template_id')
|
||||
)['data'],
|
||||
prefix=service['name']
|
||||
)
|
||||
lines = []
|
||||
for row in reader:
|
||||
non_empties = itertools.dropwhile(lambda x: x.strip() == '', row)
|
||||
has_content = []
|
||||
for item in non_empties:
|
||||
has_content.append(item)
|
||||
if has_content:
|
||||
lines.append(row)
|
||||
|
||||
if len(lines) < 2: # must be header row and at least one data row
|
||||
message = 'The file {} contained no data'.format(file.data.filename)
|
||||
raise ValueError(message)
|
||||
|
||||
content_lines = []
|
||||
for row in lines:
|
||||
content_lines.append(','.join(row).rstrip(','))
|
||||
return {'file_name': file.data.filename, 'data': content_lines}
|
||||
|
||||
|
||||
def _get_rows(contents, raw_template):
|
||||
reader = csv.DictReader(
|
||||
contents.split('\n'),
|
||||
quoting=csv.QUOTE_NONE,
|
||||
skipinitialspace=True
|
||||
recipients = RecipientCSV(
|
||||
contents,
|
||||
template_type=template.template_type,
|
||||
placeholders=template.placeholders,
|
||||
max_initial_rows_shown=5
|
||||
)
|
||||
|
||||
with suppress(StopIteration):
|
||||
template.values = next(recipients.rows)
|
||||
|
||||
session['upload_data']['notification_count'] = len(list(recipients.rows))
|
||||
session['upload_data']['valid'] = not recipients.has_errors
|
||||
|
||||
return render_template(
|
||||
'views/check.html',
|
||||
recipients=recipients,
|
||||
template=template,
|
||||
page_heading=get_page_headings(template.template_type),
|
||||
errors=get_errors_for_csv(recipients, template.template_type),
|
||||
count_of_recipients=session['upload_data']['notification_count'],
|
||||
count_of_displayed_recipients=len(list(recipients.rows_annotated_and_truncated)),
|
||||
original_file_name=session['upload_data'].get('original_file_name'),
|
||||
service_id=service_id,
|
||||
service=service,
|
||||
form=CsvUploadForm()
|
||||
)
|
||||
|
||||
|
||||
@main.route("/services/<service_id>/check/<upload_id>", methods=['POST'])
|
||||
@login_required
|
||||
@user_has_permissions('send_texts', 'send_emails', 'send_letters')
|
||||
def start_job(service_id, upload_id):
|
||||
|
||||
upload_data = session['upload_data']
|
||||
services_dao.get_service_by_id_or_404(service_id)
|
||||
|
||||
if request.files or not session['upload_data'].get('valid'):
|
||||
# The csv was invalid, validate the csv again
|
||||
return send_messages(service_id, upload_data.get('template_id'))
|
||||
|
||||
session.pop('upload_data')
|
||||
|
||||
try:
|
||||
job_api_client.create_job(
|
||||
upload_id,
|
||||
service_id,
|
||||
upload_data.get('template_id'),
|
||||
upload_data.get('original_file_name'),
|
||||
upload_data.get('notification_count')
|
||||
)
|
||||
except HTTPError as e:
|
||||
if e.status_code == 404:
|
||||
abort(404)
|
||||
else:
|
||||
raise e
|
||||
|
||||
return redirect(
|
||||
url_for('main.view_job', service_id=service_id, job_id=upload_id)
|
||||
)
|
||||
valid = True
|
||||
rows = []
|
||||
for row in reader:
|
||||
rows.append(row)
|
||||
try:
|
||||
validate_recipient(
|
||||
row, template_type=raw_template['template_type']
|
||||
)
|
||||
Template(
|
||||
raw_template,
|
||||
values=row,
|
||||
drop_values={first_column_heading[raw_template['template_type']]}
|
||||
).replaced
|
||||
except (InvalidEmailError, InvalidPhoneError, NeededByTemplateError,
|
||||
NoPlaceholderForDataError, InvalidHeaderError):
|
||||
valid = False
|
||||
return {"valid": valid, "rows": rows}
|
||||
|
||||
Reference in New Issue
Block a user