Bring in refactored CSV processing

Shouldn’t be any functional changes here, just things being named more
clearly.

app/main/views/send.py

@@ -528,29 +528,23 @@ def _check_messages(service_id, template_type, upload_id, preview_row, letters_a
         back_link = url_for('.send_messages', service_id=service_id, template_id=template.id)
         choose_time_form = ChooseTimeForm()
 
-    count_of_recipients = len(list(recipients.rows))
-
     if preview_row < 2:
         abort(404)
 
-    if preview_row < count_of_recipients + 2:
-        template.values = recipients[preview_row - 2]
+    if preview_row < len(recipients) + 2:
+        template.values = recipients[preview_row - 2].recipient_and_personalisation
     elif preview_row > 2:
         abort(404)
 
-    session['upload_data']['notification_count'] = count_of_recipients
+    session['upload_data']['notification_count'] = len(recipients)
     session['upload_data']['valid'] = not recipients.has_errors
     return dict(
         recipients=recipients,
         template=template,
         errors=recipients.has_errors,
         row_errors=get_errors_for_csv(recipients, template.template_type),
-        count_of_recipients=count_of_recipients,
-        count_of_displayed_recipients=(
-            len(list(recipients.initial_annotated_rows_with_errors))
-            if any(recipients.rows_with_errors) and not recipients.missing_column_headers else
-            len(list(recipients.initial_annotated_rows))
-        ),
+        count_of_recipients=len(recipients),
+        count_of_displayed_recipients=len(list(recipients.displayed_rows)),
         original_file_name=session['upload_data'].get('original_file_name'),
         upload_id=upload_id,
         form=CsvUploadForm(),

app/templates/views/check/column-errors.html

@@ -149,7 +149,7 @@
 
     <div class="fullscreen-content" data-module="fullscreen-table">
       {% call(item, row_number) list_table(
-        recipients.initial_annotated_rows_with_errors if row_errors and not recipients.missing_column_headers else recipients.initial_annotated_rows,
+        recipients.displayed_rows,
         caption=original_file_name,
         caption_visible=False,
         field_headings=[
@@ -162,21 +162,21 @@
           </span>
         {% endcall %}
         {% for column in column_headers %}
-          {% if item['columns'][column].error and not recipients.missing_column_headers %}
+          {% if item[column].error and not recipients.missing_column_headers %}
             {% call field() %}
               <span>
-                <span class="table-field-error-label">{{ item['columns'][column].error }}</span>
-                {{ item['columns'][column].data if item['columns'][column].data != None }}
+                <span class="table-field-error-label">{{ item[column].error }}</span>
+                {{ item[column].data if item[column].data != None }}
               </span>
             {% endcall %}
-          {% elif item['columns'][column].ignore %}
-            {{ text_field(item['columns'][column].data or '', status='default') }}
+          {% elif item[column].ignore %}
+            {{ text_field(item[column].data or '', status='default') }}
           {% else %}
-            {{ text_field(item['columns'][column].data or '') }}
+            {{ text_field(item[column].data or '') }}
           {% endif %}
         {% endfor %}
-        {% if item['columns'].get(None) %}
-          {% for column in item['columns'][None].data %}
+        {% if item[None].data %}
+          {% for column in item[None].data %}
             {{ text_field(column, status='default') }}
           {% endfor %}
         {% endif %}

app/templates/views/check/ok.html

@@ -51,7 +51,7 @@
 
     <div class="fullscreen-content" data-module="fullscreen-table">
       {% call(item, row_number) list_table(
-        recipients.initial_annotated_rows_with_errors if row_errors and not recipients.missing_column_headers else recipients.initial_annotated_rows,
+        recipients.displayed_rows,
         caption=original_file_name,
         caption_visible=False,
         field_headings=[
@@ -68,14 +68,14 @@
           </span>
         {% endcall %}
         {% for column in recipients.column_headers %}
-          {% if item['columns'][column].ignore %}
-            {{ text_field(item['columns'][column].data or '', status='default') }}
+          {% if item[column].ignore %}
+            {{ text_field(item[column].data or '', status='default') }}
           {% else %}
-            {{ text_field(item['columns'][column].data or '') }}
+            {{ text_field(item[column].data or '') }}
           {% endif %}
         {% endfor %}
-        {% if item['columns'].get(None) %}
-          {% for column in item['columns'][None].data %}
+        {% if item[None].data %}
+          {% for column in item[None].data %}
             {{ text_field(column, status='default') }}
           {% endfor %}
         {% endif %}

app/templates/views/check/row-errors.html

@@ -54,7 +54,7 @@
 
   <div class="fullscreen-content" data-module="fullscreen-table">
     {% call(item, row_number) list_table(
-      recipients.initial_annotated_rows_with_errors if row_errors and not recipients.missing_column_headers else recipients.initial_annotated_rows,
+      recipients.displayed_rows,
       caption=original_file_name,
       caption_visible=False,
       field_headings=[
@@ -62,26 +62,26 @@
       ] + recipients.column_headers
     ) %}
       {% call index_field() %}
-        <span class="{% if item.index in recipients.rows_with_errors %}table-field-error{% endif %}">
+        <span class="{% if item.has_errors %}table-field-error{% endif %}">
           {{ item.index + 2 }}
         </span>
       {% endcall %}
       {% for column in recipients.column_headers %}
-        {% if item['columns'][column].error and not recipients.missing_column_headers %}
+        {% if item[column].error and not recipients.missing_column_headers %}
           {% call field() %}
             <span>
-              <span class="table-field-error-label">{{ item['columns'][column].error }}</span>
-              {{ item['columns'][column].data if item['columns'][column].data != None }}
+              <span class="table-field-error-label">{{ item[column].error }}</span>
+              {{ item[column].data if item[column].data != None }}
             </span>
           {% endcall %}
-        {% elif item['columns'][column].ignore %}
-          {{ text_field(item['columns'][column].data or '', status='default') }}
+        {% elif item[column].ignore %}
+          {{ text_field(item[column].data or '', status='default') }}
         {% else %}
-          {{ text_field(item['columns'][column].data or '') }}
+          {{ text_field(item[column].data or '') }}
         {% endif %}
       {% endfor %}
-      {% if item['columns'].get(None) %}
-        {% for column in item['columns'][None].data %}
+      {% if item[None].data %}
+        {% for column in item[None].data %}
           {{ text_field(column, status='default') }}
         {% endfor %}
       {% endif %}

app/utils.py

@@ -102,7 +102,7 @@ def get_errors_for_csv(recipients, template_type):
 
     errors = []
 
-    if recipients.rows_with_bad_recipients:
+    if any(recipients.rows_with_bad_recipients):
         number_of_bad_recipients = len(list(recipients.rows_with_bad_recipients))
         if 'sms' == template_type:
             if 1 == number_of_bad_recipients:
@@ -120,7 +120,7 @@ def get_errors_for_csv(recipients, template_type):
             else:
                 errors.append("fix {} addresses".format(number_of_bad_recipients))
 
-    if recipients.rows_with_missing_data:
+    if any(recipients.rows_with_missing_data):
         number_of_rows_with_missing_data = len(list(recipients.rows_with_missing_data))
         if 1 == number_of_rows_with_missing_data:
             errors.append("enter missing data in 1 row")
@@ -156,7 +156,7 @@ def generate_notifications_csv(**kwargs):
                 values = [
                     notification['row_number'],
                 ] + [
-                    original_upload[notification['row_number'] - 1].get(header)
+                    original_upload[notification['row_number'] - 1].get(header).data
                     for header in original_column_headers
                 ] + [
                     notification['template_name'],

requirements.txt

@@ -18,4 +18,4 @@ notifications-python-client==4.7.2
 # PaaS
 awscli-cwlogs>=1.4,<1.5
 
-git+https://github.com/alphagov/notifications-utils.git@24.0.0#egg=notifications-utils==24.0.0
+git+https://github.com/alphagov/notifications-utils.git@25.0.0#egg=notifications-utils==25.0.0