Use Template to replace/highlight placeholders

This commit brings in the `Template` util, added here:
https://github.com/alphagov/notifications-utils/pull/1

It also does a fair bit of tidying up, which I’ve unfortunately squashed into
this one massive commit. The main change is moving 404 handling into the
templates dao, so that every view isn’t littered with `try: … except(HTTPError)`.

It also adds new features, in a prototypy sort of way, which are:
- download a prefilled example CSV
- show all the columns for your template on the 'check' page

app/__init__.py

@@ -62,8 +62,6 @@ def create_app(config_name, config_overrides=None):
 
     application.session_interface = ItsdangerousSessionInterface()
 
-    application.add_template_filter(placeholders)
-    application.add_template_filter(replace_placeholders)
     application.add_template_filter(nl2br)
     application.add_template_filter(format_datetime)
     application.add_template_filter(syntax_highlight_json)
@@ -123,16 +121,6 @@ def convert_to_boolean(value):
     return value
 
 
-def placeholders(value):
-    if not value:
-        return value
-    return Markup(re.sub(
-        r"\(\(([^\)]+)\)\)",  # anything that looks like ((registration number))
-        lambda match: "<span class='placeholder'>{}</span>".format(match.group(1)),
-        value
-    ))
-
-
 def nl2br(value):
     _paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}')
 
@@ -141,16 +129,6 @@ def nl2br(value):
     return Markup(result)
 
 
-def replace_placeholders(template, values):
-    if not template:
-        return template
-    return Markup(re.sub(
-        r"\(\(([^\)]+)\)\)",  # anything that looks like ((registration number))
-        lambda match: values.get(match.group(1), ''),
-        template
-    ))
-
-
 def syntax_highlight_json(code):
     return Markup(highlight(code, JavascriptLexer(), HtmlFormatter(noclasses=True)))
 

app/main/dao/templates_dao.py

@@ -1,4 +1,4 @@
-from flask import url_for
+from flask import url_for, abort
 from app import notifications_api_client
 from app.main.utils import BrowsableItem
 
@@ -17,9 +17,24 @@ def get_service_templates(service_id):
     return notifications_api_client.get_service_templates(service_id)
 
 
-def get_service_template(service_id, template_id):
+def get_service_templates_or_404(service_id):
-    return notifications_api_client.get_service_template(
+    try:
-        service_id, template_id)
+        get_service_templates(service_id)
+    except HTTPError as e:
+        if e.status_code == 404:
+            abort(404)
+        else:
+            raise e
+
+
+def get_service_template_or_404(service_id, template_id):
+    try:
+        return notifications_api_client.get_service_template(service_id, template_id)
+    except HTTPError as e:
+        if e.status_code == 404:
+            abort(404)
+        else:
+            raise e
 
 
 def delete_service_template(service_id, template_id):

app/main/views/jobs.py

@@ -8,6 +8,7 @@ from flask import (
 )
 from flask_login import login_required
 from notifications_python_client.errors import HTTPError
+from utils.template import Template
 
 from app import job_api_client
 from app.main import main
@@ -38,7 +39,6 @@ def view_jobs(service_id):
 def view_job(service_id, job_id):
     try:
         job = job_api_client.get_job(service_id, job_id)['data']
-        template = templates_dao.get_service_template(service_id, job['template'])['data']
         messages = []
         return render_template(
             'views/job.html',
@@ -55,7 +55,9 @@ def view_job(service_id, job_id):
             cost=u'£0.00',
             uploaded_file_name=job['original_file_name'],
             uploaded_file_time=job['created_at'],
-            template=template,
+            template=Template(
+                templates_dao.get_service_template_or_404(service_id, job['template'])['data']
+            ),
             service_id=service_id
         )
     except HTTPError as e:

app/main/views/sms.py

@@ -1,6 +1,8 @@
 import csv
 import uuid
 import botocore
+import re
+import io
 
 from datetime import date
 
@@ -12,12 +14,14 @@ from flask import (
     flash,
     abort,
     session,
-    current_app
+    current_app,
+    Markup
 )
 
 from flask_login import login_required
 from werkzeug import secure_filename
 from notifications_python_client.errors import HTTPError
+from utils.template import Template
 
 from app.main import main
 from app.main.forms import CsvUploadForm
@@ -35,22 +39,19 @@ from app.main.utils import (
 
 @main.route("/services/<service_id>/sms/send", methods=['GET'])
 def choose_sms_template(service_id):
-    try:
+    return render_template(
-        templates = templates_dao.get_service_templates(service_id)['data']
+        'views/choose-sms-template.html',
-    except HTTPError as e:
+        templates=[
-        if e.status_code == 404:
+            Template(template) for template in templates_dao.get_service_templates(service_id)['data']
-            abort(404)
+        ],
-        else:
+        service_id=service_id
-            raise e
+    )
-
-    return render_template('views/choose-sms-template.html',
-                           templates=templates,
-                           service_id=service_id)
 
 
 @main.route("/services/<service_id>/sms/send/<template_id>", methods=['GET', 'POST'])
 @login_required
 def send_sms(service_id, template_id):
+
     form = CsvUploadForm()
     if form.validate_on_submit():
         try:
@@ -63,24 +64,33 @@ def send_sms(service_id, template_id):
                                     service_id=service_id,
                                     upload_id=upload_id))
         except ValueError as e:
-            message = 'There was a problem uploading: {}'.format(
+            flash('There was a problem uploading: {}'.format(csv_file.filename))
-                      csv_file.filename)
-            flash(message)
             flash(str(e))
             return redirect(url_for('.send_sms', service_id=service_id, template_id=template_id))
 
-    try:
+    template = Template(
-        template = templates_dao.get_service_template(service_id, template_id)['data']
+       templates_dao.get_service_template_or_404(service_id, template_id)['data']
-    except HTTPError as e:
+    )
-        if e.status_code == 404:
-            abort(404)
-        else:
-            raise e
 
-    return render_template('views/send-sms.html',
+    return render_template(
-                           template=template,
+        'views/send-sms.html',
-                           form=form,
+        template=template,
-                           service_id=service_id)
+        column_headers=['phone number'] + template.placeholders_as_markup,
+        form=form,
+        service_id=service_id
+    )
+
+
+@main.route("/services/<service_id>/sms/send/<template_id>.csv", methods=['GET'])
+@login_required
+def get_example_csv(service_id, template_id):
+    template = templates_dao.get_service_template_or_404(service_id, template_id)['data']
+    output = io.StringIO()
+    csv.writer(output).writerow(
+        ['phone number'] + Template(template).list_placeholders
+    )
+
+    return(output.getvalue(), 200, {'Content-Type': 'text/csv; charset=utf-8'})
 
 
 @main.route("/services/<service_id>/sms/check/<upload_id>",
@@ -89,21 +99,23 @@ def send_sms(service_id, template_id):
 def check_sms(service_id, upload_id):
 
     if request.method == 'GET':
-
         contents = s3download(service_id, upload_id)
         if not contents:
             flash('There was a problem reading your upload file')
         upload_result = _get_numbers(contents)
         upload_data = session['upload_data']
-        original_file_name = upload_data.get('original_file_name')
         template_id = upload_data.get('template_id')
-        template = templates_dao.get_service_template(service_id, template_id)['data']
+        template = Template(
+            templates_dao.get_service_template_or_404(service_id, template_id)['data'],
+            values=upload_result['valid'][0] if upload_result['valid'] else {},
+            drop_values={'phone'}
+        )
         return render_template(
             'views/check-sms.html',
             upload_result=upload_result,
-            message_template=template['content'],
+            template=template,
-            original_file_name=original_file_name,
+            column_headers=['phone number'] + template.placeholders_as_markup,
-            template_id=template_id,
+            original_file_name=upload_data.get('original_file_name'),
             service_id=service_id
         )
     elif request.method == 'POST':
@@ -153,3 +165,16 @@ def _get_numbers(contents):
         except InvalidPhoneError:
             rejects.append({"line_number": i+2, "phone": row['phone']})
     return {"valid": valid, "rejects": rejects}
+
+
+def _get_column_headers(template_content, marked_up=False):
+    headers = re.findall(
+        r"\(\(([^\)]+)\)\)",  # anything that looks like ((registration number))
+        template_content
+    )
+    if marked_up:
+        return [
+            Markup("<span class='placeholder'>{}</span>".format(header))
+            for header in headers
+        ]
+    return headers

app/main/views/templates.py

@@ -1,13 +1,15 @@
 from flask import request, render_template, redirect, url_for, flash, abort
 from flask_login import login_required
 
+from notifications_python_client.errors import HTTPError
+from utils.template import Template
+
 from app.main import main
 from app.main.forms import TemplateForm
 from app import job_api_client
 from app.main.dao.services_dao import get_service_by_id
 from app.main.dao import templates_dao as tdao
 from app.main.dao import services_dao as sdao
-from notifications_python_client.errors import HTTPError
 
 
 @main.route("/services/<service_id>/templates")
@@ -15,7 +17,6 @@ from notifications_python_client.errors import HTTPError
 def manage_service_templates(service_id):
     try:
         jobs = job_api_client.get_job(service_id)['data']
-        templates = tdao.get_service_templates(service_id)['data']
     except HTTPError as e:
         if e.status_code == 404:
             abort(404)
@@ -25,7 +26,11 @@ def manage_service_templates(service_id):
         'views/manage-templates.html',
         service_id=service_id,
         has_jobs=bool(jobs),
-        templates=[tdao.TemplatesBrowsableItem(x) for x in templates])
+        templates=[
+            Template(template)
+            for template in tdao.get_service_templates(service_id)['data']
+        ]
+    )
 
 
 @main.route("/services/<service_id>/templates/add", methods=['GET', 'POST'])
@@ -56,14 +61,7 @@ def add_service_template(service_id):
 @main.route("/services/<service_id>/templates/<int:template_id>", methods=['GET', 'POST'])
 @login_required
 def edit_service_template(service_id, template_id):
-    try:
+    template = tdao.get_service_template_or_404(service_id, template_id)['data']
-        template = tdao.get_service_template(service_id, template_id)['data']
-    except HTTPError as e:
-        if e.status_code == 404:
-            abort(404)
-        else:
-            raise e
-
     template['template_content'] = template['content']
     form = TemplateForm(**template)
 
@@ -84,21 +82,14 @@ def edit_service_template(service_id, template_id):
 @main.route("/services/<service_id>/templates/<int:template_id>/delete", methods=['GET', 'POST'])
 @login_required
 def delete_service_template(service_id, template_id):
-    try:
+    template = tdao.get_service_template_or_404(service_id, template_id)['data']
-        template = tdao.get_service_template(service_id, template_id)['data']
-    except HTTPError as e:
-        if e.status_code == 404:
-            abort(404)
-        else:
-            raise e
-
-    template['template_content'] = template['content']
-    form = TemplateForm(**template)
 
     if request.method == 'POST':
         tdao.delete_service_template(service_id, template_id)
         return redirect(url_for('.manage_service_templates', service_id=service_id))
 
+    template['template_content'] = template['content']
+    form = TemplateForm(**template)
     flash('Are you sure you want to delete ‘{}’?'.format(form.name.data), 'delete')
     return render_template(
         'views/edit-template.html',

app/templates/components/email-message.html

@@ -10,10 +10,10 @@
   {% endif %}
   <div class="email-message">
     <div class="email-message-subject">
-      {{ subject|placeholders }}
+      {{ subject }}
     </div>
     <div class="email-message-body">
-      {{ body|nl2br|placeholders }}
+      {{ body|nl2br }}
     </div>
   </div>
 {% endmacro %}

app/templates/components/sms-message.html

@@ -11,7 +11,7 @@
     </h3>
   {% endif %}
   <div class="sms-message-wrapper{% if input_name %}-with-radio{% endif %}">
-    {{ body|placeholders }}
+    {{ body }}
   </div>
   {% if recipient %}
     <p class="sms-message-recipient">

app/templates/views/check-sms.html

@@ -17,14 +17,14 @@
     {% for rejected in upload_result.rejects %}
       <p>Line {{rejected.line_number}}: {{rejected.phone }}</a>
     {% endfor %}
-    <p><a href="{{url_for('.send_sms', service_id=service_id, template_id=template_id)}}" class="button">Go back and resolve errors</a></p>
+    <p><a href="{{url_for('.send_sms', service_id=service_id, template_id=template.id)}}" class="button">Go back and resolve errors</a></p>
 
   {% else %}
 
     <div class="grid-row">
       <div class="column-two-thirds">
         {{ sms_message(
-          message_template|replace_placeholders(upload_result.valid[0])
+          template.replaced
         )}}
       </div>
     </div>
@@ -32,13 +32,13 @@
     <form method="POST" enctype="multipart/form-data">
       <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
       <input type="submit" class="button" value="{{ "Send {} text message{}".format(upload_result.valid|count, '' if upload_result.valid|count == 1 else 's') }}" />
-      <a href="{{url_for('.send_sms', service_id=service_id, template_id=template_id)}}" class="page-footer-back-link">Back</a>
+      <a href="{{url_for('.send_sms', service_id=service_id, template_id=template.id)}}" class="page-footer-back-link">Back</a>
     </form>
 
     {% call(item) list_table(
       upload_result.valid,
       caption=original_file_name,
-      field_headings=['Phone number']
+      field_headings=column_headers
     ) %}
       {% call field() %}
         {{ item.phone }}

app/templates/views/job.html

@@ -17,7 +17,7 @@
     <div class="grid-row">
       <div class="column-two-thirds">
         {{ sms_message(
-          template['content'],
+          template.formatted_as_markup,
         )}}
       </div>
     </div>

app/templates/views/manage-templates.html

@@ -26,19 +26,19 @@ Manage templates – GOV.UK Notify
       <div class="column-two-thirds">
 
         {% for template in templates %}
-          {% if template.get_field('template_type') == 'sms' %}
+          {% if template.template_type == 'email' %}
-            {{ sms_message(
-              template.get_field('content'),
-              name=template.title,
-              id=template.get_field('id'),
-              edit_link=url_for('.edit_service_template', service_id=template.get_field('service'), template_id=template.get_field('id'))
-            ) }}
-          {% elif template.get_field('template_type') == 'email' %}
             {{ email_message(
               template.get_field('subject'),
               template.get_field('content'),
               name=template.get_field('name'),
-              edit_link=url_for('.edit_service_template', service_id=template.get_field('service'), template_id=template.get_field('id'))
+              edit_link=url_for('.edit_service_template', service_id=service_id, template_id=template.id)
+            ) }}
+          {% else %}
+            {{ sms_message(
+              template.formatted_as_markup,
+              name=template.name,
+              id=template.id,
+              edit_link=url_for('.edit_service_template', service_id=service_id, template_id=template.id)
             ) }}
           {% endif %}
         {% endfor %}

app/templates/views/send-sms.html

@@ -2,6 +2,7 @@
 {% from "components/sms-message.html" import sms_message %}
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/file-upload.html" import file_upload %}
+{% from "components/table.html" import list_table %}
 
 {% block page_title %}
   Send text messages – GOV.UK Notify
@@ -32,5 +33,20 @@
       "Continue to preview"
     ) }}
 
+    {% if column_headers %}
+      {% call(item) list_table(
+        [],
+        caption='Preview',
+        field_headings=column_headers,
+        field_headings_visible=True,
+        caption_visible=False,
+        empty_message="Your data here"
+      ) %}
+      {% endcall %}
+    {% endif %}
+    <p class="table-show-more-link">
+      <a href="{{ url_for('.get_example_csv', service_id=service_id, template_id=template.id) }}">Download this CSV file</a>
+    </p>
+
   </form>
 {% endblock %}

requirements.txt

@@ -14,4 +14,4 @@ Pygments==2.0.2
 
 git+https://github.com/alphagov/notifications-python-client.git@0.2.5#egg=notifications-python-client==0.2.5
 
-git+https://github.com/alphagov/notifications-utils.git@0.0.3#egg=notifications-utils==0.0.3
+git+https://github.com/alphagov/notifications-utils.git@0.1.0#egg=notifications-utils==0.1.0

tests/app/main/views/test_sms.py

@@ -8,9 +8,8 @@ def test_choose_sms_template(app_,
                              api_user_active,
                              mock_login,
                              mock_get_user,
-                             mock_get_service_templates,
                              mock_check_verify_code,
-                             mock_get_service_template):
+                             mock_get_service_templates):
     with app_.test_request_context():
         with app_.test_client() as client:
             client.login(api_user_active)