error when users put non-GSM chars in a sms template

additionally, this moves the formatted_list jinja macro into a python
function, so that it can be called from the form validator

app/__init__.py

@@ -131,6 +131,7 @@ def create_app():
     application.add_template_filter(format_notification_status_as_time)
     application.add_template_filter(format_notification_status_as_field_status)
     application.add_template_filter(format_notification_status_as_url)
+    application.add_template_filter(formatted_list)
 
     application.after_request(useful_headers_after_request)
     application.after_request(save_service_after_request)
@@ -345,6 +346,28 @@ def format_notification_status_as_url(status):
     }.get(status)
 
 
+def formatted_list(
+  items,
+  conjunction='and',
+  before_each='‘',
+  after_each='’',
+  separator=', ',
+  prefix='',
+  prefix_plural=''
+):
+    items = list(items)
+    if len(items) == 1:
+        return '{prefix} {before_each}{items[0]}{after_each}'.format(**locals())
+    elif items:
+        formatted_items = ['{}{}{}'.format(before_each, item, after_each) for item in items]
+
+        first_items = separator.join(formatted_items[:-1])
+        last_item = formatted_items[-1]
+        return (
+            '{prefix_plural} {first_items} {conjunction} {last_item}'
+        ).format(**locals())
+
+
 @login_manager.user_loader
 def load_user(user_id):
     return user_api_client.get_user(user_id)

app/main/forms.py

@@ -23,7 +23,7 @@ from wtforms import (
 from wtforms.fields.html5 import EmailField, TelField
 from wtforms.validators import (DataRequired, Email, Length, Regexp, Optional)
 
-from app.main.validators import (Blacklist, CsvFileValidator, ValidGovEmail, NoCommasInPlaceHolders)
+from app.main.validators import (Blacklist, CsvFileValidator, ValidGovEmail, NoCommasInPlaceHolders, OnlyGSMCharacters)
 
 
 def get_time_value_and_label(future_time):
@@ -260,7 +260,8 @@ class SMSTemplateForm(Form):
         u'Message',
         validators=[
             DataRequired(message="Can’t be empty"),
-            NoCommasInPlaceHolders()
+            NoCommasInPlaceHolders(),
+            OnlyGSMCharacters()
         ]
     )
     process_type = RadioField(

app/main/validators.py

@@ -1,13 +1,15 @@
 from wtforms import ValidationError
 from notifications_utils.template import Template
+from notifications_utils.gsm import get_non_gsm_characters
+
+from app.main._blacklisted_passwords import blacklisted_passwords
 from app.utils import (
     Spreadsheet,
     is_gov_user
 )
-from ._blacklisted_passwords import blacklisted_passwords
 
 
-class Blacklist(object):
+class Blacklist:
     def __init__(self, message=None):
         if not message:
             message = 'Password is blacklisted.'
@@ -18,7 +20,7 @@ class Blacklist(object):
             raise ValidationError(self.message)
 
 
-class CsvFileValidator(object):
+class CsvFileValidator:
 
     def __init__(self, message='Not a csv file'):
         self.message = message
@@ -28,7 +30,7 @@ class CsvFileValidator(object):
             raise ValidationError("{} isn’t a spreadsheet that Notify can read".format(field.data.filename))
 
 
-class ValidGovEmail(object):
+class ValidGovEmail:
 
     def __call__(self, form, field):
         from flask import url_for
@@ -40,7 +42,7 @@ class ValidGovEmail(object):
             raise ValidationError(message)
 
 
-class NoCommasInPlaceHolders():
+class NoCommasInPlaceHolders:
 
     def __init__(self, message='You can’t have commas in your fields'):
         self.message = message
@@ -48,3 +50,12 @@ class NoCommasInPlaceHolders():
     def __call__(self, form, field):
         if ',' in ''.join(Template({'content': field.data}).placeholders):
             raise ValidationError(self.message)
+
+
+class OnlyGSMCharacters:
+    def __call__(self, form, field):
+        non_gsm_characters = get_non_gsm_characters(field.data)
+        if non_gsm_characters:
+            raise ValidationError('The following characters are not allowed in text messages: {}'.format(
+                ', '.join(non_gsm_characters)
+            ))

app/templates/components/list.html

@@ -1,35 +1,5 @@
-{% macro formatted_list(
-  items,
-  conjunction='and',
-  before_each='‘',
-  after_each='’',
-  separator=', ',
-  prefix='',
-  prefix_plural=''
-) %}
-  {% if items|length == 1 %}
-    {{ prefix }} {{ before_each|safe }}{{ (items|list)[0] }}{{ after_each|safe }}
-  {% elif items %}
-    {{ prefix_plural }}
-    {% for item in (items|list)[0:-1] -%}
-      {{ before_each|safe -}}
-      {{ item -}}
-      {{ after_each|safe -}}
-      {% if not loop.last -%}
-        {{ separator -}}
-      {% endif -%}
-    {% endfor %}
-    {{ conjunction }}
-    {{ before_each|safe -}}
-    {{ (items|list)[-1] -}}
-    {{ after_each|safe }}
-  {%- endif %}
-{%- endmacro %}
-
-
 {% macro list_of_placeholders(placeholders) %}
-  {{ formatted_list(
-    placeholders,
+  {{ placeholders | formatted_list(
     before_each="<span class='placeholder'>((",
     after_each='))</span>',
     separator=' '

app/templates/views/check.html

@@ -4,7 +4,6 @@
 {% from "components/table.html" import list_table, field, text_field, index_field, hidden_field_heading %}
 {% from "components/file-upload.html" import file_upload %}
 {% from "components/page-footer.html" import page_footer %}
-{% from "components/list.html" import formatted_list %}
 {% from "components/message-count-label.html" import message_count_label %}
 
 {% set file_contents_header_id = 'file-preview' %}
@@ -41,15 +40,13 @@
     <div class="bottom-gutter">
       {% call banner_wrapper(type='dangerous') %}
         <h1 class='banner-title'>
-          Your file needs to have {{ formatted_list(
-            recipients.recipient_column_headers,
+          Your file needs to have {{ recipients.recipient_column_headers | formatted_list(
             prefix='a column called',
             prefix_plural='columns called'
           ) }}
         </h1>
         <p>
-          Your file has {{ formatted_list(
-            recipients.column_headers,
+          Your file has {{ recipients.column_headers | formatted_list(
             prefix='one column, called',
             prefix_plural='columns called'
           ) }}.
@@ -67,15 +64,13 @@
           your template
         </h1>
         <p>
-          Your file has {{ formatted_list(
-            recipients.column_headers,
+          Your file has {{ recipients.column_headers | formatted_list(
             prefix='one column, called',
             prefix_plural='columns called'
           ) }}.
         </p>
         <p>
-          It doesn’t have {{ formatted_list(
-            recipients.missing_column_headers,
+          It doesn’t have {{ recipients.column_headers | formatted_list(
             conjunction='or',
             prefix='a column called',
             prefix_plural='columns called'

app/templates/views/styleguide.html

@@ -8,7 +8,6 @@
 {% from "components/textbox.html" import textbox %}
 {% from "components/file-upload.html" import file_upload %}
 {% from "components/api-key.html" import api_key %}
-{% from "components/list.html" import formatted_list %}
 
 {% block per_page_title %}
   Styleguide
@@ -196,19 +195,19 @@
   <h2 class="heading-large">Formatted list</h2>
 
   <p>
-    {{ formatted_list('A', prefix="one item called") }}
+    {{ 'A' | formatted_list(prefix="one item called") }}
   </p>
 
   <p>
-    {{ formatted_list('AB', prefix_plural="two items called") }}
+    {{ 'AB' | formatted_list(prefix_plural="two items called") }}
   </p>
 
   <p>
-    {{ formatted_list('ABC') }}
+    {{ 'ABC' | formatted_list }}
   </p>
 
   <p>
-    {{ formatted_list('ABCD', before_each='<strike>', after_each='</strike>', conjunction='or') }}
+    {{ 'ABCD' | formatted_list(before_each='<strike>', after_each='</strike>', conjunction='or') }}
   </p>
 
 {% endblock %}