Bump utils and sanitise header values

Updated notifications-utils. This brings in
- the renamed character sanitization classes
- the change to allow unicode in letter addresses (this lets us delete
a test that is no longer relevant)

Also replaced non-ascii characters in headers. This fixes a bug where
non-ascii characters in a CSV filename were causing errors since the
filename is also used in the header.

app/__init__.py

@@ -33,6 +33,7 @@ from notifications_utils.recipients import (
     format_phone_number_human_readable,
 )
 from notifications_utils.formatters import formatted_list
+from notifications_utils.sanitise_text import SanitiseASCII
 from werkzeug.exceptions import abort
 from werkzeug.local import LocalProxy
 
@@ -493,6 +494,8 @@ def useful_headers_after_request(response):
         del response.headers['Cache-Control']
     response.headers.add(
         'Cache-Control', 'no-store, no-cache, private, must-revalidate')
+    for key, value in response.headers:
+        response.headers[key] = SanitiseASCII.encode(value)
     return response
 
 

app/main/validators.py

@@ -1,11 +1,11 @@
 import re
 
 from notifications_utils.field import Field
-from notifications_utils.gsm import get_non_gsm_compatible_characters
 from notifications_utils.recipients import (
     InvalidEmailError,
     validate_email_address,
 )
+from notifications_utils.sanitise_text import SanitiseGSM
 from wtforms import ValidationError
 from wtforms.validators import Email
 
@@ -72,7 +72,7 @@ class NoCommasInPlaceHolders:
 
 class OnlyGSMCharacters:
     def __call__(self, form, field):
-        non_gsm_characters = sorted(list(get_non_gsm_compatible_characters(field.data)))
+        non_gsm_characters = sorted(list(SanitiseGSM.get_non_compatible_characters(field.data)))
         if non_gsm_characters:
             raise ValidationError(
                 'You can’t use {} in text messages. {} won’t show up properly on everyone’s phones.'.format(

requirements.txt

@@ -18,4 +18,4 @@ notifications-python-client==4.8.2
 # PaaS
 awscli-cwlogs>=1.4,<1.5
 
-git+https://github.com/alphagov/notifications-utils.git@27.1.1#egg=notifications-utils==27.1.1
+git+https://github.com/alphagov/notifications-utils.git@29.0.0#egg=notifications-utils==29.0.0

tests/app/main/views/test_headers.py

@@ -16,3 +16,20 @@ def test_owasp_useful_headers_set(client, mocker):
         "img-src 'self' *.google-analytics.com *.notifications.service.gov.uk static-logos.test.com data:;"
         "frame-src www.youtube.com;"
     )
+
+
+def test_headers_non_ascii_characters_are_replaced(client, mocker):
+    mocker.patch('app.get_cdn_domain', return_value='static-logos۾.test.com')
+
+    response = client.get('/')
+
+    assert response.status_code == 200
+    assert response.headers['Content-Security-Policy'] == (
+        "default-src 'self' 'unsafe-inline';"
+        "script-src 'self' *.google-analytics.com 'unsafe-inline' 'unsafe-eval' data:;"
+        "connect-src 'self' *.google-analytics.com;"
+        "object-src 'self';"
+        "font-src 'self' data:;"
+        "img-src 'self' *.google-analytics.com *.notifications.service.gov.uk static-logos??.test.com data:;"
+        "frame-src www.youtube.com;"
+    )

tests/app/main/views/test_send.py

@@ -2382,55 +2382,6 @@ def test_check_messages_shows_over_max_row_error(
     )
 
 
-def test_non_ascii_characters_in_letter_recipients_file_shows_error(
-    logged_in_client,
-    api_user_active,
-    mock_login,
-    mock_get_users_by_service,
-    mock_get_live_service,
-    mock_has_permissions,
-    mock_get_service_letter_template,
-    mock_get_service_statistics,
-    mock_get_job_doesnt_exist,
-    fake_uuid,
-    mocker
-):
-    from tests.conftest import mock_s3_download
-    mock_s3_download(
-        mocker,
-        content=u"""
-        address line 1,address line 2,address line 3,address line 4,address line 5,address line 6,postcode
-        Петя,345 Example Street,,,,,AA1 6BB
-        """
-    )
-
-    with logged_in_client.session_transaction() as session:
-        session['file_uploads'] = {
-            fake_uuid: {
-                'template_id': fake_uuid,
-            }
-        }
-
-    response = logged_in_client.get(url_for(
-        'main.check_messages',
-        service_id=fake_uuid,
-        template_id=fake_uuid,
-        upload_id=fake_uuid,
-        original_file_name='unicode.csv'
-    ))
-
-    assert response.status_code == 200
-    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
-    assert ' '.join(
-        page.find('div', class_='banner-dangerous').text.split()
-    ) == (
-        'There is a problem with unicode.csv '
-        'You need to fix 1 address '
-        'Skip to file contents'
-    )
-    assert page.find('span', class_='table-field-error-label').text == u'Can’t include П, е, т or я'
-
-
 @pytest.mark.parametrize('existing_session_items', [
     {},
     {'recipient': '07700900001'},