From f6950ae987b9cd829ee6ccf552b8ab60699b8820 Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Mon, 6 Nov 2017 12:36:26 +0000
Subject: [PATCH] Stop escaping special characters in inbound
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

At least one of our providers gives us messages with special characters
escaped, ie a newline comes through as `\n`, not a literal newline. We
shouldn’t be showing these backslashes to any of our users.

Python has built in codecs for dealing with encoding/decoding of
strings – see
https://docs.python.org/3/library/codecs.html#text-encodings
for details. Using these builtins is safer than trying to do anything
regex or parsing-based.
---
 app/main/views/conversation.py            |  2 +-
 app/main/views/dashboard.py               |  3 +++
 tests/app/main/views/test_conversation.py | 23 +++++++++++++++++++++++
 tests/app/main/views/test_dashboard.py    | 19 +++++++++++++++++++
 tests/conftest.py                         | 20 ++++++++++++++++++++
 5 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/app/main/views/conversation.py b/app/main/views/conversation.py
index 3d1c7ccc4..e975fe244 100644
--- a/app/main/views/conversation.py
+++ b/app/main/views/conversation.py
@@ -133,6 +133,6 @@ def get_sms_thread(service_id, user_number):
 
 def get_sms_content(notification, is_inbound):
     return (
-        notification['content'] if is_inbound else
+        bytes(notification['content'], "utf-8").decode('unicode_escape') if is_inbound else
         notification['template']['content']
     )
diff --git a/app/main/views/dashboard.py b/app/main/views/dashboard.py
index ce63fdc00..b81da435f 100644
--- a/app/main/views/dashboard.py
+++ b/app/main/views/dashboard.py
@@ -203,6 +203,9 @@ def get_inbox_partials(service_id):
             format_phone_number_human_readable(message['user_number'])
             for message in messages_to_show
         }:
+            message.update({
+                'content': bytes(message['content'], 'utf-8').decode('unicode_escape')
+            })
             messages_to_show.append(message)
 
     if not inbound_messages:
diff --git a/tests/app/main/views/test_conversation.py b/tests/app/main/views/test_conversation.py
index dba58a14c..f4edb3685 100644
--- a/tests/app/main/views/test_conversation.py
+++ b/tests/app/main/views/test_conversation.py
@@ -161,6 +161,29 @@ def test_view_conversation(
         ) == expected
 
 
+def test_escaped_characters_in_inbound_messages(
+    client_request,
+    mock_get_notification,
+    mock_get_notifications,
+    mock_get_inbound_sms_with_special_characters,
+    fake_uuid,
+):
+
+    page = client_request.get(
+        'main.conversation',
+        service_id=SERVICE_ONE_ID,
+        notification_id=fake_uuid,
+    )
+
+    assert normalize_spaces(
+        str(page.select_one('.sms-message-inbound .sms-message-wrapper'))
+    ) == (
+        "<div class=\"sms-message-wrapper\"> "
+        "the first line's content<br/>the second line's content "
+        "</div>"
+    )
+
+
 def test_view_conversation_updates(
     logged_in_client,
     mocker,
diff --git a/tests/app/main/views/test_dashboard.py b/tests/app/main/views/test_dashboard.py
index 35a68dea4..b1787c691 100644
--- a/tests/app/main/views/test_dashboard.py
+++ b/tests/app/main/views/test_dashboard.py
@@ -172,6 +172,25 @@ def test_inbox_showing_inbound_messages(
     )
 
 
+def test_inbox_handles_escaped_characters(
+    client_request,
+    service_one,
+    mock_get_inbound_sms_with_special_characters,
+):
+
+    service_one['permissions'] = ['inbound_sms']
+
+    page = client_request.get('main.inbox', service_id=SERVICE_ONE_ID)
+
+    assert normalize_spaces(
+        str(page.select_one('tbody tr .file-list-hint'))
+    ) == (
+        "<span class=\"file-list-hint\">"
+        "the first line's content the second line's content"
+        "</span>"
+    )
+
+
 def test_empty_inbox(
     logged_in_client,
     service_one,
diff --git a/tests/conftest.py b/tests/conftest.py
index e1341994e..b61f228a2 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1762,6 +1762,26 @@ def mock_get_inbound_sms(mocker):
     )
 
 
+@pytest.fixture(scope='function')
+def mock_get_inbound_sms_with_special_characters(mocker):
+    def _get_inbound_sms(
+        service_id,
+        user_number=None,
+    ):
+        return [{
+            'user_number': '07900900001',
+            'notify_number': '07900000002',
+            'content': "the first line\\'s content\\nthe second line\\'s content",
+            'created_at': datetime.utcnow().isoformat(),
+            'id': sample_uuid(),
+        }]
+
+    return mocker.patch(
+        'app.service_api_client.get_inbound_sms',
+        side_effect=_get_inbound_sms,
+    )
+
+
 @pytest.fixture(scope='function')
 def mock_get_inbound_sms_with_no_messages(mocker):
     def _get_inbound_sms(