Save whole letter address into the to field

At the moment we’re not consistent:

Precompiled (API and one-off):
`to` has the whole address
`normalised_to` has nothing

Templated (API, CSV and one off):
`to` has the first line of the address
`normalised_to` has nothing

This commit makes us consistently store the whole address in the `to`
field. We think that people might want to search by postcode, not just
first line of the address.

This commit also starts to populate the normalised_to field with the
address lowercased and with all spaces removed, to make it easier to
search on.

app/celery/tasks.py

@@ -345,7 +345,7 @@ def save_letter(
     # we store the recipient as just the first item of the person's address
     recipient = PostalAddress.from_personalisation(
         Columns(notification['personalisation'])
-    ).normalised_lines[0]
+    ).normalised
 
     service = dao_fetch_service_by_id(service_id)
     template = dao_get_template_by_id(notification['template'], version=notification['template_version'])

app/notifications/process_letter_notifications.py

@@ -1,3 +1,5 @@
+from notifications_utils.postal_address import PostalAddress
+
 from app import create_random_identifier
 from app.models import LETTER_TYPE
 from app.notifications.process_notifications import persist_notification
@@ -9,7 +11,7 @@ def create_letter_notification(letter_data, template, api_key, status, reply_to_
         template_version=template.version,
         template_postage=template.postage,
         # we only accept addresses_with_underscores from the API (from CSV we also accept dashes, spaces etc)
-        recipient=letter_data['personalisation']['address_line_1'],
+        recipient=PostalAddress.from_personalisation(letter_data['personalisation']).normalised,
         service=template.service,
         personalisation=letter_data['personalisation'],
         notification_type=LETTER_TYPE,

app/notifications/process_notifications.py

@@ -109,6 +109,7 @@ def persist_notification(
         notification.normalised_to = format_email_address(notification.to)
     elif notification_type == LETTER_TYPE:
         notification.postage = postage or template_postage
+        notification.normalised_to = ''.join(notification.to.split()).lower()
 
     # if simulated create a Notification model to return but do not persist the Notification to the dB
     if not simulated:

tests/app/celery/test_tasks.py

@@ -950,8 +950,8 @@ def test_save_sms_does_not_send_duplicate_and_does_not_put_in_retry_queue(sample
     assert not retry.called
 
 
-@pytest.mark.parametrize('personalisation', (
+@pytest.mark.parametrize('personalisation, expected_to, expected_normalised', (
-    {
+    ({
         'addressline1': 'Foo',
         'addressline2': 'Bar',
         'addressline3': 'Baz',
@@ -959,17 +959,39 @@ def test_save_sms_does_not_send_duplicate_and_does_not_put_in_retry_queue(sample
         'addressline5': 'Wobble',
         'addressline6': 'Wubble',
         'postcode': 'SE1 2SA',
-    },
+    }, (
-    {
+        'Foo\n'
+        'Bar\n'
+        'Baz\n'
+        'Wibble\n'
+        'Wobble\n'
+        'Wubble\n'
+        'SE1 2SA'
+    ), (
+        'foobarbazwibblewobblewubblese12sa'
+    )),
+    ({
         # The address isn’t normalised when we store it in the
-        # `personalisation` column, but the first line is normalised for
+        # `personalisation` column, but is normalised for storing in the
-        # Storing in the `to` column
+        # `to` column
         'addressline2': '    Foo    ',
         'addressline4': 'Bar',
         'addressline6': 'se12sa',
-    },
+    }, (
+        'Foo\n'
+        'Bar\n'
+        'SE1 2SA'
+    ), (
+        'foobarse12sa'
+    )),
 ))
-def test_save_letter_saves_letter_to_database(mocker, notify_db_session, personalisation):
+def test_save_letter_saves_letter_to_database(
+    mocker,
+    notify_db_session,
+    personalisation,
+    expected_to,
+    expected_normalised,
+):
     service = create_service()
     contact_block = create_letter_contact(service=service, contact_block="Address contact", is_default=True)
     template = create_template(service=service, template_type=LETTER_TYPE, reply_to=contact_block.id)
@@ -996,7 +1018,8 @@ def test_save_letter_saves_letter_to_database(mocker, notify_db_session, persona
 
     notification_db = Notification.query.one()
     assert notification_db.id == notification_id
-    assert notification_db.to == 'Foo'
+    assert notification_db.to == expected_to
+    assert notification_db.normalised_to == expected_normalised
     assert notification_db.job_id == job.id
     assert notification_db.template_id == job.template.id
     assert notification_db.template_version == job.template.version
@@ -1097,7 +1120,15 @@ def test_save_letter_saves_letter_to_database_right_reply_to(mocker, notify_db_s
 
     notification_db = Notification.query.one()
     assert notification_db.id == notification_id
-    assert notification_db.to == 'Foo'
+    assert notification_db.to == (
+        'Foo\n'
+        'Bar\n'
+        'Baz\n'
+        'Wibble\n'
+        'Wobble\n'
+        'Wubble\n'
+        'SE1 3WS'
+    )
     assert notification_db.job_id == job.id
     assert notification_db.template_id == job.template.id
     assert notification_db.template_version == job.template.version