Localize notification_utils to the admin

This changeset pulls in all of the notification_utils code directly into the admin and removes it as an external dependency. We are doing this to cut down on operational maintenance of the project and will begin removing parts of it no longer needed for the admin.

Signed-off-by: Carlo Costino <carlo.costino@gsa.gov>

tests/notifications_utils/clients/antivirus/test_antivirus_client.py

@@ -0,0 +1,63 @@
+import io
+
+import pytest
+import requests
+
+from notifications_utils.clients.antivirus.antivirus_client import (
+    AntivirusClient,
+    AntivirusError,
+)
+
+
+@pytest.fixture(scope="function")
+def antivirus(app, mocker):
+    client = AntivirusClient()
+    app.config["ANTIVIRUS_API_HOST"] = "https://antivirus"
+    app.config["ANTIVIRUS_API_KEY"] = "test-antivirus-key"
+    client.init_app(app)
+    return client
+
+
+def test_scan_document(antivirus, rmock):
+    document = io.BytesIO(b"filecontents")
+    rmock.request(
+        "POST",
+        "https://antivirus/scan",
+        json={"ok": True},
+        request_headers={
+            "Authorization": "Bearer test-antivirus-key",
+        },
+        status_code=200,
+    )
+
+    resp = antivirus.scan(document)
+
+    assert resp
+    assert "filecontents" in rmock.last_request.text
+    assert document.tell() == 0
+
+
+def test_should_raise_for_status(antivirus, rmock):
+    with pytest.raises(AntivirusError) as excinfo:
+        rmock.request(
+            "POST",
+            "https://antivirus/scan",
+            json={"error": "Antivirus error"},
+            status_code=400,
+        )
+
+        antivirus.scan(io.BytesIO(b"document"))
+
+    assert excinfo.value.message == "Antivirus error"
+    assert excinfo.value.status_code == 400
+
+
+def test_should_raise_for_connection_errors(antivirus, rmock):
+    with pytest.raises(AntivirusError) as excinfo:
+        rmock.request(
+            "POST", "https://antivirus/scan", exc=requests.exceptions.ConnectTimeout
+        )
+        antivirus.scan(io.BytesIO(b"document"))
+
+    assert excinfo.value.message == "connection error"
+    assert excinfo.value.status_code == 503

tests/notifications_utils/clients/encryption/test_encryption_client.py

@@ -0,0 +1,88 @@
+import pytest
+
+from notifications_utils.clients.encryption.encryption_client import (
+    Encryption,
+    EncryptionError,
+)
+
+
+@pytest.fixture()
+def encryption_client(app):
+    client = Encryption()
+
+    app.config["SECRET_KEY"] = "test-notify-secret-key"
+    app.config["DANGEROUS_SALT"] = "test-notify-salt"
+
+    client.init_app(app)
+
+    return client
+
+
+def test_should_ensure_shared_salt_security(app):
+    client = Encryption()
+    app.config["SECRET_KEY"] = "test-notify-secret-key"
+    app.config["DANGEROUS_SALT"] = "too-short"
+    with pytest.raises(EncryptionError):
+        client.init_app(app)
+
+
+def test_should_ensure_custom_salt_security(encryption_client):
+    with pytest.raises(EncryptionError):
+        encryption_client.encrypt("this", salt="too-short")
+
+
+def test_should_encrypt_strings(encryption_client):
+    encrypted = encryption_client.encrypt("this")
+    assert encrypted != "this"
+    assert isinstance(encrypted, str)
+
+
+def test_should_encrypt_dicts(encryption_client):
+    to_encrypt = {"hello": "world"}
+    encrypted = encryption_client.encrypt(to_encrypt)
+    assert encrypted != to_encrypt
+    assert encryption_client.decrypt(encrypted) == to_encrypt
+
+
+def test_encryption_is_nondeterministic(encryption_client):
+    first_run = encryption_client.encrypt("this")
+    second_run = encryption_client.encrypt("this")
+    assert first_run != second_run
+
+
+def test_should_decrypt_content(encryption_client):
+    encrypted = encryption_client.encrypt("this")
+    assert encryption_client.decrypt(encrypted) == "this"
+
+
+def test_should_decrypt_content_with_custom_salt(encryption_client):
+    salt = "different-salt-value"
+    encrypted = encryption_client.encrypt("this", salt=salt)
+    assert encryption_client.decrypt(encrypted, salt=salt) == "this"
+
+
+def test_should_verify_decryption(encryption_client):
+    encrypted = encryption_client.encrypt("this")
+    with pytest.raises(EncryptionError):
+        encryption_client.decrypt(encrypted, salt="different-salt-value")
+
+
+def test_should_sign_and_serialize_string(encryption_client):
+    signed = encryption_client.sign("this")
+    assert signed != "this"
+
+
+def test_should_verify_signature_and_deserialize_string(encryption_client):
+    signed = encryption_client.sign("this")
+    assert encryption_client.verify_signature(signed) == "this"
+
+
+def test_should_raise_encryption_error_on_bad_salt(encryption_client):
+    signed = encryption_client.sign("this")
+    with pytest.raises(EncryptionError):
+        encryption_client.verify_signature(signed, salt="different-salt-value")
+
+
+def test_should_sign_and_serialize_json(encryption_client):
+    signed = encryption_client.sign({"this": "that"})
+    assert encryption_client.verify_signature(signed) == {"this": "that"}

tests/notifications_utils/clients/redis/test_redis_client.py

@@ -0,0 +1,221 @@
+import uuid
+from datetime import datetime
+from unittest.mock import Mock, call
+
+import pytest
+from freezegun import freeze_time
+
+from notifications_utils.clients.redis.redis_client import RedisClient, prepare_value
+
+
+@pytest.fixture(scope="function")
+def mocked_redis_pipeline():
+    return Mock()
+
+
+@pytest.fixture
+def delete_mock():
+    return Mock(return_value=4)
+
+
+@pytest.fixture(scope="function")
+def mocked_redis_client(app, mocked_redis_pipeline, delete_mock, mocker):
+    app.config["REDIS_ENABLED"] = True
+
+    redis_client = RedisClient()
+    redis_client.init_app(app)
+
+    mocker.patch.object(redis_client.redis_store, "get", return_value=100)
+    mocker.patch.object(redis_client.redis_store, "set")
+    mocker.patch.object(redis_client.redis_store, "incr")
+    mocker.patch.object(redis_client.redis_store, "delete")
+    mocker.patch.object(
+        redis_client.redis_store, "pipeline", return_value=mocked_redis_pipeline
+    )
+
+    mocker.patch.object(
+        redis_client, "scripts", {"delete-keys-by-pattern": delete_mock}
+    )
+
+    mocker.patch.object(
+        redis_client.redis_store,
+        "hgetall",
+        return_value={b"template-1111": b"8", b"template-2222": b"8"},
+    )
+
+    return redis_client
+
+
+@pytest.fixture
+def failing_redis_client(mocked_redis_client, delete_mock):
+    mocked_redis_client.redis_store.get.side_effect = Exception("get failed")
+    mocked_redis_client.redis_store.set.side_effect = Exception("set failed")
+    mocked_redis_client.redis_store.incr.side_effect = Exception("incr failed")
+    mocked_redis_client.redis_store.pipeline.side_effect = Exception("pipeline failed")
+    mocked_redis_client.redis_store.delete.side_effect = Exception("delete failed")
+    delete_mock.side_effect = Exception("delete by pattern failed")
+    return mocked_redis_client
+
+
+def test_should_not_raise_exception_if_raise_set_to_false(
+    app, caplog, failing_redis_client, mocker
+):
+    mock_logger = mocker.patch("flask.Flask.logger")
+
+    assert failing_redis_client.get("get_key") is None
+    assert failing_redis_client.set("set_key", "set_value") is None
+    assert failing_redis_client.incr("incr_key") is None
+    assert failing_redis_client.exceeded_rate_limit("rate_limit_key", 100, 100) is False
+    assert failing_redis_client.delete("delete_key") is None
+    assert failing_redis_client.delete("a", "b", "c") is None
+    assert failing_redis_client.delete_by_pattern("pattern") == 0
+
+    assert mock_logger.mock_calls == [
+        call.exception("Redis error performing get on get_key"),
+        call.exception("Redis error performing set on set_key"),
+        call.exception("Redis error performing incr on incr_key"),
+        call.exception("Redis error performing rate-limit-pipeline on rate_limit_key"),
+        call.exception("Redis error performing delete on delete_key"),
+        call.exception("Redis error performing delete on a, b, c"),
+        call.exception("Redis error performing delete-by-pattern on pattern"),
+    ]
+
+
+def test_should_raise_exception_if_raise_set_to_true(
+    app,
+    failing_redis_client,
+):
+    with pytest.raises(Exception) as e:
+        failing_redis_client.get("test", raise_exception=True)
+    assert str(e.value) == "get failed"
+
+    with pytest.raises(Exception) as e:
+        failing_redis_client.set("test", "test", raise_exception=True)
+    assert str(e.value) == "set failed"
+
+    with pytest.raises(Exception) as e:
+        failing_redis_client.incr("test", raise_exception=True)
+    assert str(e.value) == "incr failed"
+
+    with pytest.raises(Exception) as e:
+        failing_redis_client.exceeded_rate_limit("test", 100, 200, raise_exception=True)
+    assert str(e.value) == "pipeline failed"
+
+    with pytest.raises(Exception) as e:
+        failing_redis_client.delete("test", raise_exception=True)
+    assert str(e.value) == "delete failed"
+
+    with pytest.raises(Exception) as e:
+        failing_redis_client.delete_by_pattern("pattern", raise_exception=True)
+    assert str(e.value) == "delete by pattern failed"
+
+
+def test_should_not_call_if_not_enabled(mocked_redis_client, delete_mock):
+    mocked_redis_client.active = False
+
+    assert mocked_redis_client.get("get_key") is None
+    assert mocked_redis_client.set("set_key", "set_value") is None
+    assert mocked_redis_client.incr("incr_key") is None
+    assert mocked_redis_client.exceeded_rate_limit("rate_limit_key", 100, 100) is False
+    assert mocked_redis_client.delete("delete_key") is None
+    assert mocked_redis_client.delete_by_pattern("pattern") == 0
+
+    mocked_redis_client.redis_store.get.assert_not_called()
+    mocked_redis_client.redis_store.set.assert_not_called()
+    mocked_redis_client.redis_store.incr.assert_not_called()
+    mocked_redis_client.redis_store.delete.assert_not_called()
+    mocked_redis_client.redis_store.pipeline.assert_not_called()
+    delete_mock.assert_not_called()
+
+
+def test_should_call_set_if_enabled(mocked_redis_client):
+    mocked_redis_client.set("key", "value")
+    mocked_redis_client.redis_store.set.assert_called_with(
+        "key", "value", None, None, False, False
+    )
+
+
+def test_should_call_get_if_enabled(mocked_redis_client):
+    assert mocked_redis_client.get("key") == 100
+    mocked_redis_client.redis_store.get.assert_called_with("key")
+
+
+@freeze_time("2001-01-01 12:00:00.000000")
+def test_exceeded_rate_limit_should_add_correct_calls_to_the_pipe(
+    mocked_redis_client, mocked_redis_pipeline
+):
+    mocked_redis_client.exceeded_rate_limit("key", 100, 100)
+    assert mocked_redis_client.redis_store.pipeline.called
+    mocked_redis_pipeline.zadd.assert_called_with("key", {978350400.0: 978350400.0})
+    mocked_redis_pipeline.zremrangebyscore.assert_called_with(
+        "key", "-inf", 978350300.0
+    )
+    mocked_redis_pipeline.zcard.assert_called_with("key")
+    mocked_redis_pipeline.expire.assert_called_with("key", 100)
+    assert mocked_redis_pipeline.execute.called
+
+
+@freeze_time("2001-01-01 12:00:00.000000")
+def test_exceeded_rate_limit_should_fail_request_if_over_limit(
+    mocked_redis_client, mocked_redis_pipeline
+):
+    mocked_redis_pipeline.execute.return_value = [True, True, 100, True]
+    assert mocked_redis_client.exceeded_rate_limit("key", 99, 100)
+
+
+@freeze_time("2001-01-01 12:00:00.000000")
+def test_exceeded_rate_limit_should_allow_request_if_not_over_limit(
+    mocked_redis_client, mocked_redis_pipeline
+):
+    mocked_redis_pipeline.execute.return_value = [True, True, 100, True]
+    assert not mocked_redis_client.exceeded_rate_limit("key", 101, 100)
+
+
+@freeze_time("2001-01-01 12:00:00.000000")
+def test_exceeded_rate_limit_not_exceeded(mocked_redis_client, mocked_redis_pipeline):
+    mocked_redis_pipeline.execute.return_value = [True, True, 80, True]
+    assert not mocked_redis_client.exceeded_rate_limit("key", 90, 100)
+
+
+def test_exceeded_rate_limit_should_not_call_if_not_enabled(
+    mocked_redis_client, mocked_redis_pipeline
+):
+    mocked_redis_client.active = False
+
+    assert not mocked_redis_client.exceeded_rate_limit("key", 100, 100)
+    assert not mocked_redis_client.redis_store.pipeline.called
+
+
+def test_delete(mocked_redis_client):
+    key = "hash-key"
+    mocked_redis_client.delete(key)
+    mocked_redis_client.redis_store.delete.assert_called_with(key)
+
+
+def test_delete_multi(mocked_redis_client):
+    mocked_redis_client.delete("a", "b", "c")
+    mocked_redis_client.redis_store.delete.assert_called_with("a", "b", "c")
+
+
+@pytest.mark.parametrize(
+    "input,output",
+    [
+        (b"asdf", b"asdf"),
+        ("asdf", "asdf"),
+        (0, 0),
+        (1.2, 1.2),
+        (uuid.UUID(int=0), "00000000-0000-0000-0000-000000000000"),
+        pytest.param({"a": 1}, None, marks=pytest.mark.xfail(raises=ValueError)),
+        pytest.param(
+            datetime.utcnow(), None, marks=pytest.mark.xfail(raises=ValueError)
+        ),
+    ],
+)
+def test_prepare_value(input, output):
+    assert prepare_value(input) == output
+
+
+def test_delete_by_pattern(mocked_redis_client, delete_mock):
+    ret = mocked_redis_client.delete_by_pattern("foo")
+    assert ret == 4
+    delete_mock.assert_called_once_with(args=["foo"])

tests/notifications_utils/clients/redis/test_request_cache.py

@@ -0,0 +1,190 @@
+import pytest
+
+from notifications_utils.clients.redis import RequestCache
+from notifications_utils.clients.redis.redis_client import RedisClient
+
+
+@pytest.fixture(scope="function")
+def mocked_redis_client(app):
+    app.config["REDIS_ENABLED"] = True
+    redis_client = RedisClient()
+    redis_client.init_app(app)
+    return redis_client
+
+
+@pytest.fixture
+def cache(mocked_redis_client):
+    return RequestCache(mocked_redis_client)
+
+
+@pytest.mark.parametrize(
+    "args, kwargs, expected_cache_key",
+    (
+        ([1, 2, 3], {}, "1-2-3-None-None-None"),
+        ([1, 2, 3, 4, 5, 6], {}, "1-2-3-4-5-6"),
+        ([1, 2, 3], {"x": 4, "y": 5, "z": 6}, "1-2-3-4-5-6"),
+        ([1, 2, 3, 4], {"y": 5}, "1-2-3-4-5-None"),
+    ),
+)
+def test_set(
+    mocker,
+    mocked_redis_client,
+    cache,
+    args,
+    kwargs,
+    expected_cache_key,
+):
+    mock_redis_set = mocker.patch.object(
+        mocked_redis_client,
+        "set",
+    )
+    mock_redis_get = mocker.patch.object(
+        mocked_redis_client,
+        "get",
+        return_value=None,
+    )
+
+    @cache.set("{a}-{b}-{c}-{x}-{y}-{z}")
+    def foo(a, b, c, x=None, y=None, z=None):
+        return "bar"
+
+    assert foo(*args, **kwargs) == "bar"
+
+    mock_redis_get.assert_called_once_with(expected_cache_key)
+
+    mock_redis_set.assert_called_once_with(
+        expected_cache_key,
+        '"bar"',
+        ex=604_800,
+    )
+
+
+@pytest.mark.parametrize(
+    "cache_set_call, expected_redis_client_ttl",
+    (
+        (0, 0),
+        (1, 1),
+        (1.111, 1),
+        ("2000", 2_000),
+    ),
+)
+def test_set_with_custom_ttl(
+    mocker,
+    mocked_redis_client,
+    cache,
+    cache_set_call,
+    expected_redis_client_ttl,
+):
+    mock_redis_set = mocker.patch.object(
+        mocked_redis_client,
+        "set",
+    )
+    mocker.patch.object(
+        mocked_redis_client,
+        "get",
+        return_value=None,
+    )
+
+    @cache.set("foo", ttl_in_seconds=cache_set_call)
+    def foo():
+        return "bar"
+
+    foo()
+
+    mock_redis_set.assert_called_once_with(
+        "foo",
+        '"bar"',
+        ex=expected_redis_client_ttl,
+    )
+
+
+def test_raises_if_key_doesnt_match_arguments(cache):
+    @cache.set("{baz}")
+    def foo(bar):
+        pass
+
+    with pytest.raises(KeyError):
+        foo(1)
+
+    with pytest.raises(KeyError):
+        foo()
+
+
+def test_get(mocker, mocked_redis_client, cache):
+    mock_redis_get = mocker.patch.object(
+        mocked_redis_client,
+        "get",
+        return_value=b'"bar"',
+    )
+
+    @cache.set("{a}-{b}-{c}")
+    def foo(a, b, c):
+        # This function should not be called because the cache has
+        # returned a value
+        raise RuntimeError
+
+    assert foo(1, 2, 3) == "bar"
+
+    mock_redis_get.assert_called_once_with("1-2-3")
+
+
+def test_delete(mocker, mocked_redis_client, cache):
+    mock_redis_delete = mocker.patch.object(
+        mocked_redis_client,
+        "delete",
+    )
+
+    @cache.delete("{a}-{b}-{c}")
+    def foo(a, b, c):
+        return "bar"
+
+    assert foo(1, 2, 3) == "bar"
+
+    mock_redis_delete.assert_called_once_with("1-2-3")
+
+
+def test_delete_even_if_call_raises(mocker, mocked_redis_client, cache):
+    mock_redis_delete = mocker.patch.object(
+        mocked_redis_client,
+        "delete",
+    )
+
+    @cache.delete("bar")
+    def foo():
+        raise RuntimeError
+
+    with pytest.raises(RuntimeError):
+        foo()
+
+    mock_redis_delete.assert_called_once_with("bar")
+
+
+def test_delete_by_pattern(mocker, mocked_redis_client, cache):
+    mock_redis_delete = mocker.patch.object(
+        mocked_redis_client,
+        "delete_by_pattern",
+    )
+
+    @cache.delete_by_pattern("{a}-{b}-{c}-???")
+    def foo(a, b, c):
+        return "bar"
+
+    assert foo(1, 2, 3) == "bar"
+
+    mock_redis_delete.assert_called_once_with("1-2-3-???")
+
+
+def test_delete_by_pattern_even_if_call_raises(mocker, mocked_redis_client, cache):
+    mock_redis_delete = mocker.patch.object(
+        mocked_redis_client,
+        "delete_by_pattern",
+    )
+
+    @cache.delete_by_pattern("bar-???")
+    def foo():
+        raise RuntimeError
+
+    with pytest.raises(RuntimeError):
+        foo()
+
+    mock_redis_delete.assert_called_once_with("bar-???")

tests/notifications_utils/clients/test_redis.py

@@ -0,0 +1,7 @@
+from notifications_utils.clients.redis import rate_limit_cache_key
+
+
+def test_rate_limit_cache_key(sample_service):
+    assert rate_limit_cache_key(sample_service.id, "TEST") == "{}-TEST".format(
+        sample_service.id
+    )

tests/notifications_utils/clients/zendesk/test_zendesk_client.py

@@ -0,0 +1,220 @@
+from base64 import b64decode
+
+import pytest
+
+from notifications_utils.clients.zendesk.zendesk_client import (
+    NotifySupportTicket,
+    ZendeskClient,
+    ZendeskError,
+)
+
+
+@pytest.fixture(scope="function")
+def zendesk_client(app):
+    client = ZendeskClient()
+
+    app.config["ZENDESK_API_KEY"] = "testkey"
+
+    client.init_app(app)
+
+    return client
+
+
+def test_zendesk_client_send_ticket_to_zendesk(zendesk_client, app, mocker, rmock):
+    rmock.request(
+        "POST",
+        ZendeskClient.ZENDESK_TICKET_URL,
+        status_code=201,
+        json={
+            "ticket": {
+                "id": 12345,
+                "subject": "Something is wrong",
+            }
+        },
+    )
+    mock_logger = mocker.patch.object(app.logger, "info")
+
+    ticket = NotifySupportTicket("subject", "message", "incident")
+    zendesk_client.send_ticket_to_zendesk(ticket)
+
+    assert rmock.last_request.headers["Authorization"][:6] == "Basic "
+    b64_auth = rmock.last_request.headers["Authorization"][6:]
+    assert (
+        b64decode(b64_auth.encode()).decode()
+        == "zd-api-notify@digital.cabinet-office.gov.uk/token:testkey"
+    )
+    assert rmock.last_request.json() == ticket.request_data
+    mock_logger.assert_called_once_with("Zendesk create ticket 12345 succeeded")
+
+
+def test_zendesk_client_send_ticket_to_zendesk_error(
+    zendesk_client, app, mocker, rmock
+):
+    rmock.request(
+        "POST", ZendeskClient.ZENDESK_TICKET_URL, status_code=401, json={"foo": "bar"}
+    )
+
+    mock_logger = mocker.patch.object(app.logger, "error")
+
+    ticket = NotifySupportTicket("subject", "message", "incident")
+
+    with pytest.raises(ZendeskError):
+        zendesk_client.send_ticket_to_zendesk(ticket)
+
+    mock_logger.assert_called_with(
+        "Zendesk create ticket request failed with 401 '{'foo': 'bar'}'"
+    )
+
+
+@pytest.mark.parametrize(
+    "p1_arg, expected_tags, expected_priority",
+    (
+        (
+            {},
+            ["govuk_notify_support"],
+            "normal",
+        ),
+        (
+            {
+                "p1": False,
+            },
+            ["govuk_notify_support"],
+            "normal",
+        ),
+        (
+            {
+                "p1": True,
+            },
+            ["govuk_notify_emergency"],
+            "urgent",
+        ),
+    ),
+)
+def test_notify_support_ticket_request_data(p1_arg, expected_tags, expected_priority):
+    notify_ticket_form = NotifySupportTicket("subject", "message", "question", **p1_arg)
+
+    assert notify_ticket_form.request_data == {
+        "ticket": {
+            "subject": "subject",
+            "comment": {
+                "body": "message",
+                "public": True,
+            },
+            "group_id": NotifySupportTicket.NOTIFY_GROUP_ID,
+            "organization_id": NotifySupportTicket.NOTIFY_ORG_ID,
+            "ticket_form_id": NotifySupportTicket.NOTIFY_TICKET_FORM_ID,
+            "priority": expected_priority,
+            "tags": expected_tags,
+            "type": "question",
+            "custom_fields": [
+                {"id": "1900000744994", "value": "notify_ticket_type_non_technical"},
+                {"id": "360022836500", "value": []},
+                {"id": "360022943959", "value": None},
+                {"id": "360022943979", "value": None},
+                {"id": "1900000745014", "value": None},
+            ],
+        }
+    }
+
+
+def test_notify_support_ticket_request_data_with_message_hidden_from_requester():
+    notify_ticket_form = NotifySupportTicket(
+        "subject", "message", "problem", requester_sees_message_content=False
+    )
+
+    assert notify_ticket_form.request_data["ticket"]["comment"]["public"] is False
+
+
+@pytest.mark.parametrize(
+    "name, zendesk_name", [("Name", "Name"), (None, "(no name supplied)")]
+)
+def test_notify_support_ticket_request_data_with_user_name_and_email(
+    name, zendesk_name
+):
+    notify_ticket_form = NotifySupportTicket(
+        "subject", "message", "question", user_name=name, user_email="user@example.com"
+    )
+
+    assert (
+        notify_ticket_form.request_data["ticket"]["requester"]["email"]
+        == "user@example.com"
+    )
+    assert (
+        notify_ticket_form.request_data["ticket"]["requester"]["name"] == zendesk_name
+    )
+
+
+@pytest.mark.parametrize(
+    "custom_fields, tech_ticket_tag, categories, org_id, org_type, service_id",
+    [
+        (
+            {"technical_ticket": True},
+            "notify_ticket_type_technical",
+            [],
+            None,
+            None,
+            None,
+        ),
+        (
+            {"technical_ticket": False},
+            "notify_ticket_type_non_technical",
+            [],
+            None,
+            None,
+            None,
+        ),
+        (
+            {"ticket_categories": ["notify_billing", "notify_bug"]},
+            "notify_ticket_type_non_technical",
+            ["notify_billing", "notify_bug"],
+            None,
+            None,
+            None,
+        ),
+        (
+            {"org_id": "1234", "org_type": "local"},
+            "notify_ticket_type_non_technical",
+            [],
+            "1234",
+            "notify_org_type_local",
+            None,
+        ),
+        (
+            {"service_id": "abcd", "org_type": "nhs"},
+            "notify_ticket_type_non_technical",
+            [],
+            None,
+            "notify_org_type_nhs",
+            "abcd",
+        ),
+    ],
+)
+def test_notify_support_ticket_request_data_custom_fields(
+    custom_fields,
+    tech_ticket_tag,
+    categories,
+    org_id,
+    org_type,
+    service_id,
+):
+    notify_ticket_form = NotifySupportTicket(
+        "subject", "message", "question", **custom_fields
+    )
+
+    assert notify_ticket_form.request_data["ticket"]["custom_fields"] == [
+        {"id": "1900000744994", "value": tech_ticket_tag},
+        {"id": "360022836500", "value": categories},
+        {"id": "360022943959", "value": org_id},
+        {"id": "360022943979", "value": org_type},
+        {"id": "1900000745014", "value": service_id},
+    ]
+
+
+def test_notify_support_ticket_request_data_email_ccs():
+    notify_ticket_form = NotifySupportTicket(
+        "subject", "message", "question", email_ccs=["someone@example.com"]
+    )
+
+    assert notify_ticket_form.request_data["ticket"]["email_ccs"] == [
+        {"user_email": "someone@example.com", "action": "put"},
+    ]

tests/notifications_utils/conftest.py

@@ -0,0 +1,45 @@
+import pytest
+import requests_mock
+from flask import Flask
+
+from notifications_utils import request_helper
+
+
+class FakeService:
+    id = "1234"
+
+
+@pytest.fixture
+def app():
+    flask_app = Flask(__name__)
+    ctx = flask_app.app_context()
+    ctx.push()
+
+    yield flask_app
+
+    ctx.pop()
+
+
+@pytest.fixture
+def celery_app(mocker):
+    app = Flask(__name__)
+    app.config["CELERY"] = {"broker_url": "foo"}
+    app.config["NOTIFY_TRACE_ID_HEADER"] = "Ex-Notify-Request-Id"
+    request_helper.init_app(app)
+
+    ctx = app.app_context()
+    ctx.push()
+
+    yield app
+    ctx.pop()
+
+
+@pytest.fixture(scope="session")
+def sample_service():
+    return FakeService()
+
+
+@pytest.fixture
+def rmock():
+    with requests_mock.mock() as rmock:
+        yield rmock

tests/notifications_utils/test_base64_uuid.py

@@ -0,0 +1,57 @@
+import binascii
+import os
+from uuid import UUID
+
+import pytest
+
+from notifications_utils.base64_uuid import (
+    base64_to_bytes,
+    base64_to_uuid,
+    bytes_to_base64,
+    uuid_to_base64,
+)
+
+
+def test_bytes_to_base64_to_bytes():
+    b = os.urandom(32)
+    b64 = bytes_to_base64(b)
+    assert base64_to_bytes(b64) == b
+
+
+@pytest.mark.parametrize(
+    "url_val",
+    [
+        "AAAAAAAAAAAAAAAAAAAAAQ",
+        "AAAAAAAAAAAAAAAAAAAAAQ=",  # even though this has invalid padding we put extra =s on the end so this is okay
+        "AAAAAAAAAAAAAAAAAAAAAQ==",
+    ],
+)
+def test_base64_converter_to_python(url_val):
+    assert base64_to_uuid(url_val) == UUID(int=1)
+
+
+@pytest.mark.parametrize(
+    "python_val", [UUID(int=1), "00000000-0000-0000-0000-000000000001"]
+)
+def test_base64_converter_to_url(python_val):
+    assert uuid_to_base64(python_val) == "AAAAAAAAAAAAAAAAAAAAAQ"
+
+
+@pytest.mark.parametrize(
+    "url_val,expectation",
+    [
+        (
+            "this_is_valid_base64_but_is_too_long_to_be_a_uuid",
+            pytest.raises(binascii.Error),
+        ),
+        ("this_one_has_emoji_➕➕➕", pytest.raises(UnicodeEncodeError)),
+    ],
+)
+def test_base64_converter_to_python_raises_validation_error(url_val, expectation):
+    with expectation:
+        base64_to_uuid(url_val)
+
+
+def test_base64_converter_to_url_raises_validation_error():
+    with pytest.raises(AttributeError):
+        uuid_to_base64(object())

tests/notifications_utils/test_base_template.py

@@ -0,0 +1,119 @@
+from unittest.mock import patch
+
+import pytest
+
+from notifications_utils.template import SubjectMixin, Template
+
+
+class ConcreteImplementation:
+    template_type = None
+
+    # Can’t instantiate and test templates unless they implement __str__
+    def __str__(self):
+        pass
+
+
+class ConcreteTemplate(ConcreteImplementation, Template):
+    pass
+
+
+class ConcreteTemplateWithSubject(SubjectMixin, ConcreteTemplate):
+    pass
+
+
+def test_class():
+    assert (
+        repr(ConcreteTemplate({"content": "hello ((name))"}))
+        == 'ConcreteTemplate("hello ((name))", {})'
+    )
+
+
+def test_passes_through_template_attributes():
+    assert ConcreteTemplate({"content": ""}).name is None
+    assert (
+        ConcreteTemplate({"content": "", "name": "Two week reminder"}).name
+        == "Two week reminder"
+    )
+    assert ConcreteTemplate({"content": ""}).id is None
+    assert ConcreteTemplate({"content": "", "id": "1234"}).id == "1234"
+    assert ConcreteTemplate({"content": ""}).template_type is None
+
+
+def test_passes_through_subject():
+    assert (
+        ConcreteTemplateWithSubject(
+            {"content": "", "subject": "Your tax is due"}
+        ).subject
+        == "Your tax is due"
+    )
+
+
+def test_errors_for_missing_template_content():
+    with pytest.raises(KeyError):
+        ConcreteTemplate({})
+
+
+@pytest.mark.parametrize("template", [0, 1, 2, True, False, None])
+def test_errors_for_invalid_template_types(template):
+    with pytest.raises(TypeError):
+        ConcreteTemplate(template)
+
+
+@pytest.mark.parametrize("values", [[], False])
+def test_errors_for_invalid_values(values):
+    with pytest.raises(TypeError):
+        ConcreteTemplate({"content": ""}, values)
+
+
+def test_matches_keys_to_placeholder_names():
+    template = ConcreteTemplate({"content": "hello ((name))"})
+
+    template.values = {"NAME": "Chris"}
+    assert template.values == {"name": "Chris"}
+
+    template.values = {"NAME": "Chris", "Town": "London"}
+    assert template.values == {"name": "Chris", "Town": "London"}
+    assert template.additional_data == {"Town"}
+
+    template.values = None
+    assert template.missing_data == ["name"]
+
+
+@pytest.mark.parametrize(
+    "template_content, template_subject, expected",
+    [
+        ("the quick brown fox", "jumps", []),
+        ("the quick ((colour)) fox", "jumps", ["colour"]),
+        ("the quick ((colour)) ((animal))", "jumps", ["colour", "animal"]),
+        ("((colour)) ((animal)) ((colour)) ((animal))", "jumps", ["colour", "animal"]),
+        ("the quick brown fox", "((colour))", ["colour"]),
+        ("the quick ((colour)) ", "((animal))", ["animal", "colour"]),
+        ("((colour)) ((animal)) ", "((colour)) ((animal))", ["colour", "animal"]),
+        ("Dear ((name)), ((warning?? This is a warning))", "", ["name", "warning"]),
+        ("((warning? one question mark))", "", ["warning? one question mark"]),
+    ],
+)
+def test_extracting_placeholders(template_content, template_subject, expected):
+    assert (
+        ConcreteTemplateWithSubject(
+            {"content": template_content, "subject": template_subject}
+        ).placeholders
+        == expected
+    )
+
+
+def test_random_variable_retrieve():
+    template = ConcreteTemplate({"content": "content", "created_by": "now"})
+    assert template.get_raw("created_by") == "now"
+    assert template.get_raw("missing", default="random") == "random"
+    assert template.get_raw("missing") is None
+
+
+def test_compare_template():
+    with patch(
+        "notifications_utils.template_change.TemplateChange.__init__", return_value=None
+    ) as mocked:
+        old_template = ConcreteTemplate({"content": "faked"})
+        new_template = ConcreteTemplate({"content": "faked"})
+        old_template.compare_to(new_template)
+        mocked.assert_called_once_with(old_template, new_template)

tests/notifications_utils/test_countries.py

@@ -0,0 +1,170 @@
+import pytest
+
+from notifications_utils.countries import Country, CountryMapping, CountryNotFoundError
+from notifications_utils.countries.data import (
+    _EUROPEAN_ISLANDS_LIST,
+    _UK_ISLANDS_LIST,
+    ADDITIONAL_SYNONYMS,
+    COUNTRIES_AND_TERRITORIES,
+    ROYAL_MAIL_EUROPEAN,
+    UK,
+    UK_ISLANDS,
+    WELSH_NAMES,
+    Postage,
+)
+
+from .country_synonyms import ALL as ALL_SYNONYMS
+from .country_synonyms import CROWDSOURCED_MISTAKES
+
+
+def test_constants():
+    assert UK == "United Kingdom"
+    assert UK_ISLANDS == [
+        ("Alderney", UK),
+        ("Brecqhou", UK),
+        ("Guernsey", UK),
+        ("Herm", UK),
+        ("Isle of Man", UK),
+        ("Jersey", UK),
+        ("Jethou", UK),
+        ("Sark", UK),
+    ]
+    assert Postage.EUROPE == "europe"
+    assert Postage.REST_OF_WORLD == "rest-of-world"
+    assert Postage.UK == "united-kingdom"
+
+
+@pytest.mark.parametrize("synonym, canonical", ADDITIONAL_SYNONYMS)
+def test_hand_crafted_synonyms_map_to_canonical_countries(synonym, canonical):
+    exceptions_to_canonical_countries = [
+        "Easter Island",
+        "South Georgia and the South Sandwich Islands",
+    ]
+
+    synonyms = dict(COUNTRIES_AND_TERRITORIES).keys()
+    canonical_names = list(dict(COUNTRIES_AND_TERRITORIES).values())
+
+    assert canonical in (
+        canonical_names
+        + _EUROPEAN_ISLANDS_LIST
+        + _UK_ISLANDS_LIST
+        + exceptions_to_canonical_countries
+    )
+
+    assert synonym not in {CountryMapping.make_key(synonym_) for synonym_ in synonyms}
+    assert Country(synonym).canonical_name == canonical
+
+
+@pytest.mark.parametrize("welsh_name, canonical", WELSH_NAMES)
+def test_welsh_names_map_to_canonical_countries(welsh_name, canonical):
+    assert Country(canonical).canonical_name == canonical
+    assert Country(welsh_name).canonical_name == canonical
+
+
+def test_all_synonyms():
+    for search, expected in ALL_SYNONYMS:
+        assert Country(search).canonical_name == expected
+
+
+def test_crowdsourced_test_data():
+    for search, expected_country, expected_postage in CROWDSOURCED_MISTAKES:
+        if expected_country or expected_postage:
+            assert Country(search).canonical_name == expected_country
+            assert Country(search).postage_zone == expected_postage
+
+
+@pytest.mark.parametrize(
+    "search, expected",
+    (
+        ("u.s.a", "United States"),
+        ("america", "United States"),
+        ("United States America", "United States"),
+        ("ROI", "Ireland"),
+        ("Irish Republic", "Ireland"),
+        ("Rep of Ireland", "Ireland"),
+        ("RepOfIreland", "Ireland"),
+        ("deutschland", "Germany"),
+        ("UK", "United Kingdom"),
+        ("England", "United Kingdom"),
+        ("Northern Ireland", "United Kingdom"),
+        ("Scotland", "United Kingdom"),
+        ("Wales", "United Kingdom"),
+        ("N. Ireland", "United Kingdom"),
+        ("GB", "United Kingdom"),
+        ("NIR", "United Kingdom"),
+        ("SCT", "United Kingdom"),
+        ("WLS", "United Kingdom"),
+        ("gambia", "The Gambia"),
+        ("Jersey", "United Kingdom"),
+        ("Guernsey", "United Kingdom"),
+        ("Lubnān", "Lebanon"),
+        ("Lubnan", "Lebanon"),
+        ("ESPAÑA", "Spain"),
+        ("ESPANA", "Spain"),
+        ("the democratic people's republic of korea", "North Korea"),
+        ("the democratic peoples republic of korea", "North Korea"),
+        ("ALAND", "Åland Islands"),
+        ("Sao Tome + Principe", "Sao Tome and Principe"),
+        ("Sao Tome & Principe", "Sao Tome and Principe"),
+        ("Antigua, and Barbuda", "Antigua and Barbuda"),
+        ("Azores", "Azores"),
+        ("Autonomous Region of the Azores", "Azores"),
+        ("Canary Islands", "Canary Islands"),
+        ("Islas Canarias", "Canary Islands"),
+        ("Canaries", "Canary Islands"),
+        ("Madeira", "Madeira"),
+        ("Autonomous Region of Madeira", "Madeira"),
+        ("Região Autónoma da Madeira", "Madeira"),
+        ("Balearic Islands", "Balearic Islands"),
+        ("Islas Baleares", "Balearic Islands"),
+        ("Illes Balears", "Balearic Islands"),
+        ("Corsica", "Corsica"),
+        ("Corse", "Corsica"),
+    ),
+)
+def test_hand_crafted_synonyms(search, expected):
+    assert Country(search).canonical_name == expected
+
+
+def test_auto_checking_for_country_starting_with_the():
+    canonical_names = dict(COUNTRIES_AND_TERRITORIES).values()
+    synonyms = dict(COUNTRIES_AND_TERRITORIES).keys()
+    assert "The Gambia" in canonical_names
+    assert "Gambia" not in synonyms
+    assert Country("Gambia").canonical_name == "The Gambia"
+
+
+@pytest.mark.parametrize(
+    "search, expected_error_message",
+    (
+        ("Qumran", "Not a known country or territory (Qumran)"),
+        ("Kumrahn", "Not a known country or territory (Kumrahn)"),
+    ),
+)
+def test_non_existant_countries(search, expected_error_message):
+    with pytest.raises(KeyError) as error:
+        Country(search)
+    assert str(error.value) == repr(expected_error_message)
+    assert isinstance(error.value, CountryNotFoundError)
+
+
+@pytest.mark.parametrize(
+    "search, expected",
+    (
+        ("u.s.a", "rest-of-world"),
+        ("Rep of Ireland", "europe"),
+        ("deutschland", "europe"),
+        ("UK", "united-kingdom"),
+        ("Jersey", "united-kingdom"),
+        ("Guernsey", "united-kingdom"),
+        ("isle-of-man", "united-kingdom"),
+        ("ESPAÑA", "europe"),
+    ),
+)
+def test_get_postage(search, expected):
+    assert Country(search).postage_zone == expected
+
+
+def test_euro_postage_zone():
+    for search in ROYAL_MAIL_EUROPEAN:
+        assert Country(search).postage_zone == Postage.EUROPE

tests/notifications_utils/test_countries_iso.py

@@ -0,0 +1,526 @@
+import pytest
+
+from notifications_utils.countries import Country, CountryNotFoundError
+
+
+def _country_not_found(*test_case):
+    return pytest.param(
+        *test_case,
+        marks=pytest.mark.xfail(raises=CountryNotFoundError),
+    )
+
+
+@pytest.mark.parametrize(
+    "alpha_2, expected_name",
+    (
+        ("AF", "Afghanistan"),
+        ("AL", "Albania"),
+        ("DZ", "Algeria"),
+        ("AS", "American Samoa"),
+        ("AD", "Andorra"),
+        ("AO", "Angola"),
+        ("AI", "Anguilla"),
+        ("AQ", "Antarctica"),
+        ("AG", "Antigua and Barbuda"),
+        ("AR", "Argentina"),
+        ("AM", "Armenia"),
+        ("AW", "Aruba"),
+        ("AU", "Australia"),
+        ("AT", "Austria"),
+        ("AZ", "Azerbaijan"),
+        ("BS", "The Bahamas"),
+        ("BH", "Bahrain"),
+        ("BD", "Bangladesh"),
+        ("BB", "Barbados"),
+        ("BY", "Belarus"),
+        ("BE", "Belgium"),
+        ("BZ", "Belize"),
+        ("BJ", "Benin"),
+        ("BM", "Bermuda"),
+        ("BT", "Bhutan"),
+        ("BO", "Bolivia"),
+        _country_not_found("BQ", "Bonaire, Sint Eustatius and Saba"),
+        ("BA", "Bosnia and Herzegovina"),
+        ("BW", "Botswana"),
+        ("BV", "Bouvet Island"),
+        ("BR", "Brazil"),
+        ("IO", "British Indian Ocean Territory"),
+        ("BN", "Brunei"),
+        ("BG", "Bulgaria"),
+        ("BF", "Burkina Faso"),
+        ("BI", "Burundi"),
+        ("CV", "Cape Verde"),
+        ("KH", "Cambodia"),
+        ("CM", "Cameroon"),
+        ("CA", "Canada"),
+        ("KY", "Cayman Islands"),
+        ("CF", "Central African Republic"),
+        ("TD", "Chad"),
+        ("CL", "Chile"),
+        ("CN", "China"),
+        ("CX", "Christmas Island"),
+        ("CC", "Cocos (Keeling) Islands"),
+        ("CO", "Colombia"),
+        ("KM", "Comoros"),
+        ("CD", "Congo (Democratic Republic)"),
+        ("CG", "Congo"),
+        ("CK", "Cook Islands"),
+        ("CR", "Costa Rica"),
+        ("HR", "Croatia"),
+        ("CU", "Cuba"),
+        ("CW", "Curaçao"),
+        ("CY", "Cyprus"),
+        ("CZ", "Czechia"),
+        ("CI", "Ivory Coast"),
+        ("DK", "Denmark"),
+        ("DJ", "Djibouti"),
+        ("DM", "Dominica"),
+        ("DO", "Dominican Republic"),
+        ("EC", "Ecuador"),
+        ("EG", "Egypt"),
+        ("SV", "El Salvador"),
+        ("GQ", "Equatorial Guinea"),
+        ("ER", "Eritrea"),
+        ("EE", "Estonia"),
+        ("SZ", "Eswatini"),
+        ("ET", "Ethiopia"),
+        ("FK", "Falkland Islands"),
+        ("FO", "Faroe Islands"),
+        ("FJ", "Fiji"),
+        ("FI", "Finland"),
+        ("FR", "France"),
+        ("GF", "French Guiana"),
+        ("PF", "French Polynesia"),
+        ("TF", "French Southern Territories"),
+        ("GA", "Gabon"),
+        ("GM", "The Gambia"),
+        ("GE", "Georgia"),
+        ("DE", "Germany"),
+        ("GH", "Ghana"),
+        ("GI", "Gibraltar"),
+        ("GR", "Greece"),
+        ("GL", "Greenland"),
+        ("GD", "Grenada"),
+        ("GP", "Guadeloupe"),
+        ("GU", "Guam"),
+        ("GT", "Guatemala"),
+        ("GG", "United Kingdom"),
+        ("GN", "Guinea"),
+        ("GW", "Guinea-Bissau"),
+        ("GY", "Guyana"),
+        ("HT", "Haiti"),
+        ("HM", "Heard Island and McDonald Islands"),
+        ("VA", "Vatican City"),
+        ("HN", "Honduras"),
+        ("HK", "Hong Kong"),
+        ("HU", "Hungary"),
+        ("IS", "Iceland"),
+        ("IN", "India"),
+        ("ID", "Indonesia"),
+        ("IR", "Iran"),
+        ("IQ", "Iraq"),
+        ("IE", "Ireland"),
+        ("IM", "United Kingdom"),
+        ("IL", "Israel"),
+        ("IT", "Italy"),
+        ("JM", "Jamaica"),
+        ("JP", "Japan"),
+        ("JE", "United Kingdom"),
+        ("JO", "Jordan"),
+        ("KZ", "Kazakhstan"),
+        ("KE", "Kenya"),
+        ("KI", "Kiribati"),
+        ("KP", "North Korea"),
+        ("KR", "South Korea"),
+        ("KW", "Kuwait"),
+        ("KG", "Kyrgyzstan"),
+        ("LA", "Laos"),
+        ("LV", "Latvia"),
+        ("LB", "Lebanon"),
+        ("LS", "Lesotho"),
+        ("LR", "Liberia"),
+        ("LY", "Libya"),
+        ("LI", "Liechtenstein"),
+        ("LT", "Lithuania"),
+        ("LU", "Luxembourg"),
+        ("MO", "Macao"),
+        ("MG", "Madagascar"),
+        ("MW", "Malawi"),
+        ("MY", "Malaysia"),
+        ("MV", "Maldives"),
+        ("ML", "Mali"),
+        ("MT", "Malta"),
+        ("MH", "Marshall Islands"),
+        ("MQ", "Martinique"),
+        ("MR", "Mauritania"),
+        ("MU", "Mauritius"),
+        ("YT", "Mayotte"),
+        ("MX", "Mexico"),
+        ("FM", "Micronesia"),
+        ("MD", "Moldova"),
+        ("MC", "Monaco"),
+        ("MN", "Mongolia"),
+        ("ME", "Montenegro"),
+        ("MS", "Montserrat"),
+        ("MA", "Morocco"),
+        ("MZ", "Mozambique"),
+        ("MM", "Myanmar (Burma)"),
+        ("NA", "Namibia"),
+        ("NR", "Nauru"),
+        ("NP", "Nepal"),
+        ("NL", "Netherlands"),
+        ("NC", "New Caledonia"),
+        ("NZ", "New Zealand"),
+        ("NI", "United Kingdom"),  # NI gets interpreted as ‘Northern Ireland’
+        ("NE", "Niger"),
+        ("NG", "Nigeria"),
+        ("NU", "Niue"),
+        ("NF", "Norfolk Island"),
+        ("MK", "North Macedonia"),
+        ("MP", "Northern Mariana Islands"),
+        ("NO", "Norway"),
+        ("OM", "Oman"),
+        ("PK", "Pakistan"),
+        ("PW", "Palau"),
+        ("PS", "Occupied Palestinian Territories"),
+        ("PA", "Panama"),
+        ("PG", "Papua New Guinea"),
+        ("PY", "Paraguay"),
+        ("PE", "Peru"),
+        ("PH", "Philippines"),
+        ("PN", "Pitcairn, Henderson, Ducie and Oeno Islands"),
+        ("PL", "Poland"),
+        ("PT", "Portugal"),
+        ("PR", "Puerto Rico"),
+        ("QA", "Qatar"),
+        ("RO", "Romania"),
+        ("RU", "Russia"),
+        ("RW", "Rwanda"),
+        ("RE", "Réunion"),
+        ("BL", "Saint Barthélemy"),
+        _country_not_found("SH", "Saint Helena, Ascension and Tristan da Cunha"),
+        ("KN", "St Kitts and Nevis"),
+        ("LC", "St Lucia"),
+        ("MF", "Saint-Martin (French part)"),
+        ("PM", "Saint Pierre and Miquelon"),
+        ("VC", "St Vincent"),
+        ("WS", "Samoa"),
+        ("SM", "San Marino"),
+        ("ST", "Sao Tome and Principe"),
+        ("SA", "Saudi Arabia"),
+        ("SN", "Senegal"),
+        ("RS", "Serbia"),
+        ("SC", "Seychelles"),
+        ("SL", "Sierra Leone"),
+        ("SG", "Singapore"),
+        ("SX", "Sint Maarten (Dutch part)"),
+        ("SK", "Slovakia"),
+        ("SI", "Slovenia"),
+        ("SB", "Solomon Islands"),
+        ("SO", "Somalia"),
+        ("ZA", "South Africa"),
+        ("GS", "South Georgia and South Sandwich Islands"),
+        ("SS", "South Sudan"),
+        ("ES", "Spain"),
+        ("LK", "Sri Lanka"),
+        ("SD", "Sudan"),
+        ("SR", "Suriname"),
+        ("SJ", "Svalbard and Jan Mayen"),
+        ("SE", "Sweden"),
+        ("CH", "Switzerland"),
+        ("SY", "Syria"),
+        ("TW", "Taiwan"),
+        ("TJ", "Tajikistan"),
+        ("TZ", "Tanzania"),
+        ("TH", "Thailand"),
+        ("TL", "East Timor"),
+        ("TG", "Togo"),
+        ("TK", "Tokelau"),
+        ("TO", "Tonga"),
+        ("TT", "Trinidad and Tobago"),
+        ("TN", "Tunisia"),
+        ("TR", "Turkey"),
+        ("TM", "Turkmenistan"),
+        ("TC", "Turks and Caicos Islands"),
+        ("TV", "Tuvalu"),
+        ("UG", "Uganda"),
+        ("UA", "Ukraine"),
+        ("AE", "United Arab Emirates"),
+        ("GB", "United Kingdom"),
+        _country_not_found("UM", "United States Minor Outlying Islands"),
+        ("US", "United States"),
+        ("UY", "Uruguay"),
+        ("UZ", "Uzbekistan"),
+        ("VU", "Vanuatu"),
+        ("VE", "Venezuela"),
+        ("VN", "Vietnam"),
+        ("VG", "British Virgin Islands"),
+        ("VI", "United States Virgin Islands"),
+        ("WF", "Wallis and Futuna"),
+        ("EH", "Western Sahara"),
+        ("YE", "Yemen"),
+        ("ZM", "Zambia"),
+        ("ZW", "Zimbabwe"),
+        ("AX", "Åland Islands"),
+    ),
+)
+def test_iso_alpha_2_country_codes(alpha_2, expected_name):
+    assert Country(alpha_2).canonical_name == expected_name
+
+
+@pytest.mark.parametrize(
+    "alpha_3, expected_name",
+    (
+        _country_not_found("AFG", "Afghanistan"),
+        _country_not_found("ALB", "Albania"),
+        _country_not_found("DZA", "Algeria"),
+        _country_not_found("ASM", "American Samoa"),
+        _country_not_found("AND", "Andorra"),
+        _country_not_found("AGO", "Angola"),
+        _country_not_found("AIA", "Anguilla"),
+        _country_not_found("ATA", "Antarctica"),
+        _country_not_found("ATG", "Antigua and Barbuda"),
+        _country_not_found("ARG", "Argentina"),
+        _country_not_found("ARM", "Armenia"),
+        _country_not_found("ABW", "Aruba"),
+        _country_not_found("AUS", "Australia"),
+        _country_not_found("AUT", "Austria"),
+        _country_not_found("AZE", "Azerbaijan"),
+        _country_not_found("BHS", "The Bahamas"),
+        _country_not_found("BHR", "Bahrain"),
+        _country_not_found("BGD", "Bangladesh"),
+        _country_not_found("BRB", "Barbados"),
+        _country_not_found("BLR", "Belarus"),
+        _country_not_found("BEL", "Belgium"),
+        _country_not_found("BLZ", "Belize"),
+        _country_not_found("BEN", "Benin"),
+        _country_not_found("BMU", "Bermuda"),
+        _country_not_found("BTN", "Bhutan"),
+        _country_not_found("BOL", "Bolivia"),
+        _country_not_found("BES", "Bonaire, Sint Eustatius and Saba"),
+        ("BIH", "Bosnia and Herzegovina"),
+        _country_not_found("BWA", "Botswana"),
+        _country_not_found("BVT", "Bouvet Island"),
+        _country_not_found("BRA", "Brazil"),
+        ("IOT", "British Indian Ocean Territory"),
+        _country_not_found("BRN", "Brunei"),
+        _country_not_found("BGR", "Bulgaria"),
+        _country_not_found("BFA", "Burkina Faso"),
+        _country_not_found("BDI", "Burundi"),
+        _country_not_found("CPV", "Cape Verde"),
+        _country_not_found("KHM", "Cambodia"),
+        _country_not_found("CMR", "Cameroon"),
+        _country_not_found("CAN", "Canada"),
+        _country_not_found("CYM", "Cayman Islands"),
+        _country_not_found("CAF", "Central African Republic"),
+        _country_not_found("TCD", "Chad"),
+        _country_not_found("CHL", "Chile"),
+        _country_not_found("CHN", "China"),
+        _country_not_found("CXR", "Christmas Island"),
+        _country_not_found("CCK", "Cocos (Keeling) Islands"),
+        _country_not_found("COL", "Colombia"),
+        _country_not_found("COM", "Comoros"),
+        _country_not_found("COD", "Congo (Democratic Republic)"),
+        _country_not_found("COG", "Congo"),
+        _country_not_found("COK", "Cook Islands"),
+        _country_not_found("CRI", "Costa Rica"),
+        _country_not_found("HRV", "Croatia"),
+        _country_not_found("CUB", "Cuba"),
+        _country_not_found("CUW", "Curaçao"),
+        _country_not_found("CYP", "Cyprus"),
+        _country_not_found("CZE", "Czechia"),
+        _country_not_found("CIV", "Ivory Coast"),
+        _country_not_found("DNK", "Denmark"),
+        _country_not_found("DJI", "Djibouti"),
+        _country_not_found("DMA", "Dominica"),
+        _country_not_found("DOM", "Dominican Republic"),
+        _country_not_found("ECU", "Ecuador"),
+        _country_not_found("EGY", "Egypt"),
+        _country_not_found("SLV", "El Salvador"),
+        _country_not_found("GNQ", "Equatorial Guinea"),
+        _country_not_found("ERI", "Eritrea"),
+        _country_not_found("EST", "Estonia"),
+        _country_not_found("SWZ", "Eswatini"),
+        _country_not_found("ETH", "Ethiopia"),
+        _country_not_found("FLK", "Falkland Islands"),
+        _country_not_found("FRO", "Faroe Islands"),
+        _country_not_found("FJI", "Fiji"),
+        _country_not_found("FIN", "Finland"),
+        _country_not_found("FRA", "France"),
+        _country_not_found("GUF", "French Guiana"),
+        _country_not_found("PYF", "French Polynesia"),
+        _country_not_found("ATF", "French Southern Territories"),
+        _country_not_found("GAB", "Gabon"),
+        _country_not_found("GMB", "The Gambia"),
+        _country_not_found("GEO", "Georgia"),
+        _country_not_found("DEU", "Germany"),
+        _country_not_found("GHA", "Ghana"),
+        _country_not_found("GIB", "Gibraltar"),
+        _country_not_found("GRC", "Greece"),
+        _country_not_found("GRL", "Greenland"),
+        _country_not_found("GRD", "Grenada"),
+        _country_not_found("GLP", "Guadeloupe"),
+        _country_not_found("GUM", "Guam"),
+        _country_not_found("GTM", "Guatemala"),
+        _country_not_found("GGY", "United Kingdom"),
+        _country_not_found("GIN", "Guinea"),
+        _country_not_found("GNB", "Guinea-Bissau"),
+        _country_not_found("GUY", "Guyana"),
+        _country_not_found("HTI", "Haiti"),
+        _country_not_found("HMD", "Heard Island and McDonald Islands"),
+        _country_not_found("VAT", "Vatican City"),
+        _country_not_found("HND", "Honduras"),
+        _country_not_found("HKG", "Hong Kong"),
+        _country_not_found("HUN", "Hungary"),
+        _country_not_found("ISL", "Iceland"),
+        _country_not_found("IND", "India"),
+        _country_not_found("IDN", "Indonesia"),
+        _country_not_found("IRN", "Iran"),
+        _country_not_found("IRQ", "Iraq"),
+        _country_not_found("IRL", "Ireland"),
+        _country_not_found("IMN", "United Kingdom"),
+        _country_not_found("ISR", "Israel"),
+        _country_not_found("ITA", "Italy"),
+        _country_not_found("JAM", "Jamaica"),
+        _country_not_found("JPN", "Japan"),
+        _country_not_found("JEY", "United Kingdom"),
+        _country_not_found("JOR", "Jordan"),
+        _country_not_found("KAZ", "Kazakhstan"),
+        _country_not_found("KEN", "Kenya"),
+        _country_not_found("KIR", "Kiribati"),
+        ("PRK", "North Korea"),
+        _country_not_found("KOR", "Korea"),
+        _country_not_found("KWT", "Kuwait"),
+        _country_not_found("KGZ", "Kyrgyzstan"),
+        ("LAO", "Laos"),
+        _country_not_found("LVA", "Latvia"),
+        _country_not_found("LBN", "Lebanon"),
+        _country_not_found("LSO", "Lesotho"),
+        _country_not_found("LBR", "Liberia"),
+        _country_not_found("LBY", "Libya"),
+        _country_not_found("LIE", "Liechtenstein"),
+        _country_not_found("LTU", "Lithuania"),
+        _country_not_found("LUX", "Luxembourg"),
+        _country_not_found("MAC", "Macao"),
+        _country_not_found("MDG", "Madagascar"),
+        _country_not_found("MWI", "Malawi"),
+        _country_not_found("MYS", "Malaysia"),
+        _country_not_found("MDV", "Maldives"),
+        _country_not_found("MLI", "Mali"),
+        _country_not_found("MLT", "Malta"),
+        _country_not_found("MHL", "Marshall Islands"),
+        _country_not_found("MTQ", "Martinique"),
+        _country_not_found("MRT", "Mauritania"),
+        _country_not_found("MUS", "Mauritius"),
+        _country_not_found("MYT", "Mayotte"),
+        _country_not_found("MEX", "Mexico"),
+        _country_not_found("FSM", "Micronesia"),
+        _country_not_found("MDA", "Moldova"),
+        _country_not_found("MCO", "Monaco"),
+        _country_not_found("MNG", "Mongolia"),
+        _country_not_found("MNE", "Montenegro"),
+        _country_not_found("MSR", "Montserrat"),
+        _country_not_found("MAR", "Morocco"),
+        _country_not_found("MOZ", "Mozambique"),
+        _country_not_found("MMR", "Myanmar (Burma)"),
+        _country_not_found("NAM", "Namibia"),
+        _country_not_found("NRU", "Nauru"),
+        _country_not_found("NPL", "Nepal"),
+        _country_not_found("NLD", "Netherlands"),
+        _country_not_found("NCL", "New Caledonia"),
+        _country_not_found("NZL", "New Zealand"),
+        _country_not_found("NIC", "Nicaragua"),
+        _country_not_found("NER", "Niger"),
+        _country_not_found("NGA", "Nigeria"),
+        _country_not_found("NIU", "Niue"),
+        _country_not_found("NFK", "Norfolk Island"),
+        _country_not_found("MKD", "North Macedonia"),
+        _country_not_found("MNP", "Northern Mariana Islands"),
+        _country_not_found("NOR", "Norway"),
+        _country_not_found("OMN", "Oman"),
+        _country_not_found("PAK", "Pakistan"),
+        _country_not_found("PLW", "Palau"),
+        _country_not_found("PSE", "Occupied Palestinian Territories"),
+        _country_not_found("PAN", "Panama"),
+        ("PNG", "Papua New Guinea"),
+        _country_not_found("PRY", "Paraguay"),
+        _country_not_found("PER", "Peru"),
+        _country_not_found("PHL", "Philippines"),
+        _country_not_found("PCN", "Pitcairn, Henderson, Ducie and Oeno Islands"),
+        _country_not_found("POL", "Poland"),
+        _country_not_found("PRT", "Portugal"),
+        _country_not_found("PRI", "Puerto Rico"),
+        _country_not_found("QAT", "Qatar"),
+        _country_not_found("ROU", "Romania"),
+        _country_not_found("RUS", "Russian Federation"),
+        _country_not_found("RWA", "Rwanda"),
+        _country_not_found("REU", "Réunion"),
+        _country_not_found("BLM", "Saint Barthélemy"),
+        _country_not_found("SHN", "Saint Helena, Ascension and Tristan da Cunha"),
+        _country_not_found("KNA", "St Kitts and Nevis"),
+        _country_not_found("LCA", "St Lucia"),
+        _country_not_found("MAF", "Saint-Martin (French part)"),
+        _country_not_found("SPM", "Saint Pierre and Miquelon"),
+        _country_not_found("VCT", "Saint Vincent"),
+        _country_not_found("WSM", "Samoa"),
+        _country_not_found("SMR", "San Marino"),
+        _country_not_found("STP", "Sao Tome and Principe"),
+        _country_not_found("SAU", "Saudi Arabia"),
+        _country_not_found("SEN", "Senegal"),
+        _country_not_found("SRB", "Serbia"),
+        _country_not_found("SYC", "Seychelles"),
+        _country_not_found("SLE", "Sierra Leone"),
+        _country_not_found("SGP", "Singapore"),
+        _country_not_found("SXM", "Sint Maarten (Dutch part)"),
+        _country_not_found("SVK", "Slovakia"),
+        _country_not_found("SVN", "Slovenia"),
+        _country_not_found("SLB", "Solomon Islands"),
+        _country_not_found("SOM", "Somalia"),
+        _country_not_found("ZAF", "South Africa"),
+        _country_not_found("SGS", "South Georgia and South Sandwich Islands"),
+        _country_not_found("SSD", "South Sudan"),
+        _country_not_found("ESP", "Spain"),
+        _country_not_found("LKA", "Sri Lanka"),
+        _country_not_found("SDN", "Sudan"),
+        _country_not_found("SUR", "Suriname"),
+        _country_not_found("SJM", "Svalbard and Jan Mayen"),
+        _country_not_found("SWE", "Sweden"),
+        _country_not_found("CHE", "Switzerland"),
+        _country_not_found("SYR", "Syrian Arab Republic"),
+        _country_not_found("TWN", "Taiwan"),
+        _country_not_found("TJK", "Tajikistan"),
+        _country_not_found("TZA", "Tanzania"),
+        _country_not_found("THA", "Thailand"),
+        _country_not_found("TLS", "East Timor"),
+        _country_not_found("TGO", "Togo"),
+        _country_not_found("TKL", "Tokelau"),
+        _country_not_found("TON", "Tonga"),
+        _country_not_found("TTO", "Trinidad and Tobago"),
+        _country_not_found("TUN", "Tunisia"),
+        _country_not_found("TUR", "Turkey"),
+        _country_not_found("TKM", "Turkmenistan"),
+        _country_not_found("TCA", "Turks and Caicos Islands"),
+        _country_not_found("TUV", "Tuvalu"),
+        _country_not_found("UGA", "Uganda"),
+        _country_not_found("UKR", "Ukraine"),
+        _country_not_found("ARE", "United Arab Emirates"),
+        ("GBR", "United Kingdom"),
+        _country_not_found("UMI", "United States Minor Outlying Islands"),
+        ("USA", "United States"),
+        _country_not_found("URY", "Uruguay"),
+        _country_not_found("UZB", "Uzbekistan"),
+        _country_not_found("VUT", "Vanuatu"),
+        _country_not_found("VEN", "Venezuela"),
+        _country_not_found("VNM", "Vietnam"),
+        _country_not_found("VGB", "British Virgin Islands"),
+        _country_not_found("VIR", "United States Virgin Islands"),
+        _country_not_found("WLF", "Wallis and Futuna"),
+        _country_not_found("ESH", "Western Sahara"),
+        _country_not_found("YEM", "Yemen"),
+        _country_not_found("ZMB", "Zambia"),
+        _country_not_found("ZWE", "Zimbabwe"),
+        _country_not_found("ALA", "Åland Islands"),
+    ),
+)
+def test_iso_alpha_3_country_codes(alpha_3, expected_name):
+    assert Country(alpha_3).canonical_name == expected_name

tests/notifications_utils/test_field.py

@@ -0,0 +1,311 @@
+import pytest
+
+from notifications_utils.field import Field, str2bool
+
+
+@pytest.mark.parametrize(
+    "content",
+    [
+        "",
+        "the quick brown fox",
+        """
+        the
+        quick brown
+
+        fox
+    """,
+        "the ((quick brown fox",
+        "the (()) brown fox",
+    ],
+)
+def test_returns_a_string_without_placeholders(content):
+    assert str(Field(content)) == content
+
+
+@pytest.mark.parametrize(
+    "template_content,data,expected",
+    [
+        ("((colour))", {"colour": "red"}, "red"),
+        ("the quick ((colour)) fox", {"colour": "brown"}, "the quick brown fox"),
+        (
+            "((article)) quick ((colour)) ((animal))",
+            {"article": "the", "colour": "brown", "animal": "fox"},
+            "the quick brown fox",
+        ),
+        ("the quick (((colour))) fox", {"colour": "brown"}, "the quick (brown) fox"),
+        (
+            "the quick ((colour)) fox",
+            {"colour": "<script>alert('foo')</script>"},
+            "the quick alert('foo') fox",
+        ),
+        (
+            "before ((placeholder)) after",
+            {"placeholder": ""},
+            "before  after",
+        ),
+        (
+            "before ((placeholder)) after",
+            {"placeholder": "   "},
+            "before     after",
+        ),
+        (
+            "before ((placeholder)) after",
+            {"placeholder": True},
+            "before True after",
+        ),
+        (
+            "before ((placeholder)) after",
+            {"placeholder": False},
+            "before False after",
+        ),
+        (
+            "before ((placeholder)) after",
+            {"placeholder": 0},
+            "before 0 after",
+        ),
+        (
+            "before ((placeholder)) after",
+            {"placeholder": 0.0},
+            "before 0.0 after",
+        ),
+        (
+            "before ((placeholder)) after",
+            {"placeholder": 123},
+            "before 123 after",
+        ),
+        (
+            "before ((placeholder)) after",
+            {"placeholder": 0.1 + 0.2},
+            "before 0.30000000000000004 after",
+        ),
+        (
+            "before ((placeholder)) after",
+            {"placeholder": {"key": "value"}},
+            "before {'key': 'value'} after",
+        ),
+        (
+            "((warning?))",
+            {"warning?": "This is not a conditional"},
+            "This is not a conditional",
+        ),
+        (
+            "((warning?warning))",
+            {"warning?warning": "This is not a conditional"},
+            "This is not a conditional",
+        ),
+        (
+            "((warning??This is a conditional warning))",
+            {"warning": True},
+            "This is a conditional warning",
+        ),
+        (
+            "((warning??This is a conditional warning\nwith line break))",
+            {"warning": True},
+            "This is a conditional warning\nwith line break",
+        ),
+        ("((warning??This is a conditional warning))", {"warning": False}, ""),
+    ],
+)
+def test_replacement_of_placeholders(template_content, data, expected):
+    assert str(Field(template_content, data)) == expected
+
+
+@pytest.mark.parametrize(
+    "template_content,data,expected",
+    [
+        (
+            "((code)) is your security code",
+            {"code": "12345"},
+            "12345 is your security code",
+        ),
+        (
+            "((code)) is your security code",
+            {},
+            "<span class='placeholder-redacted'>hidden</span> is your security code",
+        ),
+        (
+            "Hey ((name)), click http://example.com/reset-password/?token=((token))",
+            {"name": "Example"},
+            (
+                "Hey Example, click "
+                "http://example.com/reset-password/?token="
+                "<span class='placeholder-redacted'>hidden</span>"
+            ),
+        ),
+    ],
+)
+def test_optional_redacting_of_missing_values(template_content, data, expected):
+    assert (
+        str(Field(template_content, data, redact_missing_personalisation=True))
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "content,expected",
+    [
+        ("((colour))", "<span class='placeholder'>((colour))</span>"),
+        (
+            "the quick ((colour)) fox",
+            "the quick <span class='placeholder'>((colour))</span> fox",
+        ),
+        (
+            "((article)) quick ((colour)) ((animal))",
+            "<span class='placeholder'>((article))</span> quick <span class='placeholder'>((colour))</span> <span class='placeholder'>((animal))</span>",  # noqa
+        ),
+        (
+            """
+                ((article)) quick
+                ((colour))
+                ((animal))
+            """,
+            """
+                <span class='placeholder'>((article))</span> quick
+                <span class='placeholder'>((colour))</span>
+                <span class='placeholder'>((animal))</span>
+            """,
+        ),
+        (
+            "the quick (((colour))) fox",
+            "the quick (<span class='placeholder'>((colour))</span>) fox",
+        ),
+        ("((warning?))", "<span class='placeholder'>((warning?))</span>"),
+        (
+            "((warning? This is not a conditional))",
+            "<span class='placeholder'>((warning? This is not a conditional))</span>",
+        ),
+        (
+            "((warning?? This is a warning))",
+            "<span class='placeholder-conditional'>((warning??</span> This is a warning))",
+        ),
+        (
+            "((warning?? This is a warning\n text after linebreak))",
+            "<span class='placeholder-conditional'>((warning??</span> This is a warning\n text after linebreak))",
+        ),
+    ],
+)
+def test_formatting_of_placeholders(content, expected):
+    assert str(Field(content)) == expected
+
+
+@pytest.mark.parametrize(
+    "content, values, expected",
+    [
+        (
+            "((name)) ((colour))",
+            {"name": "Jo"},
+            "Jo <span class='placeholder'>((colour))</span>",
+        ),
+        (
+            "((name)) ((colour))",
+            {"name": "Jo", "colour": None},
+            "Jo <span class='placeholder'>((colour))</span>",
+        ),
+        (
+            "((show_thing??thing)) ((colour))",
+            {"colour": "red"},
+            "<span class='placeholder-conditional'>((show_thing??</span>thing)) red",
+        ),
+    ],
+)
+def test_handling_of_missing_values(content, values, expected):
+    assert str(Field(content, values)) == expected
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "0",
+        0,
+        2,
+        99.99999,
+        "off",
+        "exclude",
+        "no" "any random string",
+        "false",
+        False,
+        [],
+        {},
+        (),
+        ["true"],
+        {"True": True},
+        (True, "true", 1),
+    ],
+)
+def test_what_will_not_trigger_conditional_placeholder(value):
+    assert str2bool(value) is False
+
+
+@pytest.mark.parametrize(
+    "value", [1, "1", "yes", "y", "true", "True", True, "include", "show"]
+)
+def test_what_will_trigger_conditional_placeholder(value):
+    assert str2bool(value) is True
+
+
+@pytest.mark.parametrize(
+    "values, expected, expected_as_markdown",
+    [
+        (
+            {"placeholder": []},
+            "list: ",
+            "list: ",
+        ),
+        (
+            {"placeholder": ["", ""]},
+            "list: ",
+            "list: ",
+        ),
+        (
+            {"placeholder": [" ", " \t ", "\u180E"]},
+            "list: ",
+            "list: ",
+        ),
+        (
+            {"placeholder": ["one"]},
+            "list: one",
+            "list: \n\n* one",
+        ),
+        (
+            {"placeholder": ["one", "two"]},
+            "list: one and two",
+            "list: \n\n* one\n* two",
+        ),
+        (
+            {"placeholder": ["one", "two", "three"]},
+            "list: one, two and three",
+            "list: \n\n* one\n* two\n* three",
+        ),
+        (
+            {"placeholder": ["one", None, None]},
+            "list: one",
+            "list: \n\n* one",
+        ),
+        (
+            {"placeholder": ["<script>", 'alert("foo")', "</script>"]},
+            'list: , alert("foo") and ',
+            'list: \n\n* \n* alert("foo")\n* ',
+        ),
+        (
+            {"placeholder": [1, {"two": 2}, "three", None]},
+            "list: 1, {'two': 2} and three",
+            "list: \n\n* 1\n* {'two': 2}\n* three",
+        ),
+        (
+            {"placeholder": [[1, 2], [3, 4]]},
+            "list: [1, 2] and [3, 4]",
+            "list: \n\n* [1, 2]\n* [3, 4]",
+        ),
+        (
+            {"placeholder": [0.1, True, False]},
+            "list: 0.1, True and False",
+            "list: \n\n* 0.1\n* True\n* False",
+        ),
+    ],
+)
+def test_field_renders_lists_as_strings(values, expected, expected_as_markdown):
+    assert (
+        str(Field("list: ((placeholder))", values, markdown_lists=True))
+        == expected_as_markdown
+    )
+    assert str(Field("list: ((placeholder))", values)) == expected

tests/notifications_utils/test_field_html_handling.py

@@ -0,0 +1,71 @@
+import pytest
+
+from notifications_utils.field import Field
+
+
+@pytest.mark.parametrize(
+    "content, values, expected_stripped, expected_escaped, expected_passthrough",
+    [
+        (
+            "string <em>with</em> html",
+            {},
+            "string with html",
+            "string &lt;em&gt;with&lt;/em&gt; html",
+            "string <em>with</em> html",
+        ),
+        (
+            "string ((<em>with</em>)) html",
+            {},
+            "string <span class='placeholder'>((with))</span> html",
+            "string <span class='placeholder'>((&lt;em&gt;with&lt;/em&gt;))</span> html",
+            "string <span class='placeholder'>((<em>with</em>))</span> html",
+        ),
+        (
+            "string ((placeholder)) html",
+            {"placeholder": "<em>without</em>"},
+            "string without html",
+            "string &lt;em&gt;without&lt;/em&gt; html",
+            "string <em>without</em> html",
+        ),
+        (
+            "string ((<em>conditional</em>??<em>placeholder</em>)) html",
+            {},
+            "string <span class='placeholder-conditional'>((conditional??</span>placeholder)) html",
+            (
+                "string "
+                "<span class='placeholder-conditional'>"
+                "((&lt;em&gt;conditional&lt;/em&gt;??</span>"
+                "&lt;em&gt;placeholder&lt;/em&gt;)) "
+                "html"
+            ),
+            (
+                "string "
+                "<span class='placeholder-conditional'>"
+                "((<em>conditional</em>??</span>"
+                "<em>placeholder</em>)) "
+                "html"
+            ),
+        ),
+        (
+            "string ((conditional??<em>placeholder</em>)) html",
+            {"conditional": True},
+            "string placeholder html",
+            "string &lt;em&gt;placeholder&lt;/em&gt; html",
+            "string <em>placeholder</em> html",
+        ),
+        (
+            "string & entity",
+            {},
+            "string &amp; entity",
+            "string &amp; entity",
+            "string & entity",
+        ),
+    ],
+)
+def test_field_handles_html(
+    content, values, expected_stripped, expected_escaped, expected_passthrough
+):
+    assert str(Field(content, values)) == expected_stripped
+    assert str(Field(content, values, html="strip")) == expected_stripped
+    assert str(Field(content, values, html="escape")) == expected_escaped
+    assert str(Field(content, values, html="passthrough")) == expected_passthrough

tests/notifications_utils/test_formatters.py

@@ -0,0 +1,577 @@
+import pytest
+from markupsafe import Markup
+
+from notifications_utils.formatters import (
+    autolink_urls,
+    escape_html,
+    formatted_list,
+    make_quotes_smart,
+    normalise_whitespace,
+    remove_smart_quotes_from_email_addresses,
+    remove_whitespace_before_punctuation,
+    replace_hyphens_with_en_dashes,
+    sms_encode,
+    strip_all_whitespace,
+    strip_and_remove_obscure_whitespace,
+    strip_unsupported_characters,
+    unlink_govuk_escaped,
+)
+from notifications_utils.template import (
+    HTMLEmailTemplate,
+    PlainTextEmailTemplate,
+    SMSMessageTemplate,
+    SMSPreviewTemplate,
+)
+
+
+@pytest.mark.parametrize(
+    "url, expected_html",
+    [
+        (
+            """https://example.com/"onclick="alert('hi')""",
+            """<a class="govuk-link govuk-link--no-visited-state" href="https://example.com/%22onclick=%22alert%28%27hi%27%29">https://example.com/"onclick="alert('hi')</a>""",  # noqa
+        ),
+        (
+            """https://example.com/"style='text-decoration:blink'""",
+            """<a class="govuk-link govuk-link--no-visited-state" href="https://example.com/%22style=%27text-decoration:blink%27">https://example.com/"style='text-decoration:blink'</a>""",  # noqa
+        ),
+    ],
+)
+def test_URLs_get_escaped_in_sms(url, expected_html):
+    assert expected_html in str(
+        SMSPreviewTemplate({"content": url, "template_type": "sms"})
+    )
+
+
+def test_HTML_template_has_URLs_replaced_with_links():
+    assert (
+        '<a style="word-wrap: break-word; color: #1D70B8;" href="https://service.example.com/accept_invite/a1b2c3d4">'
+        "https://service.example.com/accept_invite/a1b2c3d4"
+        "</a>"
+    ) in str(
+        HTMLEmailTemplate(
+            {
+                "content": (
+                    "You’ve been invited to a service. Click this link:\n"
+                    "https://service.example.com/accept_invite/a1b2c3d4\n"
+                    "\n"
+                    "Thanks\n"
+                ),
+                "subject": "",
+                "template_type": "email",
+            }
+        )
+    )
+
+
+def test_escaping_govuk_in_email_templates():
+    template_content = "GOV.UK"
+    expected = "GOV.\u200BUK"
+    assert unlink_govuk_escaped(template_content) == expected
+    template_json = {
+        "content": template_content,
+        "subject": "",
+        "template_type": "email",
+    }
+    assert expected in str(PlainTextEmailTemplate(template_json))
+    assert expected in str(HTMLEmailTemplate(template_json))
+
+
+@pytest.mark.parametrize(
+    "template_content,expected",
+    [
+        # Cases that we add the breaking space
+        ("GOV.UK", "GOV.\u200BUK"),
+        ("gov.uk", "gov.\u200Buk"),
+        (
+            "content with space infront GOV.UK",
+            "content with space infront GOV.\u200BUK",
+        ),
+        ("content with tab infront\tGOV.UK", "content with tab infront\tGOV.\u200BUK"),
+        (
+            "content with newline infront\nGOV.UK",
+            "content with newline infront\nGOV.\u200BUK",
+        ),
+        ("*GOV.UK", "*GOV.\u200BUK"),
+        ("#GOV.UK", "#GOV.\u200BUK"),
+        ("^GOV.UK", "^GOV.\u200BUK"),
+        (" #GOV.UK", " #GOV.\u200BUK"),
+        ("GOV.UK with CONTENT after", "GOV.\u200BUK with CONTENT after"),
+        ("#GOV.UK with CONTENT after", "#GOV.\u200BUK with CONTENT after"),
+        # Cases that we don't add the breaking space
+        ("https://gov.uk", "https://gov.uk"),
+        ("https://www.gov.uk", "https://www.gov.uk"),
+        ("www.gov.uk", "www.gov.uk"),
+        ("WWW.GOV.UK", "WWW.GOV.UK"),
+        ("WWW.GOV.UK.", "WWW.GOV.UK."),
+        (
+            "https://www.gov.uk/?utm_source=gov.uk",
+            "https://www.gov.uk/?utm_source=gov.uk",
+        ),
+        ("mygov.uk", "mygov.uk"),
+        ("www.this-site-is-not-gov.uk", "www.this-site-is-not-gov.uk"),
+        (
+            "www.gov.uk?websites=bbc.co.uk;gov.uk;nsh.scot",
+            "www.gov.uk?websites=bbc.co.uk;gov.uk;nsh.scot",
+        ),
+        ("reply to: xxxx@xxx.gov.uk", "reply to: xxxx@xxx.gov.uk"),
+        ("southwark.gov.uk", "southwark.gov.uk"),
+        ("data.gov.uk", "data.gov.uk"),
+        ("gov.uk/foo", "gov.uk/foo"),
+        ("*GOV.UK/foo", "*GOV.UK/foo"),
+        ("#GOV.UK/foo", "#GOV.UK/foo"),
+        ("^GOV.UK/foo", "^GOV.UK/foo"),
+        ("gov.uk#departments-and-policy", "gov.uk#departments-and-policy"),
+        # Cases that we know currently aren't supported by our regex and have a non breaking space added when they
+        # shouldn't however, we accept the fact that our regex isn't perfect as we think the chance of a user using a
+        # URL like this in their content is very small.
+        # We document these edge cases here
+        pytest.param("gov.uk.com", "gov.uk.com", marks=pytest.mark.xfail),
+        pytest.param("gov.ukandi.com", "gov.ukandi.com", marks=pytest.mark.xfail),
+        pytest.param("gov.uks", "gov.uks", marks=pytest.mark.xfail),
+    ],
+)
+def test_unlink_govuk_escaped(template_content, expected):
+    assert unlink_govuk_escaped(template_content) == expected
+
+
+@pytest.mark.parametrize(
+    "prefix, body, expected",
+    [
+        ("a", "b", "a: b"),
+        (None, "b", "b"),
+    ],
+)
+def test_sms_message_adds_prefix(prefix, body, expected):
+    template = SMSMessageTemplate({"content": body, "template_type": "sms"})
+    template.prefix = prefix
+    template.sender = None
+    assert str(template) == expected
+
+
+def test_sms_preview_adds_newlines():
+    template = SMSPreviewTemplate(
+        {
+            "content": """
+        the
+        quick
+
+        brown fox
+    """,
+            "template_type": "sms",
+        }
+    )
+    template.prefix = None
+    template.sender = None
+    assert "<br>" in str(template)
+
+
+def test_sms_encode(mocker):
+    sanitise_mock = mocker.patch("notifications_utils.formatters.SanitiseSMS")
+    assert sms_encode("foo") == sanitise_mock.encode.return_value
+    sanitise_mock.encode.assert_called_once_with("foo")
+
+
+@pytest.mark.parametrize(
+    "items, kwargs, expected_output",
+    [
+        ([1], {}, "‘1’"),
+        ([1, 2], {}, "‘1’ and ‘2’"),
+        ([1, 2, 3], {}, "‘1’, ‘2’ and ‘3’"),
+        ([1, 2, 3], {"prefix": "foo", "prefix_plural": "bar"}, "bar ‘1’, ‘2’ and ‘3’"),
+        ([1], {"prefix": "foo", "prefix_plural": "bar"}, "foo ‘1’"),
+        ([1, 2, 3], {"before_each": "a", "after_each": "b"}, "a1b, a2b and a3b"),
+        ([1, 2, 3], {"conjunction": "foo"}, "‘1’, ‘2’ foo ‘3’"),
+        (["&"], {"before_each": "<i>", "after_each": "</i>"}, "<i>&amp;</i>"),
+        (
+            [1, 2, 3],
+            {"before_each": "<i>", "after_each": "</i>"},
+            "<i>1</i>, <i>2</i> and <i>3</i>",
+        ),
+    ],
+)
+def test_formatted_list(items, kwargs, expected_output):
+    assert formatted_list(items, **kwargs) == expected_output
+
+
+def test_formatted_list_returns_markup():
+    assert isinstance(formatted_list([0]), Markup)
+
+
+def test_bleach_doesnt_try_to_make_valid_html_before_cleaning():
+    assert escape_html("<to cancel daily cat facts reply 'cancel'>") == (
+        "&lt;to cancel daily cat facts reply 'cancel'&gt;"
+    )
+
+
+@pytest.mark.parametrize(
+    "content, expected_escaped",
+    (
+        ("&?a;", "&amp;?a;"),
+        ("&>a;", "&amp;&gt;a;"),
+        ("&*a;", "&amp;*a;"),
+        ("&a?;", "&amp;a?;"),
+        ("&x?xa;", "&amp;x?xa;"),
+        # We need to be careful that query arguments don’t get turned into entities
+        ("&timestamp=&times;", "&amp;timestamp=×"),
+        ("&times=1,2,3", "&amp;times=1,2,3"),
+        # &minus; should have a trailing semicolon according to the HTML5
+        # spec but &micro doesn’t need one
+        ("2&minus;1", "2−1"),
+        ("200&micro;g", "200µg"),
+        # …we ignore it when it’s ambiguous
+        ("2&minus1", "2&amp;minus1"),
+        ("200&microg", "200&amp;microg"),
+        # …we still ignore when there’s a space afterwards
+        ("2 &minus 1", "2 &amp;minus 1"),
+        ("200&micro g", "200&amp;micro g"),
+        # Things which aren’t real entities are ignored, not removed
+        ("This &isnotarealentity;", "This &amp;isnotarealentity;"),
+        # We let users use &nbsp; for backwards compatibility
+        ("Before&nbsp;after", "Before&nbsp;after"),
+        # We let users use &amp; because it’s often pasted in URLs
+        ("?a=1&amp;b=2", "?a=1&amp;b=2"),
+        # We let users use &lpar; and &rpar; because otherwise it’s
+        # impossible to put brackets in the body of conditional placeholders
+        ("((var??&lpar;in brackets&rpar;))", "((var??&lpar;in brackets&rpar;))"),
+    ),
+)
+def test_escaping_html_entities(
+    content,
+    expected_escaped,
+):
+    assert escape_html(content) == expected_escaped
+
+
+@pytest.mark.parametrize(
+    "dirty, clean",
+    [
+        (
+            "Hello ((name)) ,\n\nThis is a message",
+            "Hello ((name)),\n\nThis is a message",
+        ),
+        ("Hello Jo ,\n\nThis is a message", "Hello Jo,\n\nThis is a message"),
+        (
+            "\n   \t    , word",
+            "\n, word",
+        ),
+    ],
+)
+def test_removing_whitespace_before_commas(dirty, clean):
+    assert remove_whitespace_before_punctuation(dirty) == clean
+
+
+@pytest.mark.parametrize(
+    "dirty, clean",
+    [
+        (
+            "Hello ((name)) .\n\nThis is a message",
+            "Hello ((name)).\n\nThis is a message",
+        ),
+        ("Hello Jo .\n\nThis is a message", "Hello Jo.\n\nThis is a message"),
+        (
+            "\n   \t    . word",
+            "\n. word",
+        ),
+    ],
+)
+def test_removing_whitespace_before_full_stops(dirty, clean):
+    assert remove_whitespace_before_punctuation(dirty) == clean
+
+
+@pytest.mark.parametrize(
+    "dumb, smart",
+    [
+        (
+            """And I said, "what about breakfast at Tiffany's"?""",
+            """And I said, “what about breakfast at Tiffany’s”?""",
+        ),
+        (
+            """
+            <a href="http://example.com?q='foo'">http://example.com?q='foo'</a>
+        """,
+            """
+            <a href="http://example.com?q='foo'">http://example.com?q='foo'</a>
+        """,
+        ),
+    ],
+)
+def test_smart_quotes(dumb, smart):
+    assert make_quotes_smart(dumb) == smart
+
+
+@pytest.mark.parametrize(
+    "nasty, nice",
+    [
+        (
+            (
+                "The en dash - always with spaces in running text when, as "
+                "discussed in this section, indicating a parenthesis or "
+                "pause - and the spaced em dash both have a certain "
+                "technical advantage over the unspaced em dash. "
+            ),
+            (
+                "The en dash \u2013 always with spaces in running text when, as "
+                "discussed in this section, indicating a parenthesis or "
+                "pause \u2013 and the spaced em dash both have a certain "
+                "technical advantage over the unspaced em dash. "
+            ),
+        ),
+        (
+            "double -- dash",
+            "double \u2013 dash",
+        ),
+        (
+            "triple --- dash",
+            "triple \u2013 dash",
+        ),
+        (
+            "quadruple ---- dash",
+            "quadruple ---- dash",
+        ),
+        (
+            "em — dash",
+            "em – dash",
+        ),
+        (
+            "already\u0020–\u0020correct",  # \u0020 is a normal space character
+            "already\u0020–\u0020correct",
+        ),
+        (
+            "2004-2008",
+            "2004-2008",  # no replacement
+        ),
+    ],
+)
+def test_en_dashes(nasty, nice):
+    assert replace_hyphens_with_en_dashes(nasty) == nice
+
+
+def test_unicode_dash_lookup():
+    en_dash_replacement_sequence = "\u0020\u2013"
+    hyphen = "-"
+    en_dash = "–"
+    space = " "
+    non_breaking_space = " "
+    assert en_dash_replacement_sequence == space + en_dash
+    assert non_breaking_space not in en_dash_replacement_sequence
+    assert hyphen not in en_dash_replacement_sequence
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "bar",
+        " bar ",
+        """
+        \t    bar
+    """,
+        " \u180E\u200B \u200C bar \u200D \u2060\uFEFF ",
+    ],
+)
+def test_strip_all_whitespace(value):
+    assert strip_all_whitespace(value) == "bar"
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "notifications-email",
+        "  \tnotifications-email \x0c ",
+        "\rn\u200Coti\u200Dfi\u200Bcati\u2060ons-\u180Eemai\uFEFFl\uFEFF",
+    ],
+)
+def test_strip_and_remove_obscure_whitespace(value):
+    assert strip_and_remove_obscure_whitespace(value) == "notifications-email"
+
+
+def test_strip_and_remove_obscure_whitespace_only_removes_normal_whitespace_from_ends():
+    sentence = "   words \n over multiple lines with \ttabs\t   "
+    assert (
+        strip_and_remove_obscure_whitespace(sentence)
+        == "words \n over multiple lines with \ttabs"
+    )
+
+
+def test_remove_smart_quotes_from_email_addresses():
+    assert (
+        remove_smart_quotes_from_email_addresses(
+            """
+        line one’s quote
+        first.o’last@example.com is someone’s email address
+        line ‘three’
+    """
+        )
+        == (
+            """
+        line one’s quote
+        first.o'last@example.com is someone’s email address
+        line ‘three’
+    """
+        )
+    )
+
+
+def test_strip_unsupported_characters():
+    assert strip_unsupported_characters("line one\u2028line two") == (
+        "line oneline two"
+    )
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "\u200C Your tax   is\ndue\n\n",
+        "  Your tax is due  ",
+        # Non breaking spaces replaced by single spaces
+        "\u00A0Your\u00A0tax\u00A0 is\u00A0\u00A0due\u00A0",
+        # zero width spaces are removed
+        "\u180EYour \u200Btax\u200C is \u200D\u2060due \uFEFF",
+        # tabs are replaced by single spaces
+        "\tYour tax\tis due  ",
+    ],
+)
+def test_normalise_whitespace(value):
+    assert normalise_whitespace(value) == "Your tax is due"
+
+
+@pytest.mark.parametrize(
+    "content, expected_html",
+    (
+        (
+            "http://example.com",
+            '<a href="http://example.com">http://example.com</a>',
+        ),
+        (
+            "https://example.com",
+            '<a href="https://example.com">https://example.com</a>',
+        ),
+        (
+            "example.com",
+            '<a href="http://example.com">example.com</a>',
+        ),
+        (
+            "www.foo.bar.example.com",
+            '<a href="http://www.foo.bar.example.com">www.foo.bar.example.com</a>',
+        ),
+        (
+            "example.com/",
+            '<a href="http://example.com/">example.com/</a>',
+        ),
+        (
+            "www.foo.bar.example.com/",
+            '<a href="http://www.foo.bar.example.com/">www.foo.bar.example.com/</a>',
+        ),
+        (
+            "example.com/foo",
+            '<a href="http://example.com/foo">example.com/foo</a>',
+        ),
+        (
+            "example.com?foo",
+            '<a href="http://example.com?foo">example.com?foo</a>',
+        ),
+        (
+            "example.com#foo",
+            '<a href="http://example.com#foo">example.com#foo</a>',
+        ),
+        (
+            "Go to gov.uk/example.",
+            "Go to " '<a href="http://gov.uk/example">gov.uk/example</a>.',
+        ),
+        (
+            "Go to gov.uk/example:",
+            "Go to " '<a href="http://gov.uk/example">gov.uk/example</a>:',
+        ),
+        (
+            "Go to gov.uk/example;",
+            "Go to " '<a href="http://gov.uk/example;">gov.uk/example;</a>',
+        ),
+        (
+            "(gov.uk/example)",
+            "(" '<a href="http://gov.uk/example">gov.uk/example</a>)',
+        ),
+        (
+            "(gov.uk/example)...",
+            "(" '<a href="http://gov.uk/example">gov.uk/example</a>)...',
+        ),
+        (
+            "(gov.uk/example.)",
+            "(" '<a href="http://gov.uk/example">gov.uk/example</a>.)',
+        ),
+        (
+            "(see example.com/foo_(bar))",
+            "(see "
+            '<a href="http://example.com/foo_%28bar%29">example.com/foo_(bar)</a>)',
+        ),
+        (
+            "example.com/foo(((((((bar",
+            '<a href="http://example.com/foo%28%28%28%28%28%28%28bar">example.com/foo(((((((bar</a>',
+        ),
+        (
+            "government website (gov.uk). Other websites…",
+            "government website ("
+            '<a href="http://gov.uk">gov.uk</a>). Other websites…',
+        ),
+        (
+            "[gov.uk/example]",
+            "[" '<a href="http://gov.uk/example">gov.uk/example</a>]',
+        ),
+        (
+            "gov.uk/foo, gov.uk/bar",
+            '<a href="http://gov.uk/foo">gov.uk/foo</a>, '
+            '<a href="http://gov.uk/bar">gov.uk/bar</a>',
+        ),
+        (
+            "<p>gov.uk/foo</p>",
+            "<p>" '<a href="http://gov.uk/foo">gov.uk/foo</a></p>',
+        ),
+        (
+            "gov.uk?foo&amp;",
+            '<a href="http://gov.uk?foo&amp;">gov.uk?foo&amp;</a>',
+        ),
+        (
+            "a .service.gov.uk domain",
+            "a .service.gov.uk domain",
+        ),
+        (
+            'http://foo.com/"bar"?x=1#2',
+            '<a href="http://foo.com/%22bar%22?x=1#2">http://foo.com/"bar"?x=1#2</a>',
+        ),
+        (
+            "firstname.lastname@example.com",
+            "firstname.lastname@example.com",
+        ),
+        (
+            "with-subdomain@test.example.com",
+            "with-subdomain@test.example.com",
+        ),
+    ),
+)
+def test_autolink_urls_matches_correctly(content, expected_html):
+    assert autolink_urls(content) == expected_html
+
+
+@pytest.mark.parametrize(
+    "extra_kwargs, expected_html",
+    (
+        (
+            {},
+            '<a href="http://example.com">http://example.com</a>',
+        ),
+        (
+            {
+                "classes": "govuk-link",
+            },
+            '<a class="govuk-link" href="http://example.com">http://example.com</a>',
+        ),
+    ),
+)
+def test_autolink_urls_applies_correct_attributes(extra_kwargs, expected_html):
+    assert autolink_urls("http://example.com", **extra_kwargs) == expected_html
+
+
+@pytest.mark.parametrize(
+    "content", ("without link", "with link to https://example.com")
+)
+def test_autolink_urls_returns_markup(content):
+    assert isinstance(autolink_urls(content), Markup)

tests/notifications_utils/test_insensitive_dict.py

@@ -0,0 +1,96 @@
+from functools import partial
+
+import pytest
+
+from notifications_utils.insensitive_dict import InsensitiveDict
+from notifications_utils.recipients import Cell, Row
+
+
+def test_columns_as_dict_with_keys():
+    assert InsensitiveDict(
+        {"Date of Birth": "01/01/2001", "TOWN": "London"}
+    ).as_dict_with_keys({"date_of_birth", "town"}) == {
+        "date_of_birth": "01/01/2001",
+        "town": "London",
+    }
+
+
+def test_columns_as_dict():
+    assert dict(InsensitiveDict({"date of birth": "01/01/2001", "TOWN": "London"})) == {
+        "dateofbirth": "01/01/2001",
+        "town": "London",
+    }
+
+
+def test_missing_data():
+    partial_row = partial(
+        Row,
+        row_dict={},
+        index=1,
+        error_fn=None,
+        recipient_column_headers=[],
+        placeholders=[],
+        template=None,
+        allow_international_letters=False,
+    )
+    with pytest.raises(KeyError):
+        InsensitiveDict({})["foo"]
+    assert InsensitiveDict({}).get("foo") is None
+    assert InsensitiveDict({}).get("foo", "bar") == "bar"
+    assert partial_row()["foo"] == Cell()
+    assert partial_row().get("foo") == Cell()
+    assert partial_row().get("foo", "bar") == "bar"
+
+
+@pytest.mark.parametrize(
+    "in_dictionary",
+    [
+        {"foo": "bar"},
+        {"F_O O": "bar"},
+    ],
+)
+@pytest.mark.parametrize(
+    "key, should_be_present",
+    [
+        ("foo", True),
+        ("f_o_o", True),
+        ("F O O", True),
+        ("bar", False),
+    ],
+)
+def test_lookup(key, should_be_present, in_dictionary):
+    assert (key in InsensitiveDict(in_dictionary)) == should_be_present
+
+
+@pytest.mark.parametrize(
+    "key_in",
+    [
+        "foo",
+        "F_O O",
+    ],
+)
+@pytest.mark.parametrize(
+    "lookup_key",
+    [
+        "foo",
+        "f_o_o",
+        "F O O",
+    ],
+)
+def test_set_item(key_in, lookup_key):
+    columns = InsensitiveDict({})
+    columns[key_in] = "bar"
+    assert columns[lookup_key] == "bar"
+
+
+def test_maintains_insertion_order():
+    d = InsensitiveDict(
+        {
+            "B": None,
+            "A": None,
+            "C": None,
+        }
+    )
+    assert d.keys() == ["b", "a", "c"]
+    d["BB"] = None
+    assert d.keys() == ["b", "a", "c", "bb"]

tests/notifications_utils/test_international_billing_rates.py

@@ -0,0 +1,50 @@
+import pytest
+
+from notifications_utils.international_billing_rates import (
+    COUNTRY_PREFIXES,
+    INTERNATIONAL_BILLING_RATES,
+)
+from notifications_utils.recipients import use_numeric_sender
+
+
+def test_international_billing_rates_exists():
+    assert INTERNATIONAL_BILLING_RATES["1"]["names"][0] == "Canada"
+
+
+@pytest.mark.parametrize(
+    "country_prefix, values", sorted(INTERNATIONAL_BILLING_RATES.items())
+)
+def test_international_billing_rates_are_in_correct_format(country_prefix, values):
+    assert isinstance(country_prefix, str)
+    # we don't want the prefixes to have + at the beginning for instance
+    assert country_prefix.isdigit()
+
+    assert set(values.keys()) == {"attributes", "billable_units", "names"}
+
+    assert isinstance(values["billable_units"], int)
+    assert 1 <= values["billable_units"] <= 3
+
+    assert isinstance(values["names"], list)
+    assert all(isinstance(country, str) for country in values["names"])
+
+    assert isinstance(values["attributes"], dict)
+    assert values["attributes"]["dlr"] is None or isinstance(
+        values["attributes"]["dlr"], str
+    )
+
+
+def test_country_codes():
+    assert len(COUNTRY_PREFIXES) == 214
+
+
+@pytest.mark.parametrize(
+    "number, expected",
+    [
+        ("+48123654789", False),  # Poland alpha: Yes
+        ("+1-403-123-5687", True),  # Canada alpha: No
+        ("+40123548897", False),  # Romania alpha: REG
+        ("+60123451345", True),
+    ],
+)  # Malaysia alpha: NO
+def test_use_numeric_sender(number, expected):
+    assert use_numeric_sender(number) == expected

tests/notifications_utils/test_letter_timings.py

@@ -0,0 +1,269 @@
+from datetime import datetime
+
+import pytest
+import pytz
+from freezegun import freeze_time
+
+from notifications_utils.letter_timings import (
+    get_letter_timings,
+    letter_can_be_cancelled,
+)
+
+
+@freeze_time("2017-07-14 13:59:59")  # Friday, before print deadline (3PM EST)
+@pytest.mark.parametrize(
+    "upload_time, expected_print_time, is_printed, first_class, expected_earliest, expected_latest",
+    [
+        # EST
+        # ==================================================================
+        #  First thing Monday
+        (
+            "Monday 2017-07-10 00:00:01",
+            "Tuesday 2017-07-11 15:00",
+            True,
+            "Wednesday 2017-07-12 16:00",
+            "Thursday 2017-07-13 16:00",
+            "Friday 2017-07-14 16:00",
+        ),
+        #  Monday at 17:29 EST (sent on monday)
+        (
+            "Monday 2017-07-10 16:29:59",
+            "Tuesday 2017-07-11 15:00",
+            True,
+            "Wednesday 2017-07-12 16:00",
+            "Thursday 2017-07-13 16:00",
+            "Friday 2017-07-14 16:00",
+        ),
+        #  Monday at 17:30 EST (sent on tuesday)
+        (
+            "Monday 2017-07-10 16:30:01",
+            "Wednesday 2017-07-12 15:00",
+            True,
+            "Thursday 2017-07-13 16:00",
+            "Friday 2017-07-14 16:00",
+            "Saturday 2017-07-15 16:00",
+        ),
+        #  Tuesday before 17:30 EST
+        (
+            "Tuesday 2017-07-11 12:00:00",
+            "Wednesday 2017-07-12 15:00",
+            True,
+            "Thursday 2017-07-13 16:00",
+            "Friday 2017-07-14 16:00",
+            "Saturday 2017-07-15 16:00",
+        ),
+        #  Wednesday before 17:30 EST
+        (
+            "Wednesday 2017-07-12 12:00:00",
+            "Thursday 2017-07-13 15:00",
+            True,
+            "Friday 2017-07-14 16:00",
+            "Saturday 2017-07-15 16:00",
+            "Monday 2017-07-17 16:00",
+        ),
+        #  Thursday before 17:30 EST
+        (
+            "Thursday 2017-07-13 12:00:00",
+            "Friday 2017-07-14 15:00",
+            False,
+            "Saturday 2017-07-15 16:00",
+            "Monday 2017-07-17 16:00",
+            "Tuesday 2017-07-18 16:00",
+        ),
+        #  Friday anytime
+        (
+            "Friday 2017-07-14 00:00:00",
+            "Monday 2017-07-17 15:00",
+            False,
+            "Tuesday 2017-07-18 16:00",
+            "Wednesday 2017-07-19 16:00",
+            "Thursday 2017-07-20 16:00",
+        ),
+        (
+            "Friday 2017-07-14 12:00:00",
+            "Monday 2017-07-17 15:00",
+            False,
+            "Tuesday 2017-07-18 16:00",
+            "Wednesday 2017-07-19 16:00",
+            "Thursday 2017-07-20 16:00",
+        ),
+        (
+            "Friday 2017-07-14 22:00:00",
+            "Monday 2017-07-17 15:00",
+            False,
+            "Tuesday 2017-07-18 16:00",
+            "Wednesday 2017-07-19 16:00",
+            "Thursday 2017-07-20 16:00",
+        ),
+        #  Saturday anytime
+        (
+            "Saturday 2017-07-14 12:00:00",
+            "Monday 2017-07-17 15:00",
+            False,
+            "Tuesday 2017-07-18 16:00",
+            "Wednesday 2017-07-19 16:00",
+            "Thursday 2017-07-20 16:00",
+        ),
+        #  Sunday before 1730 EST
+        (
+            "Sunday 2017-07-15 15:59:59",
+            "Monday 2017-07-17 15:00",
+            False,
+            "Tuesday 2017-07-18 16:00",
+            "Wednesday 2017-07-19 16:00",
+            "Thursday 2017-07-20 16:00",
+        ),
+        #  Sunday after 17:30 EST
+        (
+            "Sunday 2017-07-16 16:30:01",
+            "Tuesday 2017-07-18 15:00",
+            False,
+            "Wednesday 2017-07-19 16:00",
+            "Thursday 2017-07-20 16:00",
+            "Friday 2017-07-21 16:00",
+        ),
+        # GMT
+        # ==================================================================
+        #  Monday at 17:29 GMT
+        (
+            "Monday 2017-01-02 17:29:59",
+            "Tuesday 2017-01-03 15:00",
+            True,
+            "Wednesday 2017-01-04 16:00",
+            "Thursday 2017-01-05 16:00",
+            "Friday 2017-01-06 16:00",
+        ),
+        #  Monday at 17:00 GMT
+        (
+            "Monday 2017-01-02 17:30:01",
+            "Wednesday 2017-01-04 15:00",
+            True,
+            "Thursday 2017-01-05 16:00",
+            "Friday 2017-01-06 16:00",
+            "Saturday 2017-01-07 16:00",
+        ),
+    ],
+)
+@pytest.mark.skip(reason="Letters being developed later")
+def test_get_estimated_delivery_date_for_letter(
+    upload_time,
+    expected_print_time,
+    is_printed,
+    first_class,
+    expected_earliest,
+    expected_latest,
+):
+    # remove the day string from the upload_time, which is purely informational
+
+    def format_dt(x):
+        return x.astimezone(pytz.timezone("America/New_York")).strftime(
+            "%A %Y-%m-%d %H:%M"
+        )
+
+    upload_time = upload_time.split(" ", 1)[1]
+
+    timings = get_letter_timings(upload_time, postage="second")
+
+    assert format_dt(timings.printed_by) == expected_print_time
+    assert timings.is_printed == is_printed
+    assert format_dt(timings.earliest_delivery) == expected_earliest
+    assert format_dt(timings.latest_delivery) == expected_latest
+
+    first_class_timings = get_letter_timings(upload_time, postage="first")
+
+    assert format_dt(first_class_timings.printed_by) == expected_print_time
+    assert first_class_timings.is_printed == is_printed
+    assert format_dt(first_class_timings.earliest_delivery) == first_class
+    assert format_dt(first_class_timings.latest_delivery) == first_class
+
+
+@pytest.mark.parametrize("status", ["sending", "pending"])
+def test_letter_cannot_be_cancelled_if_letter_status_is_not_created_or_pending_virus_check(
+    status,
+):
+    notification_created_at = datetime.utcnow()
+
+    assert not letter_can_be_cancelled(status, notification_created_at)
+
+
+@freeze_time("2018-7-7 16:00:00")
+@pytest.mark.parametrize(
+    "notification_created_at",
+    [
+        datetime(2018, 7, 6, 18, 0),  # created yesterday after 1730
+        datetime(2018, 7, 7, 12, 0),  # created today
+    ],
+)
+@pytest.mark.skip(reason="Letters not part of release")
+def test_letter_can_be_cancelled_if_before_1730_and_letter_created_before_1730(
+    notification_created_at,
+):
+    notification_status = "pending-virus-check"
+
+    assert letter_can_be_cancelled(notification_status, notification_created_at)
+
+
+@freeze_time("2017-12-12 17:30:00")
+@pytest.mark.parametrize(
+    "notification_created_at",
+    [
+        datetime(2017, 12, 12, 17, 0),
+        datetime(2017, 12, 12, 17, 30),
+    ],
+)
+@pytest.mark.skip(reason="Letters not part of release")
+def test_letter_cannot_be_cancelled_if_1730_exactly_and_letter_created_at_or_before_1730(
+    notification_created_at,
+):
+    notification_status = "pending-virus-check"
+
+    assert not letter_can_be_cancelled(notification_status, notification_created_at)
+
+
+@freeze_time("2018-7-7 19:00:00")
+@pytest.mark.parametrize(
+    "notification_created_at",
+    [
+        datetime(2018, 7, 6, 18, 0),  # created yesterday after 1730
+        datetime(2018, 7, 7, 12, 0),  # created today before 1730
+    ],
+)
+@pytest.mark.skip(reason="Letters not part of release")
+def test_letter_cannot_be_cancelled_if_after_1730_and_letter_created_before_1730(
+    notification_created_at,
+):
+    notification_status = "created"
+
+    assert not letter_can_be_cancelled(notification_status, notification_created_at)
+
+
+@freeze_time("2018-7-7 15:00:00")
+@pytest.mark.skip(reason="Letters not part of release")
+def test_letter_cannot_be_cancelled_if_before_1730_and_letter_created_before_1730_yesterday():
+    notification_status = "created"
+
+    assert not letter_can_be_cancelled(notification_status, datetime(2018, 7, 6, 14, 0))
+
+
+@freeze_time("2018-7-7 15:00:00")
+@pytest.mark.skip(reason="Letters not part of release")
+def test_letter_cannot_be_cancelled_if_before_1730_and_letter_created_after_1730_two_days_ago():
+    notification_status = "created"
+
+    assert not letter_can_be_cancelled(notification_status, datetime(2018, 7, 5, 19, 0))
+
+
+@freeze_time("2018-7-7 19:00:00")
+@pytest.mark.parametrize(
+    "notification_created_at",
+    [
+        datetime(2018, 7, 7, 18, 30),
+        datetime(2018, 7, 7, 19, 0),
+    ],
+)
+def test_letter_can_be_cancelled_if_after_1730_and_letter_created_at_1730_today_or_later(
+    notification_created_at,
+):
+    notification_status = "created"
+
+    assert letter_can_be_cancelled(notification_status, notification_created_at)

tests/notifications_utils/test_logging.py

@@ -0,0 +1,51 @@
+import json
+import logging as builtin_logging
+
+from notifications_utils import logging
+
+
+def test_get_handlers_sets_up_logging_appropriately_with_debug():
+    class App:
+        config = {"NOTIFY_APP_NAME": "bar", "NOTIFY_LOG_LEVEL": "ERROR"}
+        debug = True
+
+    app = App()
+
+    handlers = logging.get_handlers(app)
+
+    assert len(handlers) == 1
+    assert isinstance(handlers[0], builtin_logging.StreamHandler)
+    assert isinstance(handlers[0].formatter, builtin_logging.Formatter)
+
+
+def test_get_handlers_sets_up_logging_appropriately_without_debug():
+    class App:
+        config = {"NOTIFY_APP_NAME": "bar", "NOTIFY_LOG_LEVEL": "ERROR"}
+        debug = False
+
+    app = App()
+
+    handlers = logging.get_handlers(app)
+
+    assert len(handlers) == 1
+    assert isinstance(handlers[0], builtin_logging.StreamHandler)
+    assert isinstance(handlers[0].formatter, logging.JSONFormatter)
+
+
+def test_base_json_formatter_contains_service_id():
+    record = builtin_logging.LogRecord(
+        name="log thing",
+        level="info",
+        pathname="path",
+        lineno=123,
+        msg="message to log",
+        exc_info=None,
+        args=None,
+    )
+
+    service_id_filter = logging.ServiceIdFilter()
+    assert (
+        json.loads(logging.BaseJSONFormatter().format(record))["message"]
+        == "message to log"
+    )
+    assert service_id_filter.filter(record).service_id == "no-service-id"

tests/notifications_utils/test_markdown.py

@@ -0,0 +1,667 @@
+import pytest
+
+from notifications_utils.markdown import (
+    notify_email_markdown,
+    notify_letter_preview_markdown,
+    notify_plain_text_email_markdown,
+)
+from notifications_utils.template import HTMLEmailTemplate
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "http://example.com",
+        "http://www.gov.uk/",
+        "https://www.gov.uk/",
+        "http://service.gov.uk",
+        "http://service.gov.uk/blah.ext?q=a%20b%20c&order=desc#fragment",
+        pytest.param(
+            "http://service.gov.uk/blah.ext?q=one two three", marks=pytest.mark.xfail
+        ),
+    ],
+)
+def test_makes_links_out_of_URLs(url):
+    link = '<a style="word-wrap: break-word; color: #1D70B8;" href="{}">{}</a>'.format(
+        url, url
+    )
+    assert notify_email_markdown(url) == (
+        '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+        "{}"
+        "</p>"
+    ).format(link)
+
+
+@pytest.mark.parametrize(
+    "input, output",
+    [
+        (
+            ("this is some text with a link http://example.com in the middle"),
+            (
+                "this is some text with a link "
+                '<a style="word-wrap: break-word; color: #1D70B8;" href="http://example.com">http://example.com</a>'
+                " in the middle"
+            ),
+        ),
+        (
+            ("this link is in brackets (http://example.com)"),
+            (
+                "this link is in brackets "
+                '(<a style="word-wrap: break-word; color: #1D70B8;" href="http://example.com">http://example.com</a>)'
+            ),
+        ),
+    ],
+)
+def test_makes_links_out_of_URLs_in_context(input, output):
+    assert notify_email_markdown(input) == (
+        '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+        "{}"
+        "</p>"
+    ).format(output)
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "example.com",
+        "www.example.com",
+        "ftp://example.com",
+        "test@example.com",
+        "mailto:test@example.com",
+        '<a href="https://example.com">Example</a>',
+    ],
+)
+def test_doesnt_make_links_out_of_invalid_urls(url):
+    assert notify_email_markdown(url) == (
+        '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+        "{}"
+        "</p>"
+    ).format(url)
+
+
+def test_handles_placeholders_in_urls():
+    assert notify_email_markdown(
+        "http://example.com/?token=<span class='placeholder'>((token))</span>&key=1"
+    ) == (
+        '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+        '<a style="word-wrap: break-word; color: #1D70B8;" href="http://example.com/?token=">'
+        "http://example.com/?token="
+        "</a>"
+        "<span class='placeholder'>((token))</span>&amp;key=1"
+        "</p>"
+    )
+
+
+@pytest.mark.parametrize(
+    "url, expected_html, expected_html_in_template",
+    [
+        (
+            """https://example.com"onclick="alert('hi')""",
+            """<a style="word-wrap: break-word; color: #1D70B8;" href="https://example.com%22onclick=%22alert%28%27hi">https://example.com"onclick="alert('hi</a>')""",  # noqa
+            """<a style="word-wrap: break-word; color: #1D70B8;" href="https://example.com%22onclick=%22alert%28%27hi">https://example.com"onclick="alert('hi</a>‘)""",  # noqa
+        ),
+        (
+            """https://example.com"style='text-decoration:blink'""",
+            """<a style="word-wrap: break-word; color: #1D70B8;" href="https://example.com%22style=%27text-decoration:blink">https://example.com"style='text-decoration:blink</a>'""",  # noqa
+            """<a style="word-wrap: break-word; color: #1D70B8;" href="https://example.com%22style=%27text-decoration:blink">https://example.com"style='text-decoration:blink</a>’""",  # noqa
+        ),
+    ],
+)
+def test_URLs_get_escaped(url, expected_html, expected_html_in_template):
+    assert notify_email_markdown(url) == (
+        '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+        "{}"
+        "</p>"
+    ).format(expected_html)
+    assert expected_html_in_template in str(
+        HTMLEmailTemplate(
+            {
+                "content": url,
+                "subject": "",
+                "template_type": "email",
+            }
+        )
+    )
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected_output",
+    [
+        (
+            notify_email_markdown,
+            (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+                '<a style="word-wrap: break-word; color: #1D70B8;" href="https://example.com">'
+                "https://example.com"
+                "</a>"
+                "</p>"
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+                "Next paragraph"
+                "</p>"
+            ),
+        ),
+        (
+            notify_plain_text_email_markdown,
+            ("\n" "\nhttps://example.com" "\n" "\nNext paragraph"),
+        ),
+    ],
+)
+def test_preserves_whitespace_when_making_links(markdown_function, expected_output):
+    assert (
+        markdown_function("https://example.com\n" "\n" "Next paragraph")
+        == expected_output
+    )
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [notify_letter_preview_markdown, 'print("hello")'],
+        [notify_email_markdown, 'print("hello")'],
+        [notify_plain_text_email_markdown, 'print("hello")'],
+    ),
+)
+def test_block_code(markdown_function, expected):
+    assert markdown_function('```\nprint("hello")\n```') == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [notify_letter_preview_markdown, ("<p>inset text</p>")],
+        [
+            notify_email_markdown,
+            (
+                "<blockquote "
+                'style="Margin: 0 0 20px 0; border-left: 10px solid #B1B4B6;'
+                "padding: 15px 0 0.1px 15px; font-size: 19px; line-height: 25px;"
+                '">'
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">inset text</p>'
+                "</blockquote>"
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            ("\n" "\ninset text"),
+        ],
+    ),
+)
+def test_block_quote(markdown_function, expected):
+    assert markdown_function("^ inset text") == expected
+
+
+@pytest.mark.parametrize(
+    "heading",
+    (
+        "# heading",
+        "#heading",
+    ),
+)
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [notify_letter_preview_markdown, "<h2>heading</h2>\n"],
+        [
+            notify_email_markdown,
+            (
+                '<h2 style="Margin: 0 0 20px 0; padding: 0; font-size: 27px; '
+                'line-height: 35px; font-weight: bold; color: #0B0C0C;">'
+                "heading"
+                "</h2>"
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            (
+                "\n"
+                "\n"
+                "\nheading"
+                "\n-----------------------------------------------------------------"
+            ),
+        ],
+    ),
+)
+def test_level_1_header(markdown_function, heading, expected):
+    assert markdown_function(heading) == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [notify_letter_preview_markdown, "<p>inset text</p>"],
+        [
+            notify_email_markdown,
+            '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">inset text</p>',
+        ],
+        [
+            notify_plain_text_email_markdown,
+            ("\n" "\ninset text"),
+        ],
+    ),
+)
+def test_level_2_header(markdown_function, expected):
+    assert markdown_function("## inset text") == (expected)
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [
+            notify_letter_preview_markdown,
+            ("<p>a</p>" '<div class="page-break">&nbsp;</div>' "<p>b</p>"),
+        ],
+        [
+            notify_email_markdown,
+            (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">a</p>'
+                '<hr style="border: 0; height: 1px; background: #B1B4B6; Margin: 30px 0 30px 0;">'
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">b</p>'
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            (
+                "\n"
+                "\na"
+                "\n"
+                "\n================================================================="
+                "\n"
+                "\nb"
+            ),
+        ],
+    ),
+)
+def test_hrule(markdown_function, expected):
+    assert markdown_function("a\n\n***\n\nb") == expected
+    assert markdown_function("a\n\n---\n\nb") == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [
+            notify_letter_preview_markdown,
+            ("<ol>\n" "<li>one</li>\n" "<li>two</li>\n" "<li>three</li>\n" "</ol>\n"),
+        ],
+        [
+            notify_email_markdown,
+            (
+                '<table role="presentation" style="padding: 0 0 20px 0;">'
+                "<tr>"
+                '<td style="font-family: Helvetica, Arial, sans-serif;">'
+                '<ol style="Margin: 0 0 0 20px; padding: 0; list-style-type: decimal;">'
+                '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
+                'line-height: 25px; color: #0B0C0C;">one</li>'
+                '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
+                'line-height: 25px; color: #0B0C0C;">two</li>'
+                '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
+                'line-height: 25px; color: #0B0C0C;">three</li>'
+                "</ol>"
+                "</td>"
+                "</tr>"
+                "</table>"
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            ("\n" "\n1. one" "\n2. two" "\n3. three"),
+        ],
+    ),
+)
+def test_ordered_list(markdown_function, expected):
+    assert markdown_function("1. one\n" "2. two\n" "3. three\n") == expected
+    assert markdown_function("1.one\n" "2.two\n" "3.three\n") == expected
+
+
+@pytest.mark.parametrize(
+    "markdown",
+    (
+        ("*one\n" "*two\n" "*three\n"),  # no space
+        ("* one\n" "* two\n" "* three\n"),  # single space
+        ("*  one\n" "*  two\n" "*  three\n"),  # two spaces
+        ("*  one\n" "*  two\n" "*  three\n"),  # tab
+        ("- one\n" "- two\n" "- three\n"),  # dash as bullet
+        pytest.param(
+            ("+ one\n" "+ two\n" "+ three\n"),  # plus as bullet
+            marks=pytest.mark.xfail(raises=AssertionError),
+        ),
+        ("• one\n" "• two\n" "• three\n"),  # bullet as bullet
+    ),
+)
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [
+            notify_letter_preview_markdown,
+            ("<ul>\n" "<li>one</li>\n" "<li>two</li>\n" "<li>three</li>\n" "</ul>\n"),
+        ],
+        [
+            notify_email_markdown,
+            (
+                '<table role="presentation" style="padding: 0 0 20px 0;">'
+                "<tr>"
+                '<td style="font-family: Helvetica, Arial, sans-serif;">'
+                '<ul style="Margin: 0 0 0 20px; padding: 0; list-style-type: disc;">'
+                '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
+                'line-height: 25px; color: #0B0C0C;">one</li>'
+                '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
+                'line-height: 25px; color: #0B0C0C;">two</li>'
+                '<li style="Margin: 5px 0 5px; padding: 0 0 0 5px; font-size: 19px;'
+                'line-height: 25px; color: #0B0C0C;">three</li>'
+                "</ul>"
+                "</td>"
+                "</tr>"
+                "</table>"
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            ("\n" "\n• one" "\n• two" "\n• three"),
+        ],
+    ),
+)
+def test_unordered_list(markdown, markdown_function, expected):
+    assert markdown_function(markdown) == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [
+            notify_letter_preview_markdown,
+            "<p>+ one</p><p>+ two</p><p>+ three</p>",
+        ],
+        [
+            notify_email_markdown,
+            (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">+ one</p>'
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">+ two</p>'
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">+ three</p>'
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            ("\n\n+ one" "\n\n+ two" "\n\n+ three"),
+        ],
+    ),
+)
+def test_pluses_dont_render_as_lists(markdown_function, expected):
+    assert markdown_function("+ one\n" "+ two\n" "+ three\n") == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [
+            notify_letter_preview_markdown,
+            ("<p>" "line one<br>" "line two" "</p>" "<p>" "new paragraph" "</p>"),
+        ],
+        [
+            notify_email_markdown,
+            (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">line one<br />'
+                "line two</p>"
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">new paragraph</p>'
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            ("\n" "\nline one" "\nline two" "\n" "\nnew paragraph"),
+        ],
+    ),
+)
+def test_paragraphs(markdown_function, expected):
+    assert markdown_function("line one\n" "line two\n" "\n" "new paragraph") == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [notify_letter_preview_markdown, ("<p>before</p>" "<p>after</p>")],
+        [
+            notify_email_markdown,
+            (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">before</p>'
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">after</p>'
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            ("\n" "\nbefore" "\n" "\nafter"),
+        ],
+    ),
+)
+def test_multiple_newlines_get_truncated(markdown_function, expected):
+    assert markdown_function("before\n\n\n\n\n\nafter") == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function",
+    (
+        notify_letter_preview_markdown,
+        notify_email_markdown,
+        notify_plain_text_email_markdown,
+    ),
+)
+def test_table(markdown_function):
+    assert markdown_function("col | col\n" "----|----\n" "val | val\n") == ("")
+
+
+@pytest.mark.parametrize(
+    "markdown_function, link, expected",
+    (
+        [
+            notify_letter_preview_markdown,
+            "http://example.com",
+            "<p><strong>example.com</strong></p>",
+        ],
+        [
+            notify_email_markdown,
+            "http://example.com",
+            (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+                '<a style="word-wrap: break-word; color: #1D70B8;" href="http://example.com">http://example.com</a>'
+                "</p>"
+            ),
+        ],
+        [
+            notify_email_markdown,
+            """https://example.com"onclick="alert('hi')""",
+            (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">'
+                '<a style="word-wrap: break-word; color: #1D70B8;" '
+                'href="https://example.com%22onclick=%22alert%28%27hi">'
+                'https://example.com"onclick="alert(\'hi'
+                "</a>')"
+                "</p>"
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            "http://example.com",
+            ("\n" "\nhttp://example.com"),
+        ],
+    ),
+)
+def test_autolink(markdown_function, link, expected):
+    assert markdown_function(link) == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [notify_letter_preview_markdown, "<p>variable called `thing`</p>"],
+        [
+            notify_email_markdown,
+            '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">variable called `thing`</p>',  # noqa E501
+        ],
+        [
+            notify_plain_text_email_markdown,
+            "\n\nvariable called `thing`",
+        ],
+    ),
+)
+def test_codespan(markdown_function, expected):
+    assert markdown_function("variable called `thing`") == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [notify_letter_preview_markdown, "<p>something **important**</p>"],
+        [
+            notify_email_markdown,
+            '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">something **important**</p>',  # noqa E501
+        ],
+        [
+            notify_plain_text_email_markdown,
+            "\n\nsomething **important**",
+        ],
+    ),
+)
+def test_double_emphasis(markdown_function, expected):
+    assert markdown_function("something **important**") == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, text, expected",
+    (
+        [
+            notify_letter_preview_markdown,
+            "something *important*",
+            "<p>something *important*</p>",
+        ],
+        [
+            notify_email_markdown,
+            "something *important*",
+            '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">something *important*</p>',  # noqa E501
+        ],
+        [
+            notify_plain_text_email_markdown,
+            "something *important*",
+            "\n\nsomething *important*",
+        ],
+        [
+            notify_plain_text_email_markdown,
+            "something _important_",
+            "\n\nsomething _important_",
+        ],
+        [
+            notify_plain_text_email_markdown,
+            "before*after",
+            "\n\nbefore*after",
+        ],
+        [
+            notify_plain_text_email_markdown,
+            "before_after",
+            "\n\nbefore_after",
+        ],
+    ),
+)
+def test_emphasis(markdown_function, text, expected):
+    assert markdown_function(text) == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [
+            notify_email_markdown,
+            '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">foo ****** bar</p>',
+        ],
+        [
+            notify_plain_text_email_markdown,
+            "\n\nfoo ****** bar",
+        ],
+    ),
+)
+def test_nested_emphasis(markdown_function, expected):
+    assert markdown_function("foo ****** bar") == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function",
+    (
+        notify_letter_preview_markdown,
+        notify_email_markdown,
+        notify_plain_text_email_markdown,
+    ),
+)
+def test_image(markdown_function):
+    assert markdown_function("![alt text](http://example.com/image.png)") == ("")
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [
+            notify_letter_preview_markdown,
+            ("<p>Example: <strong>example.com</strong></p>"),
+        ],
+        [
+            notify_email_markdown,
+            (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; '
+                'color: #0B0C0C;">'
+                '<a style="word-wrap: break-word; color: #1D70B8;" href="http://example.com">Example</a>'
+                "</p>"
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            ("\n" "\nExample: http://example.com"),
+        ],
+    ),
+)
+def test_link(markdown_function, expected):
+    assert markdown_function("[Example](http://example.com)") == expected
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [
+            notify_letter_preview_markdown,
+            ("<p>Example: <strong>example.com</strong></p>"),
+        ],
+        [
+            notify_email_markdown,
+            (
+                '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; '
+                'color: #0B0C0C;">'
+                '<a style="word-wrap: break-word; color: #1D70B8;" href="http://example.com" title="An example URL">'
+                "Example"
+                "</a>"
+                "</p>"
+            ),
+        ],
+        [
+            notify_plain_text_email_markdown,
+            ("\n" "\nExample (An example URL): http://example.com"),
+        ],
+    ),
+)
+def test_link_with_title(markdown_function, expected):
+    assert (
+        markdown_function('[Example](http://example.com "An example URL")') == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "markdown_function, expected",
+    (
+        [notify_letter_preview_markdown, "<p>~~Strike~~</p>"],
+        [
+            notify_email_markdown,
+            '<p style="Margin: 0 0 20px 0; font-size: 19px; line-height: 25px; color: #0B0C0C;">~~Strike~~</p>',
+        ],
+        [notify_plain_text_email_markdown, "\n\n~~Strike~~"],
+    ),
+)
+def test_strikethrough(markdown_function, expected):
+    assert markdown_function("~~Strike~~") == expected
+
+
+def test_footnotes():
+    # Can’t work out how to test this
+    pass

tests/notifications_utils/test_placeholders.py

@@ -0,0 +1,66 @@
+import re
+
+import pytest
+
+from notifications_utils.field import Placeholder
+
+
+@pytest.mark.parametrize(
+    "body, expected",
+    [
+        ("((with-brackets))", "with-brackets"),
+        ("without-brackets", "without-brackets"),
+    ],
+)
+def test_placeholder_returns_name(body, expected):
+    assert Placeholder(body).name == expected
+
+
+@pytest.mark.parametrize(
+    "body, is_conditional",
+    [
+        ("not a conditional", False),
+        ("not? a conditional", False),
+        ("a?? conditional", True),
+    ],
+)
+def test_placeholder_identifies_conditional(body, is_conditional):
+    assert Placeholder(body).is_conditional() == is_conditional
+
+
+@pytest.mark.parametrize(
+    "body, conditional_text",
+    [
+        ("a??b", "b"),
+        ("a?? b ", " b "),
+        ("a??b??c", "b??c"),
+    ],
+)
+def test_placeholder_gets_conditional_text(body, conditional_text):
+    assert Placeholder(body).conditional_text == conditional_text
+
+
+def test_placeholder_raises_if_accessing_conditional_text_on_non_conditional():
+    with pytest.raises(ValueError):
+        Placeholder("hello").conditional_text
+
+
+@pytest.mark.parametrize(
+    "body, value, result",
+    [
+        ("a??b", "Yes", "b"),
+        ("a??b", "No", ""),
+    ],
+)
+def test_placeholder_gets_conditional_body(body, value, result):
+    assert Placeholder(body).get_conditional_body(value) == result
+
+
+def test_placeholder_raises_if_getting_conditional_body_on_non_conditional():
+    with pytest.raises(ValueError):
+        Placeholder("hello").get_conditional_body("Yes")
+
+
+def test_placeholder_can_be_constructed_from_regex_match():
+    match = re.search(r"\(\(.*\)\)", "foo ((bar)) baz")
+    assert Placeholder.from_match(match).name == "bar"

tests/notifications_utils/test_postal_address.py

@@ -0,0 +1,777 @@
+import pytest
+
+from notifications_utils.countries import Country
+from notifications_utils.countries.data import Postage
+from notifications_utils.insensitive_dict import InsensitiveDict
+from notifications_utils.postal_address import (
+    PostalAddress,
+    format_postcode_for_printing,
+    is_a_real_uk_postcode,
+    normalise_postcode,
+)
+
+
+def test_raw_address():
+    raw_address = "a\n\n\tb\r       c         "
+    assert PostalAddress(raw_address).raw_address == raw_address
+
+
+@pytest.mark.parametrize(
+    "address, expected_country",
+    (
+        (
+            """
+        123 Example Street
+        City of Town
+        SW1A 1AA
+        """,
+            Country("United Kingdom"),
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        SW1A 1AA
+        United Kingdom
+        """,
+            Country("United Kingdom"),
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        Wales
+        """,
+            Country("United Kingdom"),
+        ),
+        (
+            """
+        123 Example Straße
+        Deutschland
+        """,
+            Country("Germany"),
+        ),
+    ),
+)
+def test_country(address, expected_country):
+    assert PostalAddress(address).country == expected_country
+
+
+@pytest.mark.parametrize(
+    "address, enough_lines_expected",
+    (
+        (
+            "",
+            False,
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        SW1A 1AA
+        """,
+            True,
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        United Kingdom
+        """,
+            False,
+        ),
+        (
+            """
+        123 Example Street
+
+
+        City of Town
+        """,
+            False,
+        ),
+        (
+            """
+        1
+        2
+        3
+        4
+        5
+        6
+        7
+        8
+        """,
+            True,
+        ),
+    ),
+)
+def test_has_enough_lines(address, enough_lines_expected):
+    assert PostalAddress(address).has_enough_lines is enough_lines_expected
+
+
+@pytest.mark.parametrize(
+    "address, too_many_lines_expected",
+    (
+        (
+            "",
+            False,
+        ),
+        (
+            """
+        Line 1
+        Line 2
+        Line 3
+        Line 4
+        Line 5
+        Line 6
+        Line 7
+        """,
+            False,
+        ),
+        (
+            """
+        Line 1
+
+        Line 2
+
+        Line 3
+
+        Line 4
+
+        Line 5
+
+        Line 6
+
+        Line 7
+        """,
+            False,
+        ),
+        (
+            """
+        Line 1
+        Line 2
+        Line 3
+        Line 4
+        Line 5
+        Line 6
+        Line 7
+        Scotland
+        """,
+            False,
+        ),
+        (
+            """
+        Line 1
+        Line 2
+        Line 3
+        Line 4
+        Line 5
+        Line 6
+        Line 7
+        Line 8
+        """,
+            True,
+        ),
+    ),
+)
+def test_has_too_many_lines(address, too_many_lines_expected):
+    assert PostalAddress(address).has_too_many_lines is too_many_lines_expected
+
+
+@pytest.mark.parametrize(
+    "address, expected_postcode",
+    (
+        (
+            "",
+            None,
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        SW1A 1AA
+        """,
+            "SW1A 1AA",
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        S W1 A 1 AA
+        """,
+            "SW1A 1AA",
+        ),
+        (
+            """
+        123 Example Straße
+        Deutschland
+        """,
+            None,
+        ),
+        (
+            """
+        123 Example Straße
+        SW1A 1AA
+        Deutschland
+        """,
+            None,
+        ),
+    ),
+)
+def test_postcode(address, expected_postcode):
+    assert PostalAddress(address).has_valid_postcode is bool(expected_postcode)
+    assert PostalAddress(address).postcode == expected_postcode
+
+
+@pytest.mark.parametrize(
+    "address, expected_result",
+    [
+        (
+            "",
+            False,
+        ),
+        (
+            """
+        1[23 Example Street)
+        C@ity of Town
+        SW1A 1AA
+        """,
+            False,
+        ),
+        (
+            """
+        [123 Example Street
+        (ity of Town
+        ]S W1 A 1 AA
+        """,
+            True,
+        ),
+        (
+            r"""
+        123 Example Straße
+        SW1A 1AA
+        \Deutschland
+        """,
+            True,
+        ),
+        (
+            r"""
+        >123 Example Straße
+        SW1A 1AA
+        Deutschland
+        """,
+            True,
+        ),
+        (
+            """
+        ~123 Example Street
+        City of Town
+        SW1 A 1 AA
+        """,
+            True,
+        ),
+    ],
+)
+def test_has_invalid_characters(address, expected_result):
+    assert PostalAddress(address).has_invalid_characters is expected_result
+
+
+@pytest.mark.parametrize(
+    "address, expected_international",
+    (
+        (
+            "",
+            False,
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        SW1A 1AA
+        """,
+            False,
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        United Kingdom
+        """,
+            False,
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        Guernsey
+        """,
+            False,
+        ),
+        (
+            """
+        123 Example Straße
+        Deutschland
+        """,
+            True,
+        ),
+    ),
+)
+def test_international(address, expected_international):
+    assert PostalAddress(address).international is expected_international
+
+
+@pytest.mark.parametrize(
+    "address, expected_normalised, expected_as_single_line",
+    (
+        (
+            "",
+            "",
+            "",
+        ),
+        (
+            """
+        123 Example    St  .
+        City    of Town
+
+        S W1 A 1 AA
+        """,
+            ("123 Example St.\n" "City of Town\n" "SW1A 1AA"),
+            ("123 Example St., City of Town, SW1A 1AA"),
+        ),
+        (
+            (
+                "123 Example St. \t  ,    \n"
+                ", , ,  ,   ,     ,        ,\n"
+                "City of Town, Region,\n"
+                "SW1A 1AA,,\n"
+            ),
+            ("123 Example St.\n" "City of Town, Region\n" "SW1A 1AA"),
+            ("123 Example St., City of Town, Region, SW1A 1AA"),
+        ),
+        (
+            """
+          123  Example Straße
+        Deutschland
+
+
+        """,
+            ("123 Example Straße\n" "Germany"),
+            ("123 Example Straße, Germany"),
+        ),
+    ),
+)
+def test_normalised(address, expected_normalised, expected_as_single_line):
+    assert PostalAddress(address).normalised == expected_normalised
+    assert PostalAddress(address).as_single_line == expected_as_single_line
+
+
+@pytest.mark.parametrize(
+    "address, expected_postage",
+    (
+        (
+            "",
+            Postage.UK,
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        SW1A 1AA
+        """,
+            Postage.UK,
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        Scotland
+        """,
+            Postage.UK,
+        ),
+        (
+            """
+        123 Example Straße
+        Deutschland
+        """,
+            Postage.EUROPE,
+        ),
+        (
+            """
+        123 Rue Example
+        Côte d'Ivoire
+        """,
+            Postage.REST_OF_WORLD,
+        ),
+    ),
+)
+def test_postage(address, expected_postage):
+    assert PostalAddress(address).postage == expected_postage
+
+
+@pytest.mark.parametrize(
+    "personalisation",
+    (
+        {
+            "address_line_1": "123 Example Street",
+            "address_line_3": "City of Town",
+            "address_line_4": "",
+            "postcode": "SW1A1AA",
+            "ignore me": "ignore me",
+        },
+        {
+            "address_line_1": "123 Example Street",
+            "address_line_3": "City of Town",
+            "address_line_4": "SW1A1AA",
+        },
+        {
+            "address_line_2": "123 Example Street",
+            "address_line_5": "City of Town",
+            "address_line_7": "SW1A1AA",
+        },
+        {
+            "address_line_1": "123 Example Street",
+            "address_line_3": "City of Town",
+            "address_line_7": "SW1A1AA",
+            "postcode": "ignored if address line 7 provided",
+        },
+        InsensitiveDict(
+            {
+                "address line 1": "123 Example Street",
+                "ADDRESS_LINE_2": "City of Town",
+                "Address-Line-7": "Sw1a  1aa",
+            }
+        ),
+    ),
+)
+def test_from_personalisation(personalisation):
+    assert PostalAddress.from_personalisation(personalisation).normalised == (
+        "123 Example Street\n" "City of Town\n" "SW1A 1AA"
+    )
+
+
+def test_from_personalisation_handles_int():
+    personalisation = {
+        "address_line_1": 123,
+        "address_line_2": "Example Street",
+        "address_line_3": "City of Town",
+        "address_line_4": "SW1A1AA",
+    }
+    assert PostalAddress.from_personalisation(personalisation).normalised == (
+        "123\n" "Example Street\n" "City of Town\n" "SW1A 1AA"
+    )
+
+
+@pytest.mark.parametrize(
+    "address, expected_personalisation",
+    (
+        (
+            "",
+            {
+                "address_line_1": "",
+                "address_line_2": "",
+                "address_line_3": "",
+                "address_line_4": "",
+                "address_line_5": "",
+                "address_line_6": "",
+                "address_line_7": "",
+                "postcode": "",
+            },
+        ),
+        (
+            """
+        123 Example Street
+        City of Town
+        SW1A1AA
+        """,
+            {
+                "address_line_1": "123 Example Street",
+                "address_line_2": "City of Town",
+                "address_line_3": "",
+                "address_line_4": "",
+                "address_line_5": "",
+                "address_line_6": "",
+                "address_line_7": "SW1A 1AA",
+                "postcode": "SW1A 1AA",
+            },
+        ),
+        (
+            """
+        One
+        Two
+        Three
+        Four
+        Five
+        Six
+        Seven
+        Eight
+        """,
+            {
+                "address_line_1": "One",
+                "address_line_2": "Two",
+                "address_line_3": "Three",
+                "address_line_4": "Four",
+                "address_line_5": "Five",
+                "address_line_6": "Six",
+                "address_line_7": "Eight",
+                "postcode": "Eight",
+            },
+        ),
+    ),
+)
+def test_as_personalisation(address, expected_personalisation):
+    assert PostalAddress(address).as_personalisation == expected_personalisation
+
+
+@pytest.mark.parametrize(
+    "address, expected_bool",
+    (
+        ("", False),
+        (" ", False),
+        ("\n\n  \n", False),
+        ("a", True),
+    ),
+)
+def test_bool(address, expected_bool):
+    assert bool(PostalAddress(address)) is expected_bool
+
+
+@pytest.mark.parametrize(
+    "postcode, normalised_postcode",
+    [
+        ("SW1 3EF", "SW13EF"),
+        ("SW13EF", "SW13EF"),
+        ("sw13ef", "SW13EF"),
+        ("Sw13ef", "SW13EF"),
+        ("sw1 3ef", "SW13EF"),
+        (" SW1    3EF  ", "SW13EF"),
+    ],
+)
+def test_normalise_postcode(postcode, normalised_postcode):
+    assert normalise_postcode(postcode) == normalised_postcode
+
+
+@pytest.mark.parametrize(
+    "postcode, result",
+    [
+        # real standard UK poscodes
+        ("SW1 3EF", True),
+        ("SW13EF", True),
+        ("SE1 63EF", True),
+        ("N5 1AA", True),
+        ("SO14 6WB", True),
+        ("so14 6wb", True),
+        ("so14\u00A06wb", True),
+        # invalida / incomplete postcodes
+        ("N5", False),
+        ("SO144 6WB", False),
+        ("SO14 6WBA", False),
+        ("", False),
+        ("Bad postcode", False),
+        # valid British Forces postcodes
+        ("BFPO1234", True),
+        ("BFPO C/O 1234", True),
+        ("BFPO 1234", True),
+        ("BFPO1", True),
+        # invalid British Forces postcodes
+        ("BFPO", False),
+        ("BFPO12345", False),
+        # Giro Bank valid postcode and invalid postcode
+        ("GIR0AA", True),
+        ("GIR0AB", False),
+    ],
+)
+def test_if_postcode_is_a_real_uk_postcode(postcode, result):
+    assert is_a_real_uk_postcode(postcode) is result
+
+
+def test_if_postcode_is_a_real_uk_postcode_normalises_before_checking_postcode(mocker):
+    normalise_postcode_mock = mocker.patch(
+        "notifications_utils.postal_address.normalise_postcode"
+    )
+    normalise_postcode_mock.return_value = "SW11AA"
+    assert is_a_real_uk_postcode("sw1  1aa") is True
+
+
+@pytest.mark.parametrize(
+    "postcode, postcode_with_space",
+    [
+        ("SW13EF", "SW1 3EF"),
+        ("SW1 3EF", "SW1 3EF"),
+        ("N5 3EF", "N5 3EF"),
+        ("N5     3EF", "N5 3EF"),
+        ("N53EF   ", "N5 3EF"),
+        ("n53Ef", "N5 3EF"),
+        ("n5 \u00A0 \t 3Ef", "N5 3EF"),
+        ("SO146WB", "SO14 6WB"),
+        ("BFPO2", "BFPO 2"),
+        ("BFPO232", "BFPO 232"),
+        ("BFPO 2432", "BFPO 2432"),
+        ("BFPO C/O 2", "BFPO C/O 2"),
+        ("BFPO c/o 232", "BFPO C/O 232"),
+        ("GIR0AA", "GIR 0AA"),
+    ],
+)
+def test_format_postcode_for_printing(postcode, postcode_with_space):
+    assert format_postcode_for_printing(postcode) == postcode_with_space
+
+
+@pytest.mark.parametrize(
+    "address, international, expected_valid",
+    (
+        (
+            """
+            UK address
+            Service can’t send internationally
+            SW1A 1AA
+        """,
+            False,
+            True,
+        ),
+        (
+            """
+            UK address
+            Service can send internationally
+            SW1A 1AA
+        """,
+            True,
+            True,
+        ),
+        (
+            """
+            Overseas address
+            Service can’t send internationally
+            Guinea-Bissau
+        """,
+            False,
+            False,
+        ),
+        (
+            """
+            Overseas address
+            Service can send internationally
+            Guinea-Bissau
+        """,
+            True,
+            True,
+        ),
+        (
+            """
+            Overly long address
+            2
+            3
+            4
+            5
+            6
+            7
+            8
+        """,
+            True,
+            False,
+        ),
+        (
+            """
+            Address too short
+            2
+        """,
+            True,
+            False,
+        ),
+        (
+            """
+            No postcode or country
+            Service can’t send internationally
+            3
+        """,
+            False,
+            False,
+        ),
+        (
+            """
+            No postcode or country
+            Service can send internationally
+            3
+        """,
+            True,
+            False,
+        ),
+        (
+            """
+            Postcode and country
+            Service can’t send internationally
+            SW1 1AA
+            France
+        """,
+            False,
+            False,
+        ),
+    ),
+)
+def test_valid_with_international_parameter(address, international, expected_valid):
+    postal_address = PostalAddress(
+        address,
+        allow_international_letters=international,
+    )
+    assert postal_address.valid is expected_valid
+    assert postal_address.has_valid_last_line is expected_valid
+
+
+@pytest.mark.parametrize(
+    "address",
+    (
+        """
+        Too short, valid postcode
+        SW1A 1AA
+    """,
+        """
+        Too short, valid country
+        Bhutan
+    """,
+        """
+        Too long, valid postcode
+        2
+        3
+        4
+        5
+        6
+        7
+        SW1A 1AA
+    """,
+        """
+        Too long, valid country
+        2
+        3
+        4
+        5
+        6
+        7
+        Bhutan
+    """,
+    ),
+)
+def test_valid_last_line_too_short_too_long(address):
+    postal_address = PostalAddress(address, allow_international_letters=True)
+    assert postal_address.valid is False
+    assert postal_address.has_valid_last_line is True
+
+
+def test_valid_with_invalid_characters():
+    address = "Valid\nExcept\n[For one character\nBhutan\n"
+    assert PostalAddress(address, allow_international_letters=True).valid is False
+
+
+@pytest.mark.parametrize(
+    "international, expected_valid",
+    (
+        (False, False),
+        (True, True),
+    ),
+)
+def test_valid_from_personalisation_with_international_parameter(
+    international, expected_valid
+):
+    assert (
+        PostalAddress.from_personalisation(
+            {"address_line_1": "A", "address_line_2": "B", "address_line_3": "Chad"},
+            allow_international_letters=international,
+        ).valid
+        is expected_valid
+    )

tests/notifications_utils/test_recipient_validation.py

@@ -0,0 +1,428 @@
+import pytest
+
+from notifications_utils.recipients import (
+    InvalidEmailError,
+    InvalidPhoneError,
+    allowed_to_send_to,
+    format_phone_number_human_readable,
+    format_recipient,
+    get_international_phone_info,
+    international_phone_info,
+    is_us_phone_number,
+    try_validate_and_format_phone_number,
+    validate_and_format_phone_number,
+    validate_email_address,
+    validate_phone_number,
+)
+
+valid_us_phone_numbers = [
+    "1-202-555-0104",
+    "+12025550104",
+    "12025550104",
+    "2025550104",
+    "(202) 555-0104",
+]
+
+# TODO
+# International phone number tests are commented out as a result of issue #943 in notifications-admin.  We are
+# deliberately eliminating the ability to send to numbers outside of country code 1.   These tests should
+# be removed at some point when we are sure we are never going to support international numbers
+
+valid_international_phone_numbers = [
+    # "+71234567890",  # Russia
+    # "+447123456789",  # UK
+    # "+4407123456789",  # UK
+    # "+4407123 456789",  # UK
+    # "+4407123-456-789",  # UK
+    # "+23051234567",  # Mauritius,
+    # "+682 12345",  # Cook islands
+    # "+3312345678",
+    # "+9-2345-12345-12345",  # 15 digits
+]
+
+
+valid_phone_numbers = valid_us_phone_numbers + valid_international_phone_numbers
+
+
+invalid_us_phone_numbers = sum(
+    [
+        [(phone_number, error) for phone_number in group]
+        for error, group in [
+            (
+                "Too many digits",
+                (
+                    "55512345678",
+                    "+155512345678",
+                    "(555) 1234-5678",
+                ),
+            ),
+            (
+                "Not enough digits",
+                (
+                    "555123123",
+                    "(555) 123-123",
+                    "7890x32109",
+                    "07123 ☟☜⬇⬆☞☝",
+                    "07123☟☜⬇⬆☞☝",
+                ),
+            ),
+            ("Phone number range is not in use", ("1555123123",)),
+            ("Phone number is not possible", ("07123 456789...",)),
+            (
+                "The string supplied did not seem to be a phone number.",
+                (
+                    '07";DROP TABLE;"',
+                    "ALPHANUM3R1C",
+                ),
+            ),
+        ]
+    ],
+    [],
+)
+
+
+invalid_phone_numbers = [
+    ("+80233456789", "Not a valid country prefix"),
+    ("1234567", "Not enough digits"),
+    ("+682 1234", "Invalid country code"),  # Cook Islands phone numbers can be 5 digits
+    ("+12345 12345 12345 6", "Too many digits"),
+]
+
+
+valid_email_addresses = (
+    "email@domain.com",
+    "email@domain.COM",
+    "firstname.lastname@domain.com",
+    "firstname.o'lastname@domain.com",
+    "email@subdomain.domain.com",
+    "firstname+lastname@domain.com",
+    "1234567890@domain.com",
+    "email@domain-one.com",
+    "_______@domain.com",
+    "email@domain.name",
+    "email@domain.superlongtld",
+    "email@domain.co.jp",
+    "firstname-lastname@domain.com",
+    "info@german-financial-services.vermögensberatung",
+    "info@german-financial-services.reallylongarbitrarytldthatiswaytoohugejustincase",
+    "japanese-info@例え.テスト",
+    "email@double--hyphen.com",
+)
+invalid_email_addresses = (
+    "email@123.123.123.123",
+    "email@[123.123.123.123]",
+    "plainaddress",
+    "@no-local-part.com",
+    "Outlook Contact <outlook-contact@domain.com>",
+    "no-at.domain.com",
+    "no-tld@domain",
+    ";beginning-semicolon@domain.co.uk",
+    "middle-semicolon@domain.co;uk",
+    "trailing-semicolon@domain.com;",
+    '"email+leading-quotes@domain.com',
+    'email+middle"-quotes@domain.com',
+    '"quoted-local-part"@domain.com',
+    '"quoted@domain.com"',
+    "lots-of-dots@domain..gov..uk",
+    "two-dots..in-local@domain.com",
+    "multiple@domains@domain.com",
+    "spaces in local@domain.com",
+    "spaces-in-domain@dom ain.com",
+    "underscores-in-domain@dom_ain.com",
+    "pipe-in-domain@example.com|gov.uk",
+    "comma,in-local@gov.uk",
+    "comma-in-domain@domain,gov.uk",
+    "pound-sign-in-local£@domain.com",
+    "local-with-’-apostrophe@domain.com",
+    "local-with-”-quotes@domain.com",
+    "domain-starts-with-a-dot@.domain.com",
+    "brackets(in)local@domain.com",
+    "email-too-long-{}@example.com".format("a" * 320),
+    "incorrect-punycode@xn---something.com",
+)
+
+
+@pytest.mark.parametrize("phone_number", valid_international_phone_numbers)
+def test_detect_international_phone_numbers(phone_number):
+    assert is_us_phone_number(phone_number) is False
+
+
+@pytest.mark.parametrize("phone_number", valid_us_phone_numbers)
+def test_detect_us_phone_numbers(phone_number):
+    assert is_us_phone_number(phone_number) is True
+
+
+@pytest.mark.parametrize(
+    "phone_number, expected_info",
+    [
+        # (
+        #    "+4407900900123",
+        #    international_phone_info(
+        #        international=True,
+        #        country_prefix="44",  # UK
+        #        billable_units=1,
+        #    ),
+        # ),
+        # (
+        #    "+4407700900123",
+        #    international_phone_info(
+        #        international=True,
+        #        country_prefix="44",  # Number in TV range
+        #        billable_units=1,
+        #    ),
+        # ),
+        # (
+        #    "+4407700800123",
+        #    international_phone_info(
+        #        international=True,
+        #        country_prefix="44",  # UK Crown dependency, so prefix same as UK
+        #        billable_units=1,
+        #    ),
+        # ),
+        # ( #
+        #    "+20-12-1234-1234",
+        #    international_phone_info(
+        #        international=True,
+        #        country_prefix="20",  # Egypt
+        #        billable_units=1,
+        #    ),
+        # ),
+        # (
+        #    "+201212341234",
+        #    international_phone_info(
+        #        international=True,
+        #        country_prefix="20",  # Egypt
+        #        billable_units=1,
+        #    ),
+        # ),
+        (
+            "+1 664-491-3434",
+            international_phone_info(
+                international=True,
+                country_prefix="1664",  # Montserrat
+                billable_units=1,
+            ),
+        ),
+        # (
+        #    "+71234567890",
+        #    international_phone_info(
+        #        international=True,
+        #        country_prefix="7",  # Russia
+        #        billable_units=1,
+        #    ),
+        # ),
+        (
+            "1-202-555-0104",
+            international_phone_info(
+                international=False,
+                country_prefix="1",  # USA
+                billable_units=1,
+            ),
+        ),
+        (
+            "202-555-0104",
+            international_phone_info(
+                international=False,
+                country_prefix="1",  # USA
+                billable_units=1,
+            ),
+        ),
+        # (
+        #    "+23051234567",
+        #    international_phone_info(
+        #        international=True,
+        #        country_prefix="230",  # Mauritius
+        #        billable_units=1,
+        #    ),
+        # ),
+    ],
+)
+def test_get_international_info(phone_number, expected_info):
+    assert get_international_phone_info(phone_number) == expected_info
+
+
+@pytest.mark.parametrize(
+    "phone_number",
+    [
+        "+21 4321 0987",
+        "+00997 1234 7890",
+        "+801234-7890",
+        "+8-0-1234-78901",
+    ],
+)
+def test_get_international_info_raises(phone_number):
+    with pytest.raises(InvalidPhoneError) as error:
+        get_international_phone_info(phone_number)
+    assert str(error.value) == "Not a valid country prefix"
+
+
+@pytest.mark.parametrize("phone_number", valid_us_phone_numbers)
+@pytest.mark.parametrize(
+    "extra_args",
+    [
+        {},
+        {"international": False},
+    ],
+)
+def test_phone_number_accepts_valid_values(extra_args, phone_number):
+    try:
+        validate_phone_number(phone_number, **extra_args)
+    except InvalidPhoneError:
+        pytest.fail("Unexpected InvalidPhoneError")
+
+
+@pytest.mark.parametrize("phone_number", valid_phone_numbers)
+def test_phone_number_accepts_valid_international_values(phone_number):
+    try:
+        validate_phone_number(phone_number, international=True)
+    except InvalidPhoneError:
+        pytest.fail("Unexpected InvalidPhoneError")
+
+
+@pytest.mark.parametrize("phone_number", valid_us_phone_numbers)
+def test_valid_us_phone_number_can_be_formatted_consistently(phone_number):
+    assert validate_and_format_phone_number(phone_number) == "+12025550104"
+
+
+@pytest.mark.parametrize(
+    "phone_number, expected_formatted",
+    [
+        # ("+44071234567890", "+4471234567890"),
+        ("1-202-555-0104", "+12025550104"),
+        ("+12025550104", "+12025550104"),
+        ("12025550104", "+12025550104"),
+        ("+12025550104", "+12025550104"),
+        # ("+23051234567", "+23051234567"),
+    ],
+)
+def test_valid_international_phone_number_can_be_formatted_consistently(
+    phone_number, expected_formatted
+):
+    assert (
+        validate_and_format_phone_number(phone_number, international=True)
+        == expected_formatted
+    )
+
+
+@pytest.mark.parametrize("phone_number, error_message", invalid_us_phone_numbers)
+@pytest.mark.parametrize(
+    "extra_args",
+    [
+        {},
+        {"international": False},
+    ],
+)
+def test_phone_number_rejects_invalid_values(extra_args, phone_number, error_message):
+    with pytest.raises(InvalidPhoneError) as e:
+        validate_phone_number(phone_number, **extra_args)
+    assert error_message == str(e.value)
+
+
+@pytest.mark.parametrize("phone_number, error_message", invalid_phone_numbers)
+def test_phone_number_rejects_invalid_international_values(phone_number, error_message):
+    with pytest.raises(InvalidPhoneError) as e:
+        validate_phone_number(phone_number, international=True)
+    assert error_message == str(e.value)
+
+
+@pytest.mark.parametrize("email_address", valid_email_addresses)
+def test_validate_email_address_accepts_valid(email_address):
+    try:
+        assert validate_email_address(email_address) == email_address
+    except InvalidEmailError:
+        pytest.fail("Unexpected InvalidEmailError")
+
+
+@pytest.mark.parametrize(
+    "email",
+    [
+        " email@domain.com ",
+        "\temail@domain.com",
+        "\temail@domain.com\n",
+        "\u200Bemail@domain.com\u200B",
+    ],
+)
+def test_validate_email_address_strips_whitespace(email):
+    assert validate_email_address(email) == "email@domain.com"
+
+
+@pytest.mark.parametrize("email_address", invalid_email_addresses)
+def test_validate_email_address_raises_for_invalid(email_address):
+    with pytest.raises(InvalidEmailError) as e:
+        validate_email_address(email_address)
+    assert str(e.value) == "Not a valid email address"
+
+
+@pytest.mark.parametrize("phone_number", valid_us_phone_numbers)
+def test_validates_against_guestlist_of_phone_numbers(phone_number):
+    assert allowed_to_send_to(
+        phone_number, ["2025550104", "2025550105", "test@example.com"]
+    )
+    assert not allowed_to_send_to(
+        phone_number, ["2025550105", "2028675309", "test@example.com"]
+    )
+
+
+# @pytest.mark.parametrize(
+#    "recipient_number, allowlist_number",
+#    [
+#        ["+4407123-456-789", "+4407123456789"],
+#        ["+4407123456789", "+4407123-456-789"],
+#    ],
+# )
+# def test_validates_against_guestlist_of_international_phone_numbers(
+#    recipient_number, allowlist_number
+# ):
+#    assert allowed_to_send_to(recipient_number, [allowlist_number])
+
+
+@pytest.mark.parametrize("email_address", valid_email_addresses)
+def test_validates_against_guestlist_of_email_addresses(email_address):
+    assert not allowed_to_send_to(
+        email_address, ["very_special_and_unique@example.com"]
+    )
+
+
+@pytest.mark.parametrize(
+    "phone_number, expected_formatted",
+    [
+        # ("+4407900900123", "+44 7900 900123"),  # UK
+        # ("+44(0)7900900123", "+44 7900 900123"),  # UK
+        # ("+447900900123", "+44 7900 900123"),  # UK
+        # ("+20-12-1234-1234", "+20 121 234 1234"),  # Egypt
+        # ("+201212341234", "+20 121 234 1234"),  # Egypt
+        ("+1 664 491-3434", "+1 664-491-3434"),  # Montserrat
+        # ("+7 499 1231212", "+7 499 123-12-12"),  # Moscow (Russia)
+        ("1-202-555-0104", "(202) 555-0104"),  # Washington DC (USA)
+        # ("+23051234567", "+230 5123 4567"),  # Mauritius
+        # ("+33(0)1 12345678", "+33 1 12 34 56 78"),  # Paris (France)
+    ],
+)
+def test_format_us_and_international_phone_numbers(phone_number, expected_formatted):
+    assert format_phone_number_human_readable(phone_number) == expected_formatted
+
+
+@pytest.mark.parametrize(
+    "recipient, expected_formatted",
+    [
+        (True, ""),
+        (False, ""),
+        (0, ""),
+        (0.1, ""),
+        (None, ""),
+        ("foo", "foo"),
+        ("TeSt@ExAmPl3.com", "test@exampl3.com"),
+        # ("+4407900 900 123", "+447900900123"),
+        ("+1 800 555 5555", "+18005555555"),
+    ],
+)
+def test_format_recipient(recipient, expected_formatted):
+    assert format_recipient(recipient) == expected_formatted
+
+
+def test_try_format_recipient_doesnt_throw():
+    assert try_validate_and_format_phone_number("ALPHANUM3R1C") == "ALPHANUM3R1C"
+
+
+def test_format_phone_number_human_readable_doenst_throw():
+    assert format_phone_number_human_readable("ALPHANUM3R1C") == "ALPHANUM3R1C"

tests/notifications_utils/test_request_header_authentication.py

@@ -0,0 +1,61 @@
+import pytest
+from werkzeug.test import EnvironBuilder
+
+from notifications_utils.request_helper import NotifyRequest, _check_proxy_header_secret
+
+
+@pytest.mark.parametrize(
+    "header,secrets,expected",
+    [
+        (
+            {"X-Custom-Forwarder": "right_key"},
+            ["right_key", "old_key"],
+            (True, "Key used: 1"),
+        ),
+        ({"X-Custom-Forwarder": "right_key"}, ["right_key"], (True, "Key used: 1")),
+        ({"X-Custom-Forwarder": "right_key"}, ["right_key", ""], (True, "Key used: 1")),
+        ({"My-New-Header": "right_key"}, ["right_key", ""], (True, "Key used: 1")),
+        ({"X-Custom-Forwarder": "right_key"}, ["", "right_key"], (True, "Key used: 2")),
+        (
+            {"X-Custom-Forwarder": "right_key"},
+            ["", "old_key", "right_key"],
+            (True, "Key used: 3"),
+        ),
+        (
+            {"X-Custom-Forwarder": ""},
+            ["right_key", "old_key"],
+            (False, "Header exists but is empty"),
+        ),
+        (
+            {"X-Custom-Forwarder": "right_key"},
+            ["", None],
+            (False, "Secrets are not configured"),
+        ),
+        (
+            {"X-Custom-Forwarder": "wrong_key"},
+            ["right_key", "old_key"],
+            (False, "Header didn't match any keys"),
+        ),
+    ],
+)
+def test_request_header_authorization(header, secrets, expected):
+    builder = EnvironBuilder()
+    builder.headers.extend(header)
+    request = NotifyRequest(builder.get_environ())
+
+    res = _check_proxy_header_secret(request, secrets, list(header.keys())[0])
+    assert res == expected
+
+
+@pytest.mark.parametrize(
+    "secrets,expected",
+    [
+        (["old_key", "right_key"], (False, "Header missing")),
+    ],
+)
+def test_request_header_authorization_missing_header(secrets, expected):
+    builder = EnvironBuilder()
+    request = NotifyRequest(builder.get_environ())
+
+    res = _check_proxy_header_secret(request, secrets)
+    assert res == expected

tests/notifications_utils/test_request_id.py

@@ -0,0 +1,32 @@
+from notifications_utils import request_helper
+
+
+def test_request_id_is_set_on_response(app):
+    request_helper.init_app(app)
+    client = app.test_client()
+
+    with app.app_context():
+        response = client.get(
+            "/", headers={"X-B3-TraceId": "generated", "X-B3-SpanId": "generated"}
+        )
+        assert response.headers["X-B3-TraceId"] == "generated"
+        assert response.headers["X-B3-SpanId"] == "generated"
+
+
+def test_request_id_is_set_on_error_response(app):
+    request_helper.init_app(app)
+    client = app.test_client()
+    # turn off DEBUG so that the flask default error handler gets triggered
+    app.config["DEBUG"] = False
+
+    @app.route("/")
+    def error_route():
+        raise Exception()
+
+    with app.app_context():
+        response = client.get(
+            "/", headers={"X-B3-TraceId": "generated", "X-B3-SpanId": "generated"}
+        )
+        assert response.status_code == 500
+        assert response.headers["X-B3-TraceId"] == "generated"
+        assert response.headers["X-B3-SpanId"] == "generated"

tests/notifications_utils/test_s3.py

@@ -0,0 +1,108 @@
+from urllib.parse import parse_qs
+
+import botocore
+import pytest
+
+from notifications_utils.s3 import S3ObjectNotFound, s3download, s3upload
+
+contents = "some file data"
+region = "eu-west-1"
+bucket = "some_bucket"
+location = "some_file_location"
+content_type = "binary/octet-stream"
+
+
+def test_s3upload_save_file_to_bucket(mocker):
+    mocked = mocker.patch("notifications_utils.s3.Session.resource")
+    s3upload(
+        filedata=contents, region=region, bucket_name=bucket, file_location=location
+    )
+    mocked_put = mocked.return_value.Object.return_value.put
+    mocked_put.assert_called_once_with(
+        Body=contents,
+        ServerSideEncryption="AES256",
+        ContentType=content_type,
+    )
+
+
+def test_s3upload_save_file_to_bucket_with_contenttype(mocker):
+    content_type = "image/png"
+    mocked = mocker.patch("notifications_utils.s3.Session.resource")
+    s3upload(
+        filedata=contents,
+        region=region,
+        bucket_name=bucket,
+        file_location=location,
+        content_type=content_type,
+    )
+    mocked_put = mocked.return_value.Object.return_value.put
+    mocked_put.assert_called_once_with(
+        Body=contents,
+        ServerSideEncryption="AES256",
+        ContentType=content_type,
+    )
+
+
+def test_s3upload_raises_exception(app, mocker):
+    mocked = mocker.patch("notifications_utils.s3.Session.resource")
+    response = {"Error": {"Code": 500}}
+    exception = botocore.exceptions.ClientError(response, "Bad exception")
+    mocked.return_value.Object.return_value.put.side_effect = exception
+    with pytest.raises(botocore.exceptions.ClientError):
+        s3upload(
+            filedata=contents,
+            region=region,
+            bucket_name=bucket,
+            file_location="location",
+        )
+
+
+def test_s3upload_save_file_to_bucket_with_urlencoded_tags(mocker):
+    mocked = mocker.patch("notifications_utils.s3.Session.resource")
+    s3upload(
+        filedata=contents,
+        region=region,
+        bucket_name=bucket,
+        file_location=location,
+        tags={"a": "1/2", "b": "x y"},
+    )
+    mocked_put = mocked.return_value.Object.return_value.put
+
+    # make sure tags were a urlencoded query string
+    encoded_tags = mocked_put.call_args[1]["Tagging"]
+    assert parse_qs(encoded_tags) == {"a": ["1/2"], "b": ["x y"]}
+
+
+def test_s3upload_save_file_to_bucket_with_metadata(mocker):
+    mocked = mocker.patch("notifications_utils.s3.Session.resource")
+    s3upload(
+        filedata=contents,
+        region=region,
+        bucket_name=bucket,
+        file_location=location,
+        metadata={"status": "valid", "pages": "5"},
+    )
+    mocked_put = mocked.return_value.Object.return_value.put
+
+    metadata = mocked_put.call_args[1]["Metadata"]
+    assert metadata == {"status": "valid", "pages": "5"}
+
+
+def test_s3download_gets_file(mocker):
+    mocked = mocker.patch("notifications_utils.s3.Session.resource")
+    mocked_object = mocked.return_value.Object
+    mocked_get = mocked.return_value.Object.return_value.get
+    s3download("bucket", "location.file")
+    mocked_object.assert_called_once_with("bucket", "location.file")
+    mocked_get.assert_called_once_with()
+
+
+def test_s3download_raises_on_error(mocker):
+    mocked = mocker.patch("notifications_utils.s3.Session.resource")
+    mocked.return_value.Object.side_effect = botocore.exceptions.ClientError(
+        {"Error": {"Code": 404}},
+        "Bad exception",
+    )
+
+    with pytest.raises(S3ObjectNotFound):
+        s3download("bucket", "location.file")

tests/notifications_utils/test_safe_string.py

@@ -0,0 +1,47 @@
+import pytest
+
+from notifications_utils.safe_string import (
+    make_string_safe_for_email_local_part,
+    make_string_safe_for_id,
+)
+
+
+@pytest.mark.parametrize(
+    "unsafe_string, expected_safe",
+    [
+        ("name with spaces", "name.with.spaces"),
+        ("singleword", "singleword"),
+        ("UPPER CASE", "upper.case"),
+        ("Service - with dash", "service.with.dash"),
+        ("lots      of spaces", "lots.of.spaces"),
+        ("name.with.dots", "name.with.dots"),
+        ("name-with-other-delimiters", "namewithotherdelimiters"),
+        (".leading", "leading"),
+        ("trailing.", "trailing"),
+        ("üńïçödë wördś", "unicode.words"),
+    ],
+)
+def test_email_safe_return_dot_separated_email_local_part(unsafe_string, expected_safe):
+    assert make_string_safe_for_email_local_part(unsafe_string) == expected_safe
+
+
+@pytest.mark.parametrize(
+    "unsafe_string, expected_safe",
+    [
+        ("name with spaces", "name-with-spaces"),
+        ("singleword", "singleword"),
+        ("UPPER CASE", "upper-case"),
+        ("Service - with dash", "service---with-dash"),
+        ("lots      of spaces", "lots-of-spaces"),
+        ("name.with.dots", "namewithdots"),
+        ("name-with-dashes", "name-with-dashes"),
+        ("N. London", "n-london"),
+        (".leading", "leading"),
+        ("-leading", "-leading"),
+        ("trailing.", "trailing"),
+        ("trailing-", "trailing-"),
+        ("üńïçödë wördś", "unicode-words"),
+    ],
+)
+def test_id_safe_return_dash_separated_string(unsafe_string, expected_safe):
+    assert make_string_safe_for_id(unsafe_string) == expected_safe

tests/notifications_utils/test_sanitise_text.py

@@ -0,0 +1,313 @@
+import pytest
+
+from notifications_utils.sanitise_text import SanitiseASCII, SanitiseSMS, SanitiseText
+
+params, ids = zip(
+    (("a", "a"), "ascii char (a)"),
+    # ascii control char (not in GSM)
+    (("\t", " "), "ascii control char not in gsm (tab)"),
+    # TODO we support lots of languages now not in the GSM charset so maybe make this 'downgrading' go away
+    # TODO for now comment out this line because it directly conflicts with support for Turkish
+    # these are not in GSM charset so are downgraded
+    # (("ç", "c"), "decomposed unicode char (C with cedilla)"),
+    # these unicode chars should change to something completely different for compatibility
+    # (("–", "-"), "compatibility transform unicode char (EN DASH (U+2013)"),
+    # (("—", "-"), "compatibility transform unicode char (EM DASH (U+2014)"),
+    (
+        ("…", "..."),
+        "compatibility transform unicode char (HORIZONTAL ELLIPSIS (U+2026)",
+    ),
+    (("\u200B", ""), "compatibility transform unicode char (ZERO WIDTH SPACE (U+200B)"),
+    (
+        ("‘", "'"),
+        "compatibility transform unicode char (LEFT SINGLE QUOTATION MARK (U+2018)",
+    ),
+    (
+        ("’", "'"),
+        "compatibility transform unicode char (RIGHT SINGLE QUOTATION MARK (U+2019)",
+    ),
+    # Conflict with Chinese quotes
+    # (
+    #    ("“", '"'),
+    #    "compatibility transform unicode char (LEFT DOUBLE QUOTATION MARK (U+201C)	",
+    # ),
+    # (
+    #    ("”", '"'),
+    #    "compatibility transform unicode char (RIGHT DOUBLE QUOTATION MARK (U+201D)",
+    # ),
+    (("\xa0", " "), "nobreak transform unicode char (NO-BREAK SPACE (U+00A0))"),
+    # this unicode char is not decomposable
+    (("😬", "?"), "undecomposable unicode char (grimace emoji)"),
+    (("↉", "?"), "vulgar fraction (↉) that we do not try decomposing"),
+)
+
+
+@pytest.mark.parametrize("char, expected", params, ids=ids)
+@pytest.mark.parametrize("cls", [SanitiseSMS, SanitiseASCII])
+def test_encode_chars_the_same_for_ascii_and_sms(char, expected, cls):
+    assert cls.encode_char(char) == expected
+
+
+params, ids = zip(
+    # ascii control chars are allowed in GSM but not in ASCII
+    (("\n", "\n", "?"), "ascii control char in gsm (newline)"),
+    (("\r", "\r", "?"), "ascii control char in gsm (return)"),
+    # These characters are present in GSM but not in ascii
+    (("à", "à", "a"), "non-ascii gsm char (a with accent)"),
+    (("€", "€", "?"), "non-ascii gsm char (euro)"),
+    # These characters are Welsh characters that are not present in GSM
+    (("â", "â", "a"), "non-gsm Welsh char (a with hat)"),
+    (("Ŷ", "Ŷ", "Y"), "non-gsm Welsh char (capital y with hat)"),
+    (("ë", "ë", "e"), "non-gsm Welsh char (e with dots)"),
+    # (("Ò", "Ò", "O"), "non-gsm Welsh char (capital O with grave accent)"),  # conflicts with Vietnamese
+    (("í", "í", "i"), "non-gsm Welsh char (i with accent)"),
+)
+
+
+@pytest.mark.parametrize("char, expected_sms, expected_ascii", params, ids=ids)
+def test_encode_chars_different_between_ascii_and_sms(
+    char, expected_sms, expected_ascii
+):
+    assert SanitiseSMS.encode_char(char) == expected_sms
+    assert SanitiseASCII.encode_char(char) == expected_ascii
+
+
+@pytest.mark.parametrize(
+    "codepoint, char",
+    [
+        ("0041", "A"),
+        ("0061", "a"),
+    ],
+)
+def test_get_unicode_char_from_codepoint(codepoint, char):
+    assert SanitiseText.get_unicode_char_from_codepoint(codepoint) == char
+
+
+@pytest.mark.parametrize(
+    "bad_input", ["", "GJ", "00001", '0001";import sys;sys.exit(0)"']
+)
+def test_get_unicode_char_from_codepoint_rejects_bad_input(bad_input):
+    with pytest.raises(ValueError):
+        SanitiseText.get_unicode_char_from_codepoint(bad_input)
+
+
+@pytest.mark.parametrize(
+    "content, expected",
+    [
+        ("Łōdź", "?odz"),
+        (
+            "The quick brown fox jumps over the lazy dog",
+            "The quick brown fox jumps over the lazy dog",
+        ),
+    ],
+)
+def test_encode_string(content, expected):
+    assert SanitiseSMS.encode(content) == expected
+    assert SanitiseASCII.encode(content) == expected
+
+
+@pytest.mark.parametrize(
+    "content, cls, expected",
+    [
+        ("The quick brown fox jumps over the lazy dog", SanitiseSMS, set()),
+        (
+            "The “quick” brown fox has some downgradable characters\xa0",
+            SanitiseSMS,
+            set(),
+        ),
+        ("Need more 🐮🔔", SanitiseSMS, {"🐮", "🔔"}),
+        ("Ŵêlsh chârâctêrs ârê cômpâtîblê wîth SanitiseSMS", SanitiseSMS, set()),
+        ("Lots of GSM chars that arent ascii compatible:\n\r€", SanitiseSMS, set()),
+        (
+            "Lots of GSM chars that arent ascii compatible:\n\r€",
+            SanitiseASCII,
+            {"\n", "\r", "€"},
+        ),
+        ("Αυτό είναι ένα τεστ", SanitiseSMS, set()),
+        ("。、“”()：;？！", SanitiseSMS, set()),  # Chinese punctuation
+    ],
+)
+def test_sms_encoding_get_non_compatible_characters(content, cls, expected):
+    assert cls.get_non_compatible_characters(content) == expected
+
+
+@pytest.mark.parametrize(
+    "content, expected",
+    [
+        ("이것은 테스트입니다", True),  # Korean
+        ("Αυτό είναι ένα τεστ", True),  # Greek
+        ("Это проверка", True),  # Russian
+        ("นี่คือการทดสอบ", True),  # Thai
+        ("இது ஒரு சோதனை", True),  # Tamil
+        ("これはテストです", True),  # Japanese
+        ("Đây là một bài kiểm tra", True),  # Vietnamese
+        ("𐤓𐤓𐤓𐤈𐤆", False),  # Phoenician
+        ("这是一次测试", True),  # Mandarin (Simplified)
+        ("Bunda Türkçe karakterler var", True),  # Turkish
+        (
+            "盾牌镍币是第一种采用白铜制作的5美分硬币，由詹姆斯·B·朗埃克设计，从1866年发行到1883年再由自由女神头像镍币取代。",
+            True,
+        ),  # Chinese from wikipedia 1
+        (
+            "国际志愿者日為每年的12月5日，它是由联合国大会在1985年12月17日通过的A/RES/40/212决议[1]上确定的[2]。",
+            True,
+        ),  # Chinese from wikipedia 2
+        (
+            "哪一種多邊形內部至少存在一個可以看見多邊形所有邊界和所有內部區域的點？",
+            True,
+        ),  # Chinese from wikipedia 3
+        (
+            """都柏林在官方城市邊界內的人口是大約495,000人（愛爾蘭中央統計處2002年人口調查），
+            然而這種統計已經沒有什麼太大的意義，因為都柏林的市郊地區和衛星城鎮已經大幅地發展與擴張。""",
+            True,
+        ),  # Chinese from wikipedia 4
+        (
+            "一名是Dubh Linn（愛爾蘭語，意為「黑色的水池」）的英國習語。當然也有人質疑這語源。",
+            True,
+        ),  # Chinese from wikipedia 5
+        (
+            "都柏林拥有世界闻名的文学历史，曾经产生过许多杰出的文学家，例如诺贝尔文学奖得主威廉·巴特勒·叶芝、蕭伯納和塞繆爾·貝克特。",
+            True,
+        ),  # Chinese from wikipedia 6
+        (
+            "愛爾蘭國家博物館的四个分馆中有三個分館都位於都柏林：考古学分馆在基尔代尔街，装饰艺术和历史分馆在柯林斯军营，而自然史分馆在梅林街[12]。",
+            True,
+        ),  # Chinese from wikipedia 7
+        (
+            "從17世紀開始，城市在寬闊街道事務委員會的幫助下開始迅速擴張。乔治亚都柏林曾一度是大英帝國僅次於倫敦的第二大城市。",
+            True,
+        ),  # Chinese from wikipedia 8
+        (
+            "一些著名的都柏林街道建築仍以倒閉前在此經營的酒吧和商業公司命名。",
+            True,
+        ),  # Chinese from wikipedia 9
+        (
+            "1922年，隨著愛爾蘭的分裂，都柏林成為愛爾蘭自由邦（1922年–1937年）的首都。現在則為愛爾蘭共和國的首都。",
+            True,
+        ),  # Chinese from wikipedia 10
+        (
+            """Dưới đây là danh sách tất cả các tên người dùng hiện đang có
+            tại Wikipedia, hoặc những tên người dùng trong một nhóm chỉ định. """,
+            True,
+        ),  # Vietnamese from wikipedia 1
+        (
+            """Các bảo quản viên đảm nhận những trách nhiệm này với tư cách là tình
+            nguyện viên sau khi trải qua quá trình xem xét của cộng đồng. """,
+            True,
+        ),  # Vietnamese from wikipedia 2
+        (
+            """Họ không bao giờ được yêu cầu sử dụng các công cụ của mình và không bao
+            giờ được sử dụng chúng để giành lợi thế trong một cuộc tranh chấp mà họ có
+            tham gia. Không nên nhầm lẫn bảo quản viên với quản trị viên hệ
+            thống của Wikimedia ("sysadmins").""",
+            True,
+        ),  # Vietnamese from wikipedia 3
+        (
+            "Để đạt được mục tiêu chung đó, Wikipedia đề ra một số quy định và hướng dẫn. ",
+            True,
+        ),  # Vietnamese from wikipedia 4
+        ("Wikipedia là một bách khoa toàn thư. ", True),  # Vietnamese from wikipedia 5
+        (
+            "Phải đảm bảo bài viết mang lại ích lợi cho độc giả (coi độc giả là yếu tố quan trọng khi viết bài)",
+            True,
+        ),  # Vietnamese from wikipedia 6
+        (
+            """Bài viết ở Wikipedia có thể chứa đựng từ ngữ và hình ảnh gây khó chịu
+            nhưng chỉ vì mục đích tốt đẹp. Không cần thêm vào phủ định trách nhiệm.""",
+            True,
+        ),  # Vietnamese from wikipedia 7
+        (
+            "Đừng sử dụng hình ảnh mà chỉ có thể xem được chính xác với công cụ 3D.",
+            True,
+        ),  # Vietnamese from wikipedia 8
+        (
+            """Trích dẫn bất cứ nôi dung tranh luận gốc nào cũng nên có liên quan
+            đến tranh luận đó (hoặc minh họa cho phong cách) và chỉ nên dài vừa đủ.""",
+            True,
+        ),  # Vietnamese from wikipedia 9
+        (
+            """Không tung tin vịt, thông tin sai lệch hoặc nội dung không kiểm chứng được vào bài viết.
+            Tuy nhiên, những bài viết về những tin vịt nổi bật được chấp nhận.""",
+            True,
+        ),  # Vietnamese from wikipedia 10
+        (
+            "수록되어 있으며, 넘겨주기를 포함한 일반 문서 수는 1,434,776개。",
+            True,
+        ),  # Korean from wikipedia includes circle-period
+        (
+            "日本語表記にも対応するようになり[1]、徐々に日本人のユーザーも増大していった、と述べられている。",
+            True,
+        ),  # Japanese from wikipedia includes circle-period
+        (
+            "DSHS:我们发现您的账户存在潜在欺诈行为。请致电您的 EBT 卡背面的号码废止或前往当地办公室获取一个新账户。回复 “STOP(退订)” 退订",
+            True,
+        ),  # State of Washington Chinese Simplified
+        (
+            """DSHS៖ ប ើងោនកត់សម្គា ល់ប ើញក្ដរបោកប្រោស់ជាសក្ដា នុពលបៅបលើគណនីរបស់អ្នក។ សូមបៅបៅបលខ #បៅបលើខនងក្ដត
+            EBT របស់អ្នក ប ើមបីបោោះបង់ ឬក៏បៅក្ដន់ក្ដរយាិ ល័ បៅកនុងតំបន់របស់អ្នក
+            ប ើមបីបសនើសុំក្ដតថ្មី។ ប្លើ តបជាអ្កសរ ឈប់ ប ើមបីបញ្ឈប់""",
+            True,
+        ),  # State of Washington Khmer
+        (
+            """DSHS: 귀하의 계정 상에 사기가 일어났을 가능성이 포착되었습니다. 귀하의 EBT 카드 뒷면에있는
+            번호로 전화를 걸어 취소하거나 현지 사무소로 가서 새 것을 발급 받으세요. 중단하려면중단이라고 회신하세요.""",
+            True,
+        ),  # State of WA Korean
+        (
+            """ຂ ຄໍ້ ວາມການສໍ້ໂກງທອາດເປັນໄປໄດ ໍ້ DSHS: ພວກເຮາົໄດສໍ້ງັເກດເຫນັການສໂກງທີ່ອາດເປັນໄປໄດໃໍ້ນບນັຊຂ ອງທີ່ານ.
+            ໂທຫາ # ທ ຢີ່ ດາໍ້ນຫ ງັຂອງບດັ EBT ຂອງທີ່ານເພອຍກົ ເລກ ຫ ໄປຍງັຫອໍ້ງການປະຈາ ທອໍ້ງຖ ນຂອງທີ່ານ ເພີ່ອຂ
+            ບດັ ໃຫມີ່ . ຕອບກບັດວໍ້ ຍ STOP (ຢຸດເຊາົ) ເພອຢຸດເຊາົ""",
+            True,
+        ),  # State of WA Lao
+        (
+            """Fariin Khiyaamo Suurtogal ah DSHS: Waxaanu ka ogaanay khiyaamo suurtogal ah akoonkaaga.
+            Wax # ee ku yaal xaga danbe ee kadadhka
+            EBT si aad u joojisid ama u aadid xafiiska deegaanka uguna dalbatid a new one (mid cusub).
+            Ku jawaab JOOJI si aad u joojisid""",
+            True,
+        ),  # State of WA Somali
+        (
+            "إدارة الخدمات الاجتماعية والصحية في ولاية واشنطن (Washington State Department of Social and Health Services, WA DSHS): ستُجرى المقابلة الهاتفية معك المعنية بمراقبة جودة الطعام يوم xx/xx/xx الساعة 00:00 صباحًا/مساءً. قد يؤدي الفشل إلى إغلاق مخصصاتك. اتصل بالرقم 1-800-473-5661 إذا كانت لديك أسئلة.",  # noqa
+            True,
+        ),  # State of WA Arabic
+        (
+            "WA DSHS: ਤੁਹਾਡੀ ਗੁਣਵੱਤਾ ਨਿਯੰਤਰਣ ਭੋਜਨ ਫ਼ੋਨ ਇੰਟਰਵਿਊ xx/xx/xx 'ਤੇ ਸਵੇਰੇ 00:00 ਵਜੇ/ਸ਼ਾਮ 'ਤੇ ਹੈ। ਅਸਫਲਤਾ ਤੁਹਾਡੇ ਲਾਭਾਂ ਨੂੰ ਬੰਦ ਕਰਨ ਦਾ ਕਾਰਨ ਬਣ ਸਕਦੀ ਹੈ। ਸਵਾਲਾਂ ਨਾਲ 1-800-473-5661 'ਤੇ ਕਾਲ ਕਰੋ।",  # noqa
+            True,
+        ),  # State of WA Punjabi
+        (
+            "WA DSHS: ਵਿਅਕਤੀਗਤ ਭੋਜਨ ਵਿੱਚ ਤੁਹਾਡਾ ਗੁਣਵੱਤਾ ਨਿਯੰਤਰਣ ਇੰਟਰਵਿਊ xx/xx/xx 'ਤੇ ਸਵੇਰੇ 00:00 ਵਜੇ /ਸ਼ਾਮ 00:00 ਵਜੇ ਹੈ। ਅਸਫਲਤਾ ਤੁਹਾਡੇ ਲਾਭਾਂ ਨੂੰ ਬੰਦ ਕਰਨ ਦਾ ਕਾਰਨ ਬਣ ਸਕਦੀ ਹੈ। 1-800-473-5661 'ਤੇ w/ਸਵਾਲ ਨਾਲ ਕਾਲ ਕਰੋ।",  # noqa
+            True,
+        ),  # State of WA Punjabi
+    ],
+)
+def test_sms_supporting_additional_languages(content, expected):
+    assert SanitiseSMS.is_extended_language(content) is expected
+
+
+@pytest.mark.parametrize(
+    "content, expected",
+    [
+        ("이것은 테스트입니다", set()),  # Korean
+        ("Αυτό είναι ένα τεστ", set()),  # Greek
+        ("Это проверка", set()),  # Russian
+        ("นี่คือการทดสอบ", set()),  # Thai
+        ("இது ஒரு சோதனை", set()),  # Tamil
+        ("これはテストです", set()),  # Japanese
+        ("Đây là một bài kiểm tra", set()),  # Vietnamese
+        ("𐤓𐤓𐤓𐤈𐤆", {"𐤆", "𐤈", "𐤓"}),  # Phoenician
+        ("这是一次测试", set()),  # Mandarin (Simplified)
+        ("Bunda Türkçe karakterler var", set()),  # Turkish
+        ("。、“”()：;？！", set()),  # Chinese punctuation
+        (" ُ ُ", set()),  # Arabic diacritics
+        (
+            "WA DSHS: ਤੁਹਾਡੀ ਗੁਣਵੱਤਾ ਨਿਯੰਤਰਣ ਭੋਜਨ ਫ਼ੋਨ ਇੰਟਰਵਿਊ xx/xx/xx 'ਤੇ ਸਵੇਰੇ 00:00 ਵਜੇ/ਸ਼ਾਮ 'ਤੇ ਹੈ। ਅਸਫਲਤਾ ਤੁਹਾਡੇ ਲਾਭਾਂ ਨੂੰ ਬੰਦ ਕਰਨ ਦਾ ਕਾਰਨ ਬਣ ਸਕਦੀ ਹੈ। ਸਵਾਲਾਂ ਨਾਲ 1-800-473-5661 'ਤੇ ਕਾਲ ਕਰੋ।",  # noqa
+            set(),
+        ),  # Punjabi
+        (
+            "WA DSHS: ਵਿਅਕਤੀਗਤ ਭੋਜਨ ਵਿੱਚ ਤੁਹਾਡਾ ਗੁਣਵੱਤਾ ਨਿਯੰਤਰਣ ਇੰਟਰਵਿਊ xx/xx/xx 'ਤੇ ਸਵੇਰੇ 00:00 ਵਜੇ /ਸ਼ਾਮ 00:00 ਵਜੇ ਹੈ। ਅਸਫਲਤਾ ਤੁਹਾਡੇ ਲਾਭਾਂ ਨੂੰ ਬੰਦ ਕਰਨ ਦਾ ਕਾਰਨ ਬਣ ਸਕਦੀ ਹੈ। 1-800-473-5661 'ਤੇ w/ਸਵਾਲ ਨਾਲ ਕਾਲ ਕਰੋ।",  # noqa
+            set(),
+        ),  # more Punjabi
+    ],
+)
+def test_get_non_compatible_characters(content, expected):
+    assert SanitiseSMS.get_non_compatible_characters(content) == expected

tests/notifications_utils/test_serialised_model.py

@@ -0,0 +1,220 @@
+import sys
+
+import pytest
+
+from notifications_utils.serialised_model import (
+    SerialisedModel,
+    SerialisedModelCollection,
+)
+
+
+def test_cant_be_instatiated_with_abstract_properties():
+    class Custom(SerialisedModel):
+        pass
+
+    class CustomCollection(SerialisedModelCollection):
+        pass
+
+    with pytest.raises(TypeError) as e:
+        SerialisedModel()
+
+    if sys.version_info < (3, 9):
+        assert str(e.value) == (
+            "Can't instantiate abstract class SerialisedModel with abstract methods ALLOWED_PROPERTIES"
+        )
+    else:
+        assert "Can't instantiate abstract class SerialisedModel with abstract method ALLOWED_PROPERTIES"
+
+    with pytest.raises(TypeError) as e:
+        Custom()
+
+    if sys.version_info < (3, 9):
+        assert str(e.value) == (
+            "Can't instantiate abstract class Custom with abstract methods ALLOWED_PROPERTIES"
+        )
+    else:
+        assert str(e.value) == (
+            "Can't instantiate abstract class Custom without an implementation for abstract method 'ALLOWED_PROPERTIES'"
+        )
+
+    with pytest.raises(TypeError) as e:
+        SerialisedModelCollection()
+
+    if sys.version_info < (3, 9):
+        assert str(e.value) == (
+            "Can't instantiate abstract class SerialisedModelCollection with abstract methods model"
+        )
+    else:
+        assert str(e.value).startswith(
+            "Can't instantiate abstract class SerialisedModelCollection without an implementation"
+        )
+
+    with pytest.raises(TypeError) as e:
+        CustomCollection()
+
+    if sys.version_info < (3, 9):
+        assert str(e.value) == (
+            "Can't instantiate abstract class CustomCollection with abstract methods model"
+        )
+    else:
+        assert str(e.value) == (
+            "Can't instantiate abstract class CustomCollection without an implementation for abstract method 'model'"
+        )
+
+
+def test_looks_up_from_dict():
+    class Custom(SerialisedModel):
+        ALLOWED_PROPERTIES = {"foo"}
+
+    assert Custom({"foo": "bar"}).foo == "bar"
+
+
+def test_cant_override_custom_property_from_dict():
+    class Custom(SerialisedModel):
+        ALLOWED_PROPERTIES = {"foo"}
+
+        @property
+        def foo(self):
+            return "bar"
+
+    with pytest.raises(AttributeError) as e:
+        assert Custom({"foo": "NOPE"}).foo == "bar"
+    assert (
+        str(e.value)
+        == "property 'foo' of 'test_cant_override_custom_property_from_dict.<locals>.Custom' object has no setter"
+    )
+
+
+@pytest.mark.parametrize(
+    "json_response",
+    (
+        {},
+        {"foo": "bar"},  # Should still raise an exception
+    ),
+)
+def test_model_raises_for_unknown_attributes(json_response):
+    class Custom(SerialisedModel):
+        ALLOWED_PROPERTIES = set()
+
+    model = Custom(json_response)
+
+    assert model.ALLOWED_PROPERTIES == set()
+
+    with pytest.raises(AttributeError) as e:
+        model.foo
+
+    assert str(e.value) == ("'Custom' object has no attribute 'foo'")
+
+
+def test_model_raises_keyerror_if_item_missing_from_dict():
+    class Custom(SerialisedModel):
+        ALLOWED_PROPERTIES = {"foo"}
+
+    with pytest.raises(KeyError) as e:
+        Custom({}).foo
+
+    assert str(e.value) == "'foo'"
+
+
+@pytest.mark.parametrize(
+    "json_response",
+    (
+        {},
+        {"foo": "bar"},  # Should be ignored
+    ),
+)
+def test_model_doesnt_swallow_attribute_errors(json_response):
+    class Custom(SerialisedModel):
+        ALLOWED_PROPERTIES = set()
+
+        @property
+        def foo(self):
+            raise AttributeError("Something has gone wrong")
+
+    with pytest.raises(AttributeError) as e:
+        Custom(json_response).foo
+
+    assert str(e.value) == "Something has gone wrong"
+
+
+def test_dynamic_properties_are_introspectable():
+    class Custom(SerialisedModel):
+        ALLOWED_PROPERTIES = {"foo", "bar", "baz"}
+
+    instance = Custom({"foo": "", "bar": "", "baz": ""})
+
+    assert dir(instance)[-3:] == ["bar", "baz", "foo"]
+
+
+def test_empty_serialised_model_collection():
+    class CustomCollection(SerialisedModelCollection):
+        model = None
+
+    instance = CustomCollection([])
+
+    assert not instance
+    assert len(instance) == 0
+
+
+def test_serialised_model_collection_returns_models_from_list():
+    class Custom(SerialisedModel):
+        ALLOWED_PROPERTIES = {"x"}
+
+    class CustomCollection(SerialisedModelCollection):
+        model = Custom
+
+    instance = CustomCollection(
+        [
+            {"x": "foo"},
+            {"x": "bar"},
+            {"x": "baz"},
+        ]
+    )
+
+    assert instance
+    assert len(instance) == 3
+
+    assert instance[0].x == "foo"
+    assert instance[1].x == "bar"
+    assert instance[2].x == "baz"
+
+    assert [item.x for item in instance] == [
+        "foo",
+        "bar",
+        "baz",
+    ]
+
+    assert [type(item) for item in instance + [1, 2, 3]] == [
+        Custom,
+        Custom,
+        Custom,
+        int,
+        int,
+        int,
+    ]
+
+    instance_2 = CustomCollection(
+        [
+            {"x": "red"},
+            {"x": "green"},
+            {"x": "blue"},
+        ]
+    )
+
+    assert [item.x for item in instance + instance_2] == [
+        "foo",
+        "bar",
+        "baz",
+        "red",
+        "green",
+        "blue",
+    ]
+
+    assert [item.x for item in instance_2 + instance] == [
+        "red",
+        "green",
+        "blue",
+        "foo",
+        "bar",
+        "baz",
+    ]

tests/notifications_utils/test_take.py

@@ -0,0 +1,19 @@
+from notifications_utils.take import Take
+
+
+def _uppercase(value):
+    return value.upper()
+
+
+def _append(value, to_append):
+    return value + to_append
+
+
+def _prepend_with_service_name(value, service_name=None):
+    return "{}: {}".format(service_name, value)
+
+
+def test_take():
+    assert "Service name: HELLO WORLD!" == Take("hello world").then(_uppercase).then(
+        _append, "!"
+    ).then(_prepend_with_service_name, service_name="Service name")

tests/notifications_utils/test_template_change.py

@@ -0,0 +1,135 @@
+import pytest
+
+from notifications_utils.template_change import TemplateChange
+
+from .test_base_template import ConcreteTemplate
+
+
+@pytest.mark.parametrize(
+    "old_template, new_template, should_differ",
+    [
+        (
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            False,
+        ),
+        (
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            ConcreteTemplate({"content": "((3)) ((2)) ((1))"}),
+            False,
+        ),
+        (
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            ConcreteTemplate({"content": "((1)) ((1)) ((2)) ((2)) ((3)) ((3))"}),
+            False,
+        ),
+        (
+            ConcreteTemplate({"content": "((1))"}),
+            ConcreteTemplate({"content": "((1)) ((2))"}),
+            True,
+        ),
+        (
+            ConcreteTemplate({"content": "((1)) ((2))"}),
+            ConcreteTemplate({"content": "((1))"}),
+            True,
+        ),
+        (
+            ConcreteTemplate({"content": "((a)) ((b))"}),
+            ConcreteTemplate({"content": "((A)) (( B_ ))"}),
+            False,
+        ),
+    ],
+)
+def test_checking_for_difference_between_templates(
+    old_template, new_template, should_differ
+):
+    assert (
+        TemplateChange(old_template, new_template).has_different_placeholders
+        == should_differ
+    )
+
+
+@pytest.mark.parametrize(
+    "old_template, new_template, placeholders_added",
+    [
+        (
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            set(),
+        ),
+        (
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            ConcreteTemplate({"content": "((1)) ((1)) ((2)) ((2)) ((3)) ((3))"}),
+            set(),
+        ),
+        (
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            ConcreteTemplate({"content": "((1))"}),
+            set(),
+        ),
+        (
+            ConcreteTemplate({"content": "((1))"}),
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            set(["2", "3"]),
+        ),
+        (
+            ConcreteTemplate({"content": "((a))"}),
+            ConcreteTemplate({"content": "((A)) ((B)) ((C))"}),
+            set(["B", "C"]),
+        ),
+    ],
+)
+def test_placeholders_added(old_template, new_template, placeholders_added):
+    assert (
+        TemplateChange(old_template, new_template).placeholders_added
+        == placeholders_added
+    )
+
+
+@pytest.mark.parametrize(
+    "old_template, new_template, placeholders_removed",
+    [
+        (
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            set(),
+        ),
+        (
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            ConcreteTemplate({"content": "((1)) ((1)) ((2)) ((2)) ((3)) ((3))"}),
+            set(),
+        ),
+        (
+            ConcreteTemplate({"content": "((1))"}),
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            set(),
+        ),
+        (
+            ConcreteTemplate({"content": "((1)) ((2)) ((3))"}),
+            ConcreteTemplate({"content": "((1))"}),
+            set(["2", "3"]),
+        ),
+        (
+            ConcreteTemplate({"content": "((a)) ((b)) ((c))"}),
+            ConcreteTemplate({"content": "((A))"}),
+            set(["b", "c"]),
+        ),
+    ],
+)
+def test_placeholders_removed(old_template, new_template, placeholders_removed):
+    assert (
+        TemplateChange(old_template, new_template).placeholders_removed
+        == placeholders_removed
+    )
+
+
+def test_ordering_of_placeholders_is_preserved():
+    before = ConcreteTemplate({"content": "((dog)) ((cat)) ((rat))"})
+    after = ConcreteTemplate({"content": "((platypus)) ((echidna)) ((quokka))"})
+    change = TemplateChange(before, after)
+    assert change.placeholders_removed == ["dog", "cat", "rat"] == before.placeholders
+    assert (
+        change.placeholders_added
+        == ["platypus", "echidna", "quokka"]
+        == after.placeholders
+    )

tests/notifications_utils/test_timezones.py

@@ -0,0 +1,36 @@
+import dateutil
+import pytest
+
+from notifications_utils.timezones import utc_string_to_aware_gmt_datetime
+
+
+@pytest.mark.parametrize(
+    "input_value,expectation",
+    [
+        ("foo", pytest.raises(dateutil.parser._parser.ParserError)),
+        (100, pytest.raises(TypeError)),
+        (True, pytest.raises(TypeError)),
+        (False, pytest.raises(TypeError)),
+        (None, pytest.raises(TypeError)),
+    ],
+)
+def test_utc_string_to_aware_gmt_datetime_rejects_bad_input(input_value, expectation):
+    with expectation:
+        utc_string_to_aware_gmt_datetime(input_value)
+
+
+@pytest.mark.parametrize(
+    "naive_time, expected_aware_hour",
+    [
+        ("2000-12-1 20:01", "15:01"),
+        ("2000-06-1 20:01", "16:01"),
+    ],
+)
+def test_utc_string_to_aware_gmt_datetime_handles_summer_and_winter(
+    naive_time,
+    expected_aware_hour,
+):
+    assert (
+        utc_string_to_aware_gmt_datetime(naive_time).strftime("%H:%M")
+        == expected_aware_hour
+    )

tests/notifications_utils/test_url_safe_tokens.py

@@ -0,0 +1,36 @@
+import urllib
+
+from itsdangerous import BadSignature, SignatureExpired
+from pytest import fail
+
+from notifications_utils.url_safe_token import check_token, generate_token
+
+
+def test_should_return_payload_from_signed_token():
+    payload = "email@something.com"
+    token = generate_token(payload, "secret-key", "dangerous-salt")
+    token = urllib.parse.unquote(token)
+    assert payload == check_token(token, "secret-key", "dangerous-salt", 30)
+
+
+def test_should_throw_exception_when_token_is_tampered_with():
+    import uuid
+
+    token = generate_token(str(uuid.uuid4()), "secret-key", "dangerous-salt")
+    try:
+        check_token(token + "qerqwer", "secret-key", "dangerous-salt", 30)
+        fail()
+    except BadSignature:
+        pass
+
+
+def test_return_none_when_token_is_expired():
+    max_age = -1000
+    payload = "some_payload"
+    token = generate_token(payload, "secret-key", "dangerous-salt")
+    token = urllib.parse.unquote(token)
+    try:
+        assert check_token(token, "secret-key", "dangerous-salt", max_age) is None
+        fail("Expected a SignatureExpired exception")
+    except SignatureExpired:
+        pass