Update h11 to 0.16.0

refreshing login.gov cert instructions

Makefile

@@ -5,7 +5,8 @@ DATE = $(shell date +%Y-%m-%d:%H:%M:%S)
 APP_VERSION_FILE = app/version.py
 
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
-GIT_COMMIT ?= $(shell git rev-parse HEAD)
+GIT_COMMIT ?= $(shell git rev-parse HEAD 2> /dev/null || echo "")
+GIT_HOOKS_PATH ?= $(shell git config --global core.hooksPath || echo "")
 
 ## DEVELOPMENT
 
@@ -23,6 +24,19 @@ bootstrap: ## Set up everything to run the app
 	createdb test_notification_api || true
 	(poetry run flask db upgrade) || true
 
+.PHONY: bootstrap-with-git-hooks
+bootstrap-with-git-hooks:  ## Sets everything up and accounts for pre-existing git hooks
+	make generate-version-file
+	poetry self add poetry-dotenv-plugin
+	poetry lock --no-update
+	poetry install --sync --no-root
+	git config --global --unset-all core.hooksPath
+	poetry run pre-commit install
+	git config --global core.hookspath "${GIT_HOOKS_PATH}"
+	createdb notification_api || true
+	createdb test_notification_api || true
+	(poetry run flask db upgrade) || true
+
 .PHONY: bootstrap-with-docker
 bootstrap-with-docker: ## Build the image to run the app in Docker
 	docker build -f docker/Dockerfile -t notifications-api .
@@ -50,7 +64,7 @@ too-complex:
 	poetry run radon cc ./app -a -nc
 
 .PHONY: run-flask
-run-flask: ## Run flask
+run-flask:
 	poetry run newrelic-admin run-program flask run -p 6011 --host=0.0.0.0
 
 .PHONY: run-celery

app/__init__.py

@@ -13,6 +13,7 @@ from flask import current_app, g, has_request_context, jsonify, make_response, r
 from flask.ctx import has_app_context
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
+from flask_socketio import SocketIO
 from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
 from sqlalchemy import event
 from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
@@ -94,6 +95,14 @@ zendesk_client = ZendeskClient()
 redis_store = RedisClient()
 document_download_client = DocumentDownloadClient()
 
+socketio = SocketIO(
+    cors_allowed_origins=[
+        config.Config.ADMIN_BASE_URL,
+    ],
+    message_queue=config.Config.REDIS_URL,
+    logger=True,
+    engineio_logger=True,
+)
 
 notification_provider_clients = NotificationProviderClients()
 
@@ -111,6 +120,11 @@ def create_app(application):
     application.config["NOTIFY_APP_NAME"] = application.name
     init_app(application)
 
+    socketio.init_app(application)
+
+    from app.socket_handlers import register_socket_handlers
+
+    register_socket_handlers(socketio)
     request_helper.init_app(application)
     db.init_app(application)
     migrate.init_app(application, db=db)

app/clients/document_download.py

@@ -33,7 +33,7 @@ class DocumentDownloadClient:
                     "document": file_contents,
                     "is_csv": is_csv or False,
                 },
-                timeout=30
+                timeout=30,
             )
 
             response.raise_for_status()

app/clients/performance_platform/performance_platform_client.py

@@ -29,7 +29,7 @@ class PerformancePlatformClient:
                 self.performance_platform_url + payload["dataType"],
                 json=payload,
                 headers=headers,
-                timeout=30
+                timeout=30,
             )
 
             if resp.status_code == 200:

app/config.py

@@ -179,6 +179,7 @@ class Config(object):
     S3_RESOURCE = session.resource("s3", config=AWS_CLIENT_CONFIG)
 
     CELERY = {
+        "broker_connection_retry_on_startup": True,
         "worker_max_tasks_per_child": 500,
         "task_ignore_result": True,
         "result_persistent": False,

app/cronitor.py

@@ -35,7 +35,7 @@ def cronitor(task_name):
                     params={
                         "host": current_app.config["API_HOST_NAME"],
                     },
-                    timeout=30
+                    timeout=30,
                 )
                 resp.raise_for_status()
             except requests.RequestException as e:

app/dao/notifications_dao.py

@@ -26,9 +26,11 @@ from werkzeug.datastructures import MultiDict
 from app import create_uuid, db
 from app.dao.dao_utils import autocommit
 from app.dao.inbound_sms_dao import Pagination
+from app.dao.jobs_dao import dao_get_job_by_id
 from app.enums import KeyType, NotificationStatus, NotificationType
 from app.models import FactNotificationStatus, Notification, NotificationHistory
 from app.utils import (
+    emit_job_update_summary,
     escape_special_characters,
     get_midnight_in_utc,
     midnight_n_days_ago,
@@ -895,6 +897,19 @@ def dao_update_delivery_receipts(receipts, delivered):
         f"#loadtestperformance batch update query time: \
         updated {len(receipts)} notification in {elapsed_time} ms"
     )
+    job_ids = (
+        db.session.execute(
+            select(Notification.job_id).where(
+                Notification.message_id.in_(id_to_carrier.keys())
+            )
+        )
+        .scalars()
+        .all()
+    )
+
+    for job_id in set(job_ids):
+        job = dao_get_job_by_id(job_id)
+        emit_job_update_summary(job)
 
 
 def dao_close_out_delivery_receipts():

app/delivery/send_to_providers.py

@@ -105,6 +105,17 @@ def send_sms_to_provider(notification):
                 # The future home of the validation is TBD
                 _experimentally_validate_phone_numbers(recipient)
 
+                # TODO current we allow US phone numbers to be uploaded without the country code (1)
+                # This will break certain international phone numbers (Norway, Denmark, East Timor)
+                # When we officially announce support for international numbers, US numbers must contain
+                # their country code.
+                recipient = str(recipient)
+                if len(recipient) == 10:
+                    if os.getenv("NOTIFY_ENVIRONMENT") not in [
+                        "test"
+                    ]:  # we want to test intl support
+                        recipient = f"1{recipient}"
+
                 sender_numbers = get_sender_numbers(notification)
                 if notification.reply_to_text not in sender_numbers:
                     raise ValueError(

app/socket_handlers.py

@@ -0,0 +1,16 @@
+from flask import current_app, request
+from flask_socketio import join_room, leave_room
+
+
+def register_socket_handlers(socketio):
+    @socketio.on("join")
+    def on_join(data):  # noqa: F401
+        room = data.get("room")
+        join_room(room)
+        current_app.logger.info(f"Socket {request.sid} joined room {room}")
+
+    @socketio.on("leave")
+    def on_leave(data):  # noqa: F401
+        room = data.get("room")
+        leave_room(room)
+        current_app.logger.info(f"Socket {request.sid} left room {room}")

app/utils.py

@@ -131,3 +131,18 @@ def utc_now():
 def debug_not_production(msg):
     if os.getenv("NOTIFY_ENVIRONMENT") not in ["production"]:
         current_app.logger.info(msg)
+
+
+def emit_job_update_summary(job):
+    from app import socketio
+
+    current_app.logger.info(f"Emitting summary for job {job.id}")
+    socketio.emit(
+        "job_updated",
+        {
+            "job_id": str(job.id),
+            "job_status": job.job_status,
+            "notification_count": job.notification_count,
+        },
+        room=f"job-{job.id}",
+    )

docs/all.md

@@ -1507,3 +1507,19 @@ Note: better to search on space 'notify-production' rather than specifically for
 #notify-admin-1505 (general login issues)
 #notify-admin-1701 (wrong sender phone number)
 #notify-admin-1859 (job is created with created_at being the wrong time)
+
+### refreshing the login.gov certificate
+
+1. generate certificate:   `openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.crt -nodes`
+2. update the github secrets for staging, demo, production (contents of key.pem go in LOGIN_PEM and contents of cert.crt in LOGIN_PUB).
+DO NOT RESTAGE YET.
+3. use the same certificate for staging, demo, and production
+4. login to the login.gov partner app (https://portal.int.identitysandbox.gov)
+5. add the new certificate to the production version of Notify in the partner app (our partner app account has sandbox and production)
+6. Make a Zendesk support request for login.gov to push the new version of Notify (https://zendesk.login.gov)
+7. Do not delete the old certificate, because you need things to keep working until you complete the transition.
+8. When you receive an email from login.gov that the app has been pushed successfully, restage notify on the staging tier
+9. If staging works, you can restage demo and production
+10. Delete the old certificate in the partner app, send another zendesk request to push again.  This is best practice but a lower
+priority, because certificates eventually expire anyway and we have changed the certificate in github secrets, so the old cert is
+no longer relevant.

migrations/versions/0216_remove_colours.py

@@ -2,7 +2,7 @@
  Revision ID: 0216_remove_colours
 Revises: 0215_email_brand_type
 Create Date: 2018-08-24 13:36:49.346156
- """
+"""
 
 import sqlalchemy as sa
 from alembic import op

migrations/versions/0219_default_email_branding.py

@@ -2,7 +2,7 @@
  Revision ID: 0219_default_email_branding
 Revises: 0216_remove_colours
 Create Date: 2018-08-24 13:36:49.346156
- """
+"""
 
 from alembic import op
 from sqlalchemy import text

migrations/versions/0220_email_brand_type_non_null.py

@@ -2,7 +2,7 @@
  Revision ID: 0220_email_brand_type_non_null
 Revises: 0219_default_email_branding
 Create Date: 2018-08-24 13:36:49.346156
- """
+"""
 
 from alembic import op
 

migrations/versions/0221_nullable_service_branding.py

@@ -2,7 +2,7 @@
  Revision ID: 0221_nullable_service_branding
 Revises: 0220_email_brand_type_non_null
 Create Date: 2018-08-24 13:36:49.346156
- """
+"""
 
 from alembic import op
 

migrations/versions/0222_drop_service_branding.py

@@ -2,7 +2,7 @@
  Revision ID: 0222_drop_service_branding
 Revises: 0221_nullable_service_branding
 Create Date: 2018-08-24 13:36:49.346156
- """
+"""
 
 import sqlalchemy as sa
 from alembic import op

migrations/versions/0223_add_domain_constraint.py

@@ -2,7 +2,7 @@
 Revision ID: 0223_add_domain_constraint
 Revises: 0222_drop_service_branding
 Create Date: 2018-08-24 13:36:49.346156
- """
+"""
 
 from alembic import op
 

notifications_utils/formatters.py

@@ -12,15 +12,15 @@ from notifications_utils.sanitise_text import SanitiseSMS
 from . import email_with_smart_quotes_regex
 
 OBSCURE_ZERO_WIDTH_WHITESPACE = (
-    "\u180E"  # Mongolian vowel separator
-    "\u200B"  # zero width space
-    "\u200C"  # zero width non-joiner
-    "\u200D"  # zero width joiner
+    "\u180e"  # Mongolian vowel separator
+    "\u200b"  # zero width space
+    "\u200c"  # zero width non-joiner
+    "\u200d"  # zero width joiner
     "\u2060"  # word joiner
-    "\uFEFF"  # zero width non-breaking space
+    "\ufeff"  # zero width non-breaking space
 )
 
-OBSCURE_FULL_WIDTH_WHITESPACE = "\u00A0"  # non breaking space
+OBSCURE_FULL_WIDTH_WHITESPACE = "\u00a0"  # non breaking space
 
 ALL_WHITESPACE = (
     string.whitespace + OBSCURE_ZERO_WIDTH_WHITESPACE + OBSCURE_FULL_WIDTH_WHITESPACE
@@ -61,7 +61,7 @@ more_than_two_newlines_in_a_row = re.compile(r"\n{3,}")
 def unlink_govuk_escaped(message):
     return re.sub(
         govuk_not_a_link,
-        r"\1\2\3" + ".\u200B" + r"\4",  # Unicode zero-width space
+        r"\1\2\3" + ".\u200b" + r"\4",  # Unicode zero-width space
         message,
     )
 

notifications_utils/sanitise_text.py

@@ -15,13 +15,13 @@ class SanitiseText:
         "’": "'",  # RIGHT SINGLE QUOTATION MARK (U+2019)
         "“": '"',  # LEFT DOUBLE QUOTATION MARK (U+201C)
         "”": '"',  # RIGHT DOUBLE QUOTATION MARK (U+201D)
-        "\u180E": "",  # Mongolian vowel separator
-        "\u200B": "",  # zero width space
-        "\u200C": "",  # zero width non-joiner
-        "\u200D": "",  # zero width joiner
+        "\u180e": "",  # Mongolian vowel separator
+        "\u200b": "",  # zero width space
+        "\u200c": "",  # zero width non-joiner
+        "\u200d": "",  # zero width joiner
         "\u2060": "",  # word joiner
-        "\uFEFF": "",  # zero width non-breaking space
-        "\u00A0": " ",  # NON BREAKING WHITE SPACE (U+200B)
+        "\ufeff": "",  # zero width non-breaking space
+        "\u00a0": " ",  # NON BREAKING WHITE SPACE (U+200B)
         "\t": " ",  # TAB
     }
 

package-lock.json

@@ -0,0 +1,120 @@
+{
+  "name": "notifications-api",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "dependencies": {
+        "socket.io-client": "^4.8.1"
+      }
+    },
+    "node_modules/@socket.io/component-emitter": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz",
+      "integrity": "sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==",
+      "license": "MIT"
+    },
+    "node_modules/debug": {
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
+      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/engine.io-client": {
+      "version": "6.6.3",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-6.6.3.tgz",
+      "integrity": "sha512-T0iLjnyNWahNyv/lcjS2y4oE358tVS/SYQNxYXGAJ9/GLgH4VCvOQ/mhTjqU88mLZCQgiG8RIegFHYCdVC+j5w==",
+      "license": "MIT",
+      "dependencies": {
+        "@socket.io/component-emitter": "~3.1.0",
+        "debug": "~4.3.1",
+        "engine.io-parser": "~5.2.1",
+        "ws": "~8.17.1",
+        "xmlhttprequest-ssl": "~2.1.1"
+      }
+    },
+    "node_modules/engine.io-parser": {
+      "version": "5.2.3",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz",
+      "integrity": "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
+    "node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+      "license": "MIT"
+    },
+    "node_modules/socket.io-client": {
+      "version": "4.8.1",
+      "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.1.tgz",
+      "integrity": "sha512-hJVXfu3E28NmzGk8o1sHhN3om52tRvwYeidbj7xKy2eIIse5IoKX3USlS6Tqt3BHAtflLIkCQBkzVrEEfWUyYQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@socket.io/component-emitter": "~3.1.0",
+        "debug": "~4.3.2",
+        "engine.io-client": "~6.6.1",
+        "socket.io-parser": "~4.2.4"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
+    "node_modules/socket.io-parser": {
+      "version": "4.2.4",
+      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.4.tgz",
+      "integrity": "sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==",
+      "license": "MIT",
+      "dependencies": {
+        "@socket.io/component-emitter": "~3.1.0",
+        "debug": "~4.3.1"
+      },
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
+    "node_modules/ws": {
+      "version": "8.17.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
+      "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/xmlhttprequest-ssl": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.1.2.tgz",
+      "integrity": "sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    }
+  }
+}

package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "socket.io-client": "^4.8.1"
+  }
+}

pyproject.toml

@@ -18,18 +18,18 @@ celery = {version = "==5.4.0", extras = ["redis"]}
 certifi = ">=2022.12.7"
 cffi = "==1.17.1"
 charset-normalizer = "^3.4.1"
-click = "==8.1.7"
+click = "==8.1.8"
 click-datetime = "==0.4.0"
 click-didyoumean = "==0.3.1"
 click-plugins = "==1.1.1"
 click-repl = "==0.3.0"
 deprecated = "==1.2.14"
-eventlet = "==0.36.1"
+eventlet = "==0.39.1"
 expiringdict = "==1.2.2"
 flask = "~=3.0"
 flask-bcrypt = "==1.0.1"
 flask-marshmallow = "==1.2.1"
-flask-migrate = "==4.0.7"
+flask-migrate = "==4.1.0"
 flask-redis = "==0.4.0"
 flask-sqlalchemy = "==3.1.1"
 gunicorn = {version = "==23.0.0", extras = ["eventlet"]}
@@ -41,7 +41,7 @@ marshmallow-sqlalchemy = "==1.0.0"
 newrelic = "*"
 notifications-python-client = "==10.0.0"
 oscrypto = { git = "https://github.com/wbond/oscrypto.git", rev = "1547f53" }
-packaging = "==24.2"
+packaging = "==25.0"
 poetry-dotenv-plugin = "==0.2.0"
 psycopg2-binary = "==2.9.9"
 pyjwt = "==2.10.1"
@@ -52,11 +52,11 @@ faker = "^26.0.0"
 async-timeout = "^5.0.1"
 bleach = "^6.1.0"
 geojson = "^3.2.0"
-numpy = "^2.2.3"
+numpy = "^2.2.5"
 ordered-set = "^4.1.0"
 phonenumbers = "^8.13.42"
-python-json-logger = "^2.0.7"
-regex = "^2024.7.24"
+python-json-logger = "^3.3.0"
+regex = "^2024.11.6"
 shapely = "^2.0.5"
 smartypants = "^2.0.1"
 mistune = "^3.1.3"
@@ -76,6 +76,7 @@ itsdangerous = "^2.2.0"
 jinja2 = "^3.1.6"
 redis = "^5.0.8"
 requests = "^2.32.3"
+flask-socketio = "^5.5.1"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -90,7 +91,7 @@ freezegun = "^1.5.1"
 honcho = "*"
 isort = "^5.13.2"
 jinja2-cli = {version = "==0.8.2", extras = ["yaml"]}
-moto = "==5.1.0"
+moto = "==5.1.4"
 pip-audit = "*"
 pre-commit = "^4.2.0"
 pytest = "^8.3.2"

tests/app/dao/notification_dao/test_notification_dao.py

@@ -2029,12 +2029,23 @@ def test_update_delivery_receipts(mocker):
     mock_update.where.return_value = mock_where
     mock_where.values.return_value = mock_values
 
-    mock_session.execute.return_value = None
+    FakeJob = type(
+        "FakeJob",
+        (object,),
+        {"id": "job-123", "notification_count": 5, "job_status": "delivered"},
+    )
+
+    fake_result = MagicMock()
+    fake_result.scalars.return_value.all.return_value = ["job-1", "job-2"]
+    fake_result.scalars.return_value.one.return_value = FakeJob()
+
+    mock_session.execute.side_effect = lambda *args, **kwargs: fake_result
     with patch("app.dao.notifications_dao.update", return_value=mock_update):
         dao_update_delivery_receipts(receipts, delivered)
     mock_update.where.assert_called_once()
     mock_where.values.assert_called_once()
-    mock_session.execute.assert_called_once_with(mock_values)
+    mock_session.execute.assert_any_call(mock_values)
+    assert mock_session.execute.call_count == 4
     mock_session.commit.assert_called_once()
 
     args, kwargs = mock_where.values.call_args

tests/app/notifications/test_process_notification.py

@@ -278,7 +278,7 @@ def test_send_notification_to_queue_throws_exception_deletes_notification(
     with pytest.raises(Boto3Error):
         send_notification_to_queue(sample_notification, False)
     mocked.assert_called_once_with(
-        [(str(sample_notification.id))], queue="send-sms-tasks", countdown=60
+        [str(sample_notification.id)], queue="send-sms-tasks", countdown=60
     )
 
     assert _get_notification_query_count() == 0

tests/app/organization/test_invite_rest.py

@@ -75,7 +75,7 @@ def test_create_invited_org_user(
     # assert len(notification.personalisation["url"]) > len(expected_start_of_invite_url)
 
     mocked.assert_called_once_with(
-        [(str(notification.id))], queue="notify-internal-tasks", countdown=60
+        [str(notification.id)], queue="notify-internal-tasks", countdown=60
     )
 
 

tests/app/service_invite/test_service_invite_rest.py

@@ -92,7 +92,7 @@ def test_create_invited_user(
     )
 
     mocked.assert_called_once_with(
-        [(str(notification.id))], queue="notify-internal-tasks", countdown=60
+        [str(notification.id)], queue="notify-internal-tasks", countdown=60
     )
 
 

tests/notifications_utils/test_field.py

@@ -257,7 +257,7 @@ def test_what_will_trigger_conditional_placeholder(value):
             "list: ",
         ),
         (
-            {"placeholder": [" ", " \t ", "\u180E"]},
+            {"placeholder": [" ", " \t ", "\u180e"]},
             "list: ",
             "list: ",
         ),

tests/notifications_utils/test_formatters.py

@@ -66,7 +66,7 @@ def test_HTML_template_has_URLs_replaced_with_links():
 
 def test_escaping_govuk_in_email_templates():
     template_content = "GOV.UK"
-    expected = "GOV.\u200BUK"
+    expected = "GOV.\u200bUK"
     assert unlink_govuk_escaped(template_content) == expected
     template_json = {
         "content": template_content,
@@ -81,23 +81,23 @@ def test_escaping_govuk_in_email_templates():
     ("template_content", "expected"),
     [
         # Cases that we add the breaking space
-        ("GOV.UK", "GOV.\u200BUK"),
-        ("gov.uk", "gov.\u200Buk"),
+        ("GOV.UK", "GOV.\u200bUK"),
+        ("gov.uk", "gov.\u200buk"),
         (
             "content with space infront GOV.UK",
-            "content with space infront GOV.\u200BUK",
+            "content with space infront GOV.\u200bUK",
         ),
-        ("content with tab infront\tGOV.UK", "content with tab infront\tGOV.\u200BUK"),
+        ("content with tab infront\tGOV.UK", "content with tab infront\tGOV.\u200bUK"),
         (
             "content with newline infront\nGOV.UK",
-            "content with newline infront\nGOV.\u200BUK",
+            "content with newline infront\nGOV.\u200bUK",
         ),
-        ("*GOV.UK", "*GOV.\u200BUK"),
-        ("#GOV.UK", "#GOV.\u200BUK"),
-        ("^GOV.UK", "^GOV.\u200BUK"),
-        (" #GOV.UK", " #GOV.\u200BUK"),
-        ("GOV.UK with CONTENT after", "GOV.\u200BUK with CONTENT after"),
-        ("#GOV.UK with CONTENT after", "#GOV.\u200BUK with CONTENT after"),
+        ("*GOV.UK", "*GOV.\u200bUK"),
+        ("#GOV.UK", "#GOV.\u200bUK"),
+        ("^GOV.UK", "^GOV.\u200bUK"),
+        (" #GOV.UK", " #GOV.\u200bUK"),
+        ("GOV.UK with CONTENT after", "GOV.\u200bUK with CONTENT after"),
+        ("#GOV.UK with CONTENT after", "#GOV.\u200bUK with CONTENT after"),
         # Cases that we don't add the breaking space
         ("https://gov.uk", "https://gov.uk"),
         ("https://www.gov.uk", "https://www.gov.uk"),
@@ -366,7 +366,7 @@ def test_unicode_dash_lookup():
         """
         \t    bar
     """,
-        " \u180E\u200B \u200C bar \u200D \u2060\uFEFF ",
+        " \u180e\u200b \u200c bar \u200d \u2060\ufeff ",
     ],
 )
 def test_strip_all_whitespace(value):
@@ -378,7 +378,7 @@ def test_strip_all_whitespace(value):
     [
         "notifications-email",
         "  \tnotifications-email \x0c ",
-        "\rn\u200Coti\u200Dfi\u200Bcati\u2060ons-\u180Eemai\uFEFFl\uFEFF",
+        "\rn\u200coti\u200dfi\u200bcati\u2060ons-\u180eemai\ufeffl\ufeff",
     ],
 )
 def test_strip_and_remove_obscure_whitespace(value):
@@ -421,12 +421,12 @@ def test_strip_unsupported_characters():
 @pytest.mark.parametrize(
     "value",
     [
-        "\u200C Your tax   is\ndue\n\n",
+        "\u200c Your tax   is\ndue\n\n",
         "  Your tax is due  ",
         # Non breaking spaces replaced by single spaces
-        "\u00A0Your\u00A0tax\u00A0 is\u00A0\u00A0due\u00A0",
+        "\u00a0Your\u00a0tax\u00a0 is\u00a0\u00a0due\u00a0",
         # zero width spaces are removed
-        "\u180EYour \u200Btax\u200C is \u200D\u2060due \uFEFF",
+        "\u180eYour \u200btax\u200c is \u200d\u2060due \ufeff",
         # tabs are replaced by single spaces
         "\tYour tax\tis due  ",
     ],

tests/notifications_utils/test_postal_address.py

@@ -559,7 +559,7 @@ def test_normalise_postcode(postcode, normalised_postcode):
         ("N5 1AA", True),
         ("SO14 6WB", True),
         ("so14 6wb", True),
-        ("so14\u00A06wb", True),
+        ("so14\u00a06wb", True),
         # invalida / incomplete postcodes
         ("N5", False),
         ("SO144 6WB", False),
@@ -600,7 +600,7 @@ def test_if_postcode_is_a_real_uk_postcode_normalises_before_checking_postcode(m
         ("N5     3EF", "N5 3EF"),
         ("N53EF   ", "N5 3EF"),
         ("n53Ef", "N5 3EF"),
-        ("n5 \u00A0 \t 3Ef", "N5 3EF"),
+        ("n5 \u00a0 \t 3Ef", "N5 3EF"),
         ("SO146WB", "SO14 6WB"),
         ("BFPO2", "BFPO 2"),
         ("BFPO232", "BFPO 232"),

tests/notifications_utils/test_recipient_csv.py

@@ -879,14 +879,14 @@ def test_ignores_spaces_and_case_in_placeholders(key, expected):
         ("\n", None),  # newline
         ("\r", None),  # carriage return
         ("\t", None),  # tab
-        ("\u180E", "MONGOLIAN VOWEL SEPARATOR"),
-        ("\u200B", "ZERO WIDTH SPACE"),
-        ("\u200C", "ZERO WIDTH NON-JOINER"),
-        ("\u200D", "ZERO WIDTH JOINER"),
+        ("\u180e", "MONGOLIAN VOWEL SEPARATOR"),
+        ("\u200b", "ZERO WIDTH SPACE"),
+        ("\u200c", "ZERO WIDTH NON-JOINER"),
+        ("\u200d", "ZERO WIDTH JOINER"),
         ("\u2060", "WORD JOINER"),
-        ("\uFEFF", "ZERO WIDTH NO-BREAK SPACE"),
+        ("\ufeff", "ZERO WIDTH NO-BREAK SPACE"),
         # all the things
-        (" \n\r\t\u000A\u000D\u180E\u200B\u200C\u200D\u2060\uFEFF", None),
+        (" \n\r\t\u000a\u000d\u180e\u200b\u200c\u200d\u2060\ufeff", None),
     ],
 )
 def test_ignores_leading_whitespace_in_file(character, name):

tests/notifications_utils/test_recipient_validation.py

@@ -339,7 +339,7 @@ def test_validate_email_address_accepts_valid(email_address):
         " email@domain.com ",
         "\temail@domain.com",
         "\temail@domain.com\n",
-        "\u200Bemail@domain.com\u200B",
+        "\u200bemail@domain.com\u200b",
     ],
 )
 def test_validate_email_address_strips_whitespace(email):

tests/notifications_utils/test_sanitise_text.py

@@ -17,7 +17,7 @@ params, ids = zip(
         ("…", "..."),
         "compatibility transform unicode char (HORIZONTAL ELLIPSIS (U+2026)",
     ),
-    (("\u200B", ""), "compatibility transform unicode char (ZERO WIDTH SPACE (U+200B)"),
+    (("\u200b", ""), "compatibility transform unicode char (ZERO WIDTH SPACE (U+200B)"),
     (
         ("‘", "'"),
         "compatibility transform unicode char (LEFT SINGLE QUOTATION MARK (U+2018)",