From b553ea4c77f2461258bc3136c37b8e670988cdc6 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Wed, 7 Dec 2022 14:55:05 -0500
Subject: [PATCH 01/46] Update uses of encryption.encrypt to more accurate
 encryption.sign

---
 Pipfile                                       |   2 +-
 Pipfile.lock                                  |  48 ++++++--
 app/celery/letters_pdf_tasks.py               |   4 +-
 app/celery/service_callback_tasks.py          |   8 +-
 app/celery/tasks.py                           |  10 +-
 app/models.py                                 |  20 +--
 app/v2/notifications/post_notifications.py    |   2 +-
 .../celery/test_process_ses_receipts_tasks.py |   2 +-
 tests/app/celery/test_scheduled_tasks.py      |   6 +-
 .../app/celery/test_service_callback_tasks.py |   4 +-
 tests/app/celery/test_tasks.py                | 116 +++++++++---------
 .../app/dao/test_service_callback_api_dao.py  |   4 +-
 tests/app/dao/test_service_inbound_api_dao.py |   4 +-
 tests/app/test_model.py                       |   4 +-
 14 files changed, 133 insertions(+), 101 deletions(-)

diff --git a/Pipfile b/Pipfile
index dc8437e5f..f1120001a 100644
--- a/Pipfile
+++ b/Pipfile
@@ -58,7 +58,7 @@ werkzeug = "~=2.1.1"
 # gds metrics packages
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
-notifications-utils = {editable = true, ref = "37ae9753c050851453d072994fb03b1415601716", git = "https://github.com/GSA/notifications-utils"}
+notifications-utils = {editable = true, ref = "723cd96915310f12bf5c02df3fed44e7794dccfd", git = "https://github.com/GSA/notifications-utils"}
 
 [dev-packages]
 flake8 = "==4.0.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index 9a64eced5..e3481cbc4 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "d652255c8ca6f6cb96778b0159228628816a114f867510bd4ce13a9e9692b101"
+            "sha256": "4e91ab05773366a04e60924ffca46e9157515716e6a98c4cc43097097713e651"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -275,6 +275,38 @@
             "index": "pypi",
             "version": "==0.4.4"
         },
+        "cryptography": {
+            "hashes": [
+                "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd",
+                "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db",
+                "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290",
+                "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744",
+                "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb",
+                "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d",
+                "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70",
+                "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b",
+                "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876",
+                "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083",
+                "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6",
+                "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1",
+                "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00",
+                "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b",
+                "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b",
+                "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285",
+                "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9",
+                "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0",
+                "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d",
+                "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2",
+                "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8",
+                "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee",
+                "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b",
+                "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7",
+                "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353",
+                "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==38.0.4"
+        },
         "defusedxml": {
             "hashes": [
                 "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69",
@@ -710,7 +742,7 @@
         "notifications-utils": {
             "editable": true,
             "git": "https://github.com/GSA/notifications-utils",
-            "ref": "37ae9753c050851453d072994fb03b1415601716"
+            "ref": "723cd96915310f12bf5c02df3fed44e7794dccfd"
         },
         "orderedset": {
             "hashes": [
@@ -751,11 +783,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:535c29c31216c77302877d5120aef6c94ff573748a5b5ca5b1b1f76f5e700c73",
-                "sha256:ced598b222f6f4029c0800cefaa6a17373fb580cd093223003475ce32805c35b"
+                "sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63",
+                "sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.33"
+            "version": "==3.0.36"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -1003,10 +1035,10 @@
         },
         "redis": {
             "hashes": [
-                "sha256:30c07511627a4c5c4d970e060000772f323174f75e745a26938319817ead7a12",
-                "sha256:46652271dc7525cd5a9667e5b0ca983c848c75b2b8f7425403395bb8379dcf25"
+                "sha256:7b8c87d19c45d3f1271b124858d2a5c13160c4e74d4835e28273400fa34d5228",
+                "sha256:cae3ee5d1f57d8caf534cd8764edf3163c77e073bdd74b6f54a87ffafdc5e7d9"
             ],
-            "version": "==4.3.5"
+            "version": "==4.4.0"
         },
         "requests": {
             "hashes": [
diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py
index cd7a18021..754501567 100644
--- a/app/celery/letters_pdf_tasks.py
+++ b/app/celery/letters_pdf_tasks.py
@@ -79,7 +79,7 @@ def get_pdf_for_templated_letter(self, notification_id):
             'key_type': notification.key_type
         }
 
-        encrypted_data = encryption.encrypt(letter_data)
+        encrypted_data = encryption.sign(letter_data)
 
         notify_celery.send_task(
             name=TaskNames.CREATE_PDF_FOR_TEMPLATED_LETTER,
@@ -328,7 +328,7 @@ def sanitise_letter(self, filename):
 
 @notify_celery.task(bind=True, name='process-sanitised-letter', max_retries=15, default_retry_delay=300)
 def process_sanitised_letter(self, sanitise_data):
-    letter_details = encryption.decrypt(sanitise_data)
+    letter_details = encryption.verify_signature(sanitise_data)
 
     filename = letter_details['filename']
     notification_id = letter_details['notification_id']
diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py
index 0e81d38d0..d06f1a173 100644
--- a/app/celery/service_callback_tasks.py
+++ b/app/celery/service_callback_tasks.py
@@ -12,7 +12,7 @@ from app.utils import DATETIME_FORMAT
 def send_delivery_status_to_service(
     self, notification_id, encrypted_status_update
 ):
-    status_update = encryption.decrypt(encrypted_status_update)
+    status_update = encryption.verify_signature(encrypted_status_update)
 
     data = {
         "id": str(notification_id),
@@ -38,7 +38,7 @@ def send_delivery_status_to_service(
 
 @notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
 def send_complaint_to_service(self, complaint_data):
-    complaint = encryption.decrypt(complaint_data)
+    complaint = encryption.verify_signature(complaint_data)
 
     data = {
         "notification_id": complaint['notification_id'],
@@ -125,7 +125,7 @@ def create_delivery_status_callback_data(notification, service_callback_api):
         "template_id": str(notification.template_id),
         "template_version": notification.template_version,
     }
-    return encryption.encrypt(data)
+    return encryption.sign(data)
 
 
 def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
@@ -138,4 +138,4 @@ def create_complaint_callback_data(complaint, notification, service_callback_api
         "service_callback_api_url": service_callback_api.url,
         "service_callback_api_bearer_token": service_callback_api.bearer_token,
     }
-    return encryption.encrypt(data)
+    return encryption.sign(data)
diff --git a/app/celery/tasks.py b/app/celery/tasks.py
index dd37229f1..ca8ad8957 100644
--- a/app/celery/tasks.py
+++ b/app/celery/tasks.py
@@ -125,7 +125,7 @@ def get_recipient_csv_and_template_and_sender_id(job):
 
 def process_row(row, template, job, service, sender_id=None):
     template_type = template.template_type
-    encrypted = encryption.encrypt({
+    encrypted = encryption.sign({
         'template': str(template.id),
         'template_version': job.template_version,
         'job': str(job.id),
@@ -183,7 +183,7 @@ def save_sms(self,
              notification_id,
              encrypted_notification,
              sender_id=None):
-    notification = encryption.decrypt(encrypted_notification)
+    notification = encryption.verify_signature(encrypted_notification)
     service = SerialisedService.from_id(service_id)
     template = SerialisedTemplate.from_id_and_service_id(
         notification['template'],
@@ -241,7 +241,7 @@ def save_email(self,
                notification_id,
                encrypted_notification,
                sender_id=None):
-    notification = encryption.decrypt(encrypted_notification)
+    notification = encryption.verify_signature(encrypted_notification)
 
     service = SerialisedService.from_id(service_id)
     template = SerialisedTemplate.from_id_and_service_id(
@@ -298,7 +298,7 @@ def save_api_sms(self, encrypted_notification):
 
 
 def save_api_email_or_sms(self, encrypted_notification):
-    notification = encryption.decrypt(encrypted_notification)
+    notification = encryption.verify_signature(encrypted_notification)
     service = SerialisedService.from_id(notification['service_id'])
     q = QueueNames.SEND_EMAIL if notification['notification_type'] == EMAIL_TYPE else QueueNames.SEND_SMS
     provider_task = provider_tasks.deliver_email if notification['notification_type'] == EMAIL_TYPE \
@@ -348,7 +348,7 @@ def save_letter(
         notification_id,
         encrypted_notification,
 ):
-    notification = encryption.decrypt(encrypted_notification)
+    notification = encryption.verify_signature(encrypted_notification)
 
     postal_address = PostalAddress.from_personalisation(
         InsensitiveDict(notification['personalisation'])
diff --git a/app/models.py b/app/models.py
index e2c5c5bed..bb91bcc79 100644
--- a/app/models.py
+++ b/app/models.py
@@ -738,13 +738,13 @@ class ServiceInboundApi(db.Model, Versioned):
     @property
     def bearer_token(self):
         if self._bearer_token:
-            return encryption.decrypt(self._bearer_token)
+            return encryption.verify_signature(self._bearer_token)
         return None
 
     @bearer_token.setter
     def bearer_token(self, bearer_token):
         if bearer_token:
-            self._bearer_token = encryption.encrypt(str(bearer_token))
+            self._bearer_token = encryption.sign(str(bearer_token))
 
     def serialize(self):
         return {
@@ -777,13 +777,13 @@ class ServiceCallbackApi(db.Model, Versioned):
     @property
     def bearer_token(self):
         if self._bearer_token:
-            return encryption.decrypt(self._bearer_token)
+            return encryption.verify_signature(self._bearer_token)
         return None
 
     @bearer_token.setter
     def bearer_token(self, bearer_token):
         if bearer_token:
-            self._bearer_token = encryption.encrypt(str(bearer_token))
+            self._bearer_token = encryption.sign(str(bearer_token))
 
     def serialize(self):
         return {
@@ -834,13 +834,13 @@ class ApiKey(db.Model, Versioned):
     @property
     def secret(self):
         if self._secret:
-            return encryption.decrypt(self._secret)
+            return encryption.verify_signature(self._secret)
         return None
 
     @secret.setter
     def secret(self, secret):
         if secret:
-            self._secret = encryption.encrypt(str(secret))
+            self._secret = encryption.sign(str(secret))
 
 
 KEY_TYPE_NORMAL = 'normal'
@@ -1512,12 +1512,12 @@ class Notification(db.Model):
     @property
     def personalisation(self):
         if self._personalisation:
-            return encryption.decrypt(self._personalisation)
+            return encryption.verify_signature(self._personalisation)
         return {}
 
     @personalisation.setter
     def personalisation(self, personalisation):
-        self._personalisation = encryption.encrypt(personalisation or {})
+        self._personalisation = encryption.sign(personalisation or {})
 
     def completed_at(self):
         if self.status in NOTIFICATION_STATUS_TYPES_COMPLETED:
@@ -1960,11 +1960,11 @@ class InboundSms(db.Model):
 
     @property
     def content(self):
-        return encryption.decrypt(self._content)
+        return encryption.verify_signature(self._content)
 
     @content.setter
     def content(self, content):
-        self._content = encryption.encrypt(content)
+        self._content = encryption.sign(content)
 
     def serialize(self):
         return {
diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py
index 756c4d846..c6402e735 100644
--- a/app/v2/notifications/post_notifications.py
+++ b/app/v2/notifications/post_notifications.py
@@ -300,7 +300,7 @@ def save_email_or_sms_to_queue(
         "status": NOTIFICATION_CREATED,
         "created_at": datetime.utcnow().strftime(DATETIME_FORMAT),
     }
-    encrypted = encryption.encrypt(
+    encrypted = encryption.sign(
         data
     )
 
diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py
index a90f9f6f8..7311f4171 100644
--- a/tests/app/celery/test_process_ses_receipts_tasks.py
+++ b/tests/app/celery/test_process_ses_receipts_tasks.py
@@ -331,7 +331,7 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email
     response = ses_complaint_callback()
     assert process_ses_results(response)
     assert send_mock.call_count == 1
-    assert encryption.decrypt(send_mock.call_args[0][0][0]) == {
+    assert encryption.verify_signature(send_mock.call_args[0][0][0]) == {
         'complaint_date': '2018-06-05T13:59:58.000000Z',
         'complaint_id': str(Complaint.query.one().id),
         'notification_id': str(notification.id),
diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py
index 55298af2c..36aa7e2db 100644
--- a/tests/app/celery/test_scheduled_tasks.py
+++ b/tests/app/celery/test_scheduled_tasks.py
@@ -465,7 +465,7 @@ def test_check_for_missing_rows_in_completed_jobs_ignores_old_and_new_jobs(
 ):
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_email'), {"sender_id": None}))
-    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.encryption.sign', return_value="something_encrypted")
     process_row = mocker.patch('app.celery.scheduled_tasks.process_row')
 
     job = create_job(
@@ -485,7 +485,7 @@ def test_check_for_missing_rows_in_completed_jobs_ignores_old_and_new_jobs(
 def test_check_for_missing_rows_in_completed_jobs(mocker, sample_email_template):
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_email'), {"sender_id": None}))
-    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.encryption.sign', return_value="something_encrypted")
     process_row = mocker.patch('app.celery.scheduled_tasks.process_row')
 
     job = create_job(template=sample_email_template,
@@ -506,7 +506,7 @@ def test_check_for_missing_rows_in_completed_jobs_calls_save_email(mocker, sampl
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_email'), {'sender_id': None}))
     save_email_task = mocker.patch('app.celery.tasks.save_email.apply_async')
-    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.encryption.sign', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value='uuid')
 
     job = create_job(template=sample_email_template,
diff --git a/tests/app/celery/test_service_callback_tasks.py b/tests/app/celery/test_service_callback_tasks.py
index d5249c678..1996d2239 100644
--- a/tests/app/celery/test_service_callback_tasks.py
+++ b/tests/app/celery/test_service_callback_tasks.py
@@ -191,7 +191,7 @@ def _set_up_data_for_status_update(callback_api, notification):
         "template_id": str(notification.template_id),
         "template_version": notification.template_version,
     }
-    encrypted_status_update = encryption.encrypt(data)
+    encrypted_status_update = encryption.sign(data)
     return encrypted_status_update
 
 
@@ -205,5 +205,5 @@ def _set_up_data_for_complaint(callback_api, complaint, notification):
         "service_callback_api_url": callback_api.url,
         "service_callback_api_bearer_token": callback_api.bearer_token,
     }
-    obscured_status_update = encryption.encrypt(data)
+    obscured_status_update = encryption.sign(data)
     return obscured_status_update
diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py
index 4b3fe9bdb..9556ac0ea 100644
--- a/tests/app/celery/test_tasks.py
+++ b/tests/app/celery/test_tasks.py
@@ -109,7 +109,7 @@ def test_should_process_sms_job(sample_job, mocker):
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('sms'), {'sender_id': None}))
     mocker.patch('app.celery.tasks.save_sms.apply_async')
-    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.encryption.sign', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(sample_job.id)
@@ -117,11 +117,11 @@ def test_should_process_sms_job(sample_job, mocker):
         service_id=str(sample_job.service.id),
         job_id=str(sample_job.id)
     )
-    assert encryption.encrypt.call_args[0][0]['to'] == '+441234123123'
-    assert encryption.encrypt.call_args[0][0]['template'] == str(sample_job.template.id)
-    assert encryption.encrypt.call_args[0][0]['template_version'] == sample_job.template.version
-    assert encryption.encrypt.call_args[0][0]['personalisation'] == {'phonenumber': '+441234123123'}
-    assert encryption.encrypt.call_args[0][0]['row_number'] == 0
+    assert encryption.sign.call_args[0][0]['to'] == '+441234123123'
+    assert encryption.sign.call_args[0][0]['template'] == str(sample_job.template.id)
+    assert encryption.sign.call_args[0][0]['template_version'] == sample_job.template.version
+    assert encryption.sign.call_args[0][0]['personalisation'] == {'phonenumber': '+441234123123'}
+    assert encryption.sign.call_args[0][0]['row_number'] == 0
     tasks.save_sms.apply_async.assert_called_once_with(
         (str(sample_job.service_id),
          "uuid",
@@ -137,7 +137,7 @@ def test_should_process_sms_job_with_sender_id(sample_job, mocker, fake_uuid):
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('sms'), {'sender_id': fake_uuid}))
     mocker.patch('app.celery.tasks.save_sms.apply_async')
-    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.encryption.sign', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(sample_job.id, sender_id=fake_uuid)
@@ -211,7 +211,7 @@ def test_should_process_job_if_send_limits_are_not_exceeded(
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_email'), {"sender_id": None}))
     mocker.patch('app.celery.tasks.save_email.apply_async')
-    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.encryption.sign', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
     mocker.patch('app.celery.tasks.check_service_over_daily_message_limit', return_value=0)
     process_job(job.id)
@@ -255,7 +255,7 @@ def test_should_process_email_job(email_job_with_placeholders, mocker):
     """
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3', return_value=(email_csv, {"sender_id": None}))
     mocker.patch('app.celery.tasks.save_email.apply_async')
-    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.encryption.sign', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(email_job_with_placeholders.id)
@@ -264,10 +264,10 @@ def test_should_process_email_job(email_job_with_placeholders, mocker):
         service_id=str(email_job_with_placeholders.service.id),
         job_id=str(email_job_with_placeholders.id)
     )
-    assert encryption.encrypt.call_args[0][0]['to'] == 'test@test.com'
-    assert encryption.encrypt.call_args[0][0]['template'] == str(email_job_with_placeholders.template.id)
-    assert encryption.encrypt.call_args[0][0]['template_version'] == email_job_with_placeholders.template.version
-    assert encryption.encrypt.call_args[0][0]['personalisation'] == {'emailaddress': 'test@test.com', 'name': 'foo'}
+    assert encryption.sign.call_args[0][0]['to'] == 'test@test.com'
+    assert encryption.sign.call_args[0][0]['template'] == str(email_job_with_placeholders.template.id)
+    assert encryption.sign.call_args[0][0]['template_version'] == email_job_with_placeholders.template.version
+    assert encryption.sign.call_args[0][0]['personalisation'] == {'emailaddress': 'test@test.com', 'name': 'foo'}
     tasks.save_email.apply_async.assert_called_once_with(
         (
             str(email_job_with_placeholders.service_id),
@@ -287,7 +287,7 @@ def test_should_process_email_job_with_sender_id(email_job_with_placeholders, mo
     """
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3', return_value=(email_csv, {"sender_id": fake_uuid}))
     mocker.patch('app.celery.tasks.save_email.apply_async')
-    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.encryption.sign', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(email_job_with_placeholders.id, sender_id=fake_uuid)
@@ -341,7 +341,7 @@ def test_should_process_all_sms_job(sample_job_with_placeholdered_template,
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_sms'), {"sender_id": None}))
     mocker.patch('app.celery.tasks.save_sms.apply_async')
-    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
+    mocker.patch('app.encryption.sign', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(sample_job_with_placeholdered_template.id)
@@ -350,11 +350,11 @@ def test_should_process_all_sms_job(sample_job_with_placeholdered_template,
         service_id=str(sample_job_with_placeholdered_template.service.id),
         job_id=str(sample_job_with_placeholdered_template.id)
     )
-    assert encryption.encrypt.call_args[0][0]['to'] == '+441234123120'
-    assert encryption.encrypt.call_args[0][0]['template'] == str(sample_job_with_placeholdered_template.template.id)
-    assert encryption.encrypt.call_args[0][0][
+    assert encryption.sign.call_args[0][0]['to'] == '+441234123120'
+    assert encryption.sign.call_args[0][0]['template'] == str(sample_job_with_placeholdered_template.template.id)
+    assert encryption.sign.call_args[0][0][
                'template_version'] == sample_job_with_placeholdered_template.template.version  # noqa
-    assert encryption.encrypt.call_args[0][0]['personalisation'] == {'phonenumber': '+441234123120', 'name': 'chris'}
+    assert encryption.sign.call_args[0][0]['personalisation'] == {'phonenumber': '+441234123120', 'name': 'chris'}
     assert tasks.save_sms.apply_async.call_count == 10
     job = jobs_dao.dao_get_job_by_id(sample_job_with_placeholdered_template.id)
     assert job.job_status == 'finished'
@@ -374,7 +374,7 @@ def test_should_process_all_sms_job(sample_job_with_placeholdered_template,
 def test_process_row_sends_letter_task(template_type, research_mode, expected_function, expected_queue, mocker):
     mocker.patch('app.celery.tasks.create_uuid', return_value='noti_uuid')
     task_mock = mocker.patch('app.celery.tasks.{}.apply_async'.format(expected_function))
-    encrypt_mock = mocker.patch('app.celery.tasks.encryption.encrypt')
+    encrypt_mock = mocker.patch('app.celery.tasks.encryption.sign')
     template = Mock(id='template_id', template_type=template_type)
     job = Mock(id='job_id', template_version='temp_vers')
     service = Mock(id='service_id', research_mode=research_mode)
@@ -417,7 +417,7 @@ def test_process_row_sends_letter_task(template_type, research_mode, expected_fu
 def test_process_row_when_sender_id_is_provided(mocker, fake_uuid):
     mocker.patch('app.celery.tasks.create_uuid', return_value='noti_uuid')
     task_mock = mocker.patch('app.celery.tasks.save_sms.apply_async')
-    encrypt_mock = mocker.patch('app.celery.tasks.encryption.encrypt')
+    encrypt_mock = mocker.patch('app.celery.tasks.encryption.sign')
     template = Mock(id='template_id', template_type=SMS_TYPE)
     job = Mock(id='job_id', template_version='temp_vers')
     service = Mock(id='service_id', research_mode=False)
@@ -460,7 +460,7 @@ def test_should_send_template_to_correct_sms_task_and_persist(sample_template_wi
     save_sms(
         sample_template_with_placeholders.service_id,
         uuid.uuid4(),
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -473,7 +473,7 @@ def test_should_send_template_to_correct_sms_task_and_persist(sample_template_wi
     assert not persisted_notification.sent_by
     assert not persisted_notification.job_id
     assert persisted_notification.personalisation == {'name': 'Jo'}
-    assert persisted_notification._personalisation == encryption.encrypt({"name": "Jo"})
+    assert persisted_notification._personalisation == encryption.sign({"name": "Jo"})
     assert persisted_notification.notification_type == 'sms'
     mocked_deliver_sms.assert_called_once_with(
         [str(persisted_notification.id)],
@@ -495,7 +495,7 @@ def test_should_put_save_sms_task_in_research_mode_queue_if_research_mode_servic
     save_sms(
         template.service_id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
     persisted_notification = Notification.query.one()
     provider_tasks.deliver_sms.apply_async.assert_called_once_with(
@@ -514,7 +514,7 @@ def test_should_save_sms_if_restricted_service_and_valid_number(notify_db_sessio
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     notification_id = uuid.uuid4()
-    encrypt_notification = encryption.encrypt(notification)
+    encrypt_notification = encryption.sign(notification)
     save_sms(
         service.id,
         notification_id,
@@ -550,7 +550,7 @@ def test_save_email_should_save_default_email_reply_to_text_on_notification(noti
     save_email(
         service.id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -568,7 +568,7 @@ def test_save_sms_should_save_default_smm_sender_notification_reply_to_text_on(n
     save_sms(
         service.id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -587,7 +587,7 @@ def test_should_not_save_sms_if_restricted_service_and_invalid_number(notify_db_
     save_sms(
         service.id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
     assert provider_tasks.deliver_sms.apply_async.called is False
     assert Notification.query.count() == 0
@@ -603,7 +603,7 @@ def test_should_not_save_email_if_restricted_service_and_invalid_email_address(n
     save_email(
         service.id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
 
     assert Notification.query.count() == 0
@@ -625,7 +625,7 @@ def test_should_put_save_email_task_in_research_mode_queue_if_research_mode_serv
     save_email(
         template.service_id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -648,7 +648,7 @@ def test_should_save_sms_template_to_and_persist_with_job_id(sample_job, mocker)
     save_sms(
         sample_job.service.id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
     persisted_notification = Notification.query.one()
     assert persisted_notification.to == '+447234123123'
@@ -685,7 +685,7 @@ def test_should_not_save_sms_if_team_key_and_recipient_not_in_team(notify_db_ses
     save_sms(
         service.id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
     assert provider_tasks.deliver_sms.apply_async.called is False
     assert Notification.query.count() == 0
@@ -708,7 +708,7 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh
         save_email(
             sample_email_template_with_placeholders.service_id,
             notification_id,
-            encryption.encrypt(notification),
+            encryption.sign(notification),
         )
 
     persisted_notification = Notification.query.one()
@@ -721,7 +721,7 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh
     assert not persisted_notification.sent_by
     assert persisted_notification.job_row_number == 1
     assert persisted_notification.personalisation == {'name': 'Jo'}
-    assert persisted_notification._personalisation == encryption.encrypt({"name": "Jo"})
+    assert persisted_notification._personalisation == encryption.sign({"name": "Jo"})
     assert persisted_notification.api_key_id is None
     assert persisted_notification.key_type == KEY_TYPE_NORMAL
     assert persisted_notification.notification_type == 'email'
@@ -747,7 +747,7 @@ def test_save_email_should_use_template_version_from_job_not_latest(sample_email
     save_email(
         sample_email_template.service_id,
         uuid.uuid4(),
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -773,7 +773,7 @@ def test_should_use_email_template_subject_placeholders(sample_email_template_wi
     save_email(
         sample_email_template_with_placeholders.service_id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
     persisted_notification = Notification.query.one()
     assert persisted_notification.to == 'my_email@my_email.com'
@@ -802,7 +802,7 @@ def test_save_email_uses_the_reply_to_text_when_provided(sample_email_template,
     save_email(
         sample_email_template.service_id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
         sender_id=other_email_reply_to.id,
     )
     persisted_notification = Notification.query.one()
@@ -821,7 +821,7 @@ def test_save_email_uses_the_default_reply_to_text_if_sender_id_is_none(sample_e
     save_email(
         sample_email_template.service_id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
         sender_id=None,
     )
     persisted_notification = Notification.query.one()
@@ -839,7 +839,7 @@ def test_should_use_email_template_and_persist_without_personalisation(sample_em
     save_email(
         sample_email_template.service_id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
     persisted_notification = Notification.query.one()
     assert persisted_notification.to == 'my_email@my_email.com'
@@ -870,7 +870,7 @@ def test_save_sms_should_go_to_retry_queue_if_database_errors(sample_template, m
         save_sms(
             sample_template.service_id,
             notification_id,
-            encryption.encrypt(notification),
+            encryption.sign(notification),
         )
     assert provider_tasks.deliver_sms.apply_async.called is False
     tasks.save_sms.retry.assert_called_with(exc=expected_exception, queue="retry-tasks")
@@ -893,7 +893,7 @@ def test_save_email_should_go_to_retry_queue_if_database_errors(sample_email_tem
         save_email(
             sample_email_template.service_id,
             notification_id,
-            encryption.encrypt(notification),
+            encryption.sign(notification),
         )
     assert not provider_tasks.deliver_email.apply_async.called
     tasks.save_email.retry.assert_called_with(exc=expected_exception, queue="retry-tasks")
@@ -911,7 +911,7 @@ def test_save_email_does_not_send_duplicate_and_does_not_put_in_retry_queue(samp
     save_email(
         sample_notification.service_id,
         notification_id,
-        encryption.encrypt(json),
+        encryption.sign(json),
     )
     assert Notification.query.count() == 1
     assert not deliver_email.called
@@ -928,7 +928,7 @@ def test_save_sms_does_not_send_duplicate_and_does_not_put_in_retry_queue(sample
     save_sms(
         sample_notification.service_id,
         notification_id,
-        encryption.encrypt(json),
+        encryption.sign(json),
     )
     assert Notification.query.count() == 1
     assert not deliver_sms.called
@@ -998,7 +998,7 @@ def test_save_letter_saves_letter_to_database(
     save_letter(
         job.service_id,
         notification_id,
-        encryption.encrypt(notification_json),
+        encryption.sign(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1042,7 +1042,7 @@ def test_save_letter_saves_letter_to_database_with_correct_postage(
     save_letter(
         letter_job.service_id,
         notification_id,
-        encryption.encrypt(notification_json),
+        encryption.sign(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1075,7 +1075,7 @@ def test_save_letter_saves_letter_to_database_with_correct_client_reference(
     save_letter(
         letter_job.service_id,
         notification_id,
-        encryption.encrypt(notification_json),
+        encryption.sign(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1100,7 +1100,7 @@ def test_save_letter_saves_letter_to_database_with_formatted_postcode(mocker, no
     save_letter(
         letter_job.service_id,
         notification_id,
-        encryption.encrypt(notification_json),
+        encryption.sign(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1139,7 +1139,7 @@ def test_save_letter_saves_letter_to_database_right_reply_to(mocker, notify_db_s
     save_letter(
         job.service_id,
         notification_id,
-        encryption.encrypt(notification_json),
+        encryption.sign(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1201,7 +1201,7 @@ def test_save_letter_uses_template_reply_to_text(mocker, notify_db_session):
     save_letter(
         job.service_id,
         uuid.uuid4(),
-        encryption.encrypt(notification_json),
+        encryption.sign(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1219,7 +1219,7 @@ def test_save_sms_uses_sms_sender_reply_to_text(mocker, notify_db_session):
     save_sms(
         service.id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -1238,7 +1238,7 @@ def test_save_sms_uses_non_default_sms_sender_reply_to_text_if_provided(mocker,
     save_sms(
         service.id,
         notification_id,
-        encryption.encrypt(notification),
+        encryption.sign(notification),
         sender_id=new_sender.id,
     )
 
@@ -1276,7 +1276,7 @@ def test_save_letter_sets_delivered_letters_as_pdf_permission_in_research_mode_i
         save_letter(
             sample_letter_job.service_id,
             notification_id,
-            encryption.encrypt(notification_json),
+            encryption.sign(notification_json),
         )
 
     notification = Notification.query.filter(Notification.id == notification_id).one()
@@ -1314,7 +1314,7 @@ def test_save_letter_calls_create_fake_response_for_letters_in_research_mode_on_
         save_letter(
             sample_letter_job.service_id,
             notification_id,
-            encryption.encrypt(notification_json),
+            encryption.sign(notification_json),
         )
 
     mock_create_fake_letter_response_file.assert_called_once_with(
@@ -1345,7 +1345,7 @@ def test_save_letter_calls_get_pdf_for_templated_letter_task_not_in_research(
     save_letter(
         sample_letter_job.service_id,
         notification_id,
-        encryption.encrypt(notification_json),
+        encryption.sign(notification_json),
     )
 
     assert mock_create_letters_pdf.called
@@ -1843,7 +1843,7 @@ def test_save_api_email_or_sms(mocker, sample_service, notification_type):
         data.update({"to": "+447700900855"})
         expected_queue = QueueNames.SEND_SMS
 
-    encrypted = encryption.encrypt(
+    encrypted = encryption.sign(
         data
     )
 
@@ -1890,7 +1890,7 @@ def test_save_api_email_dont_retry_if_notification_already_exists(sample_service
         data.update({"to": "+447700900855"})
         expected_queue = QueueNames.SEND_SMS
 
-    encrypted = encryption.encrypt(
+    encrypted = encryption.sign(
         data
     )
     assert len(Notification.query.all()) == 0
@@ -1953,7 +1953,7 @@ def test_save_tasks_use_cached_service_and_template(
         task_function(
             service.id,
             uuid.uuid4(),
-            encryption.encrypt(notification),
+            encryption.sign(notification),
         )
 
     # We talk to the database once for the service and once for the
@@ -1995,7 +1995,7 @@ def test_save_api_tasks_use_cache(
     api_key = create_api_key(service=template.service)
 
     def create_encrypted_notification():
-        return encryption.encrypt({
+        return encryption.sign({
             "to": recipient,
             "id": str(uuid.uuid4()),
             "template_id": str(template.id),
diff --git a/tests/app/dao/test_service_callback_api_dao.py b/tests/app/dao/test_service_callback_api_dao.py
index 504e470d8..573cf5b72 100644
--- a/tests/app/dao/test_service_callback_api_dao.py
+++ b/tests/app/dao/test_service_callback_api_dao.py
@@ -40,7 +40,7 @@ def test_save_service_callback_api(sample_service):
     assert versioned.service_id == sample_service.id
     assert versioned.updated_by_id == sample_service.users[0].id
     assert versioned.url == "https://some_service/callback_endpoint"
-    assert encryption.decrypt(versioned._bearer_token) == "some_unique_string"
+    assert encryption.verify_signature(versioned._bearer_token) == "some_unique_string"
     assert versioned.updated_at is None
     assert versioned.version == 1
 
@@ -140,7 +140,7 @@ def test_update_service_callback_api(sample_service):
         assert x.id is not None
         assert x.service_id == sample_service.id
         assert x.updated_by_id == sample_service.users[0].id
-        assert encryption.decrypt(x._bearer_token) == "some_unique_string"
+        assert encryption.verify_signature(x._bearer_token) == "some_unique_string"
 
 
 def test_get_service_callback_api(sample_service):
diff --git a/tests/app/dao/test_service_inbound_api_dao.py b/tests/app/dao/test_service_inbound_api_dao.py
index 26a750533..6f7f57af5 100644
--- a/tests/app/dao/test_service_inbound_api_dao.py
+++ b/tests/app/dao/test_service_inbound_api_dao.py
@@ -40,7 +40,7 @@ def test_save_service_inbound_api(sample_service):
     assert versioned.service_id == sample_service.id
     assert versioned.updated_by_id == sample_service.users[0].id
     assert versioned.url == "https://some_service/inbound_messages"
-    assert encryption.decrypt(versioned._bearer_token) == "some_unique_string"
+    assert encryption.verify_signature(versioned._bearer_token) == "some_unique_string"
     assert versioned.updated_at is None
     assert versioned.version == 1
 
@@ -97,7 +97,7 @@ def test_update_service_inbound_api(sample_service):
         assert x.id is not None
         assert x.service_id == sample_service.id
         assert x.updated_by_id == sample_service.users[0].id
-        assert encryption.decrypt(x._bearer_token) == "some_unique_string"
+        assert encryption.verify_signature(x._bearer_token) == "some_unique_string"
 
 
 def test_get_service_inbound_api(sample_service):
diff --git a/tests/app/test_model.py b/tests/app/test_model.py
index f8464a16e..57c4b40cd 100644
--- a/tests/app/test_model.py
+++ b/tests/app/test_model.py
@@ -156,7 +156,7 @@ def test_notification_personalisation_getter_returns_empty_dict_from_None():
 
 def test_notification_personalisation_getter_always_returns_empty_dict():
     noti = Notification()
-    noti._personalisation = encryption.encrypt({})
+    noti._personalisation = encryption.sign({})
     assert noti.personalisation == {}
 
 
@@ -168,7 +168,7 @@ def test_notification_personalisation_setter_always_sets_empty_dict(input_value)
     noti = Notification()
     noti.personalisation = input_value
 
-    assert noti._personalisation == encryption.encrypt({})
+    assert noti._personalisation == encryption.sign({})
 
 
 def test_notification_subject_is_none_for_sms(sample_service):

From 9969bd5304d0db900bef4114417dbdadb5a2bce3 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 8 Dec 2022 10:57:01 -0500
Subject: [PATCH 02/46] Update notifications_utils

---
 Pipfile      |  3 ++-
 Pipfile.lock | 38 +++++++++++++++++++-------------------
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/Pipfile b/Pipfile
index dc8437e5f..470d79012 100644
--- a/Pipfile
+++ b/Pipfile
@@ -58,7 +58,8 @@ werkzeug = "~=2.1.1"
 # gds metrics packages
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
-notifications-utils = {editable = true, ref = "37ae9753c050851453d072994fb03b1415601716", git = "https://github.com/GSA/notifications-utils"}
+notifications-utils = {editable = true, ref = "3096f2937b5e3b59a2e7a4b4ffb05b7fa1ab72fa", git = "https://github.com/GSA/notifications-utils"}
+packaging = "==21.3"
 
 [dev-packages]
 flake8 = "==4.0.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index 9a64eced5..c04ae2aa5 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "d652255c8ca6f6cb96778b0159228628816a114f867510bd4ce13a9e9692b101"
+            "sha256": "445280351af84a8b83d60a5f18715d7f19b9b221e618ff5084b243824ecec4e5"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -710,7 +710,7 @@
         "notifications-utils": {
             "editable": true,
             "git": "https://github.com/GSA/notifications-utils",
-            "ref": "37ae9753c050851453d072994fb03b1415601716"
+            "ref": "3096f2937b5e3b59a2e7a4b4ffb05b7fa1ab72fa"
         },
         "orderedset": {
             "hashes": [
@@ -731,15 +731,15 @@
                 "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
                 "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
             ],
-            "markers": "python_version >= '3.6'",
+            "index": "pypi",
             "version": "==21.3"
         },
         "phonenumbers": {
             "hashes": [
-                "sha256:07a95c2f178687fd1c3f722cf792b3d33e3a225ae71577e500c99b28544cd6d0",
-                "sha256:7cadfe900e833857500b7bafa3e5a7eddc3263eb66b66a767870b33e44665f92"
+                "sha256:0179f688d48c0e7e161eb7b9d86d587940af1f5174f97c1fdfd893c599c0d94a",
+                "sha256:884b26f775205261f4dc861371dce217c1661a4942fb3ec3624e290fb51869bf"
             ],
-            "version": "==8.13.1"
+            "version": "==8.13.2"
         },
         "prometheus-client": {
             "hashes": [
@@ -751,11 +751,11 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:535c29c31216c77302877d5120aef6c94ff573748a5b5ca5b1b1f76f5e700c73",
-                "sha256:ced598b222f6f4029c0800cefaa6a17373fb580cd093223003475ce32805c35b"
+                "sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63",
+                "sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305"
             ],
             "markers": "python_full_version >= '3.6.2'",
-            "version": "==3.0.33"
+            "version": "==3.0.36"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -1003,10 +1003,10 @@
         },
         "redis": {
             "hashes": [
-                "sha256:30c07511627a4c5c4d970e060000772f323174f75e745a26938319817ead7a12",
-                "sha256:46652271dc7525cd5a9667e5b0ca983c848c75b2b8f7425403395bb8379dcf25"
+                "sha256:7b8c87d19c45d3f1271b124858d2a5c13160c4e74d4835e28273400fa34d5228",
+                "sha256:cae3ee5d1f57d8caf534cd8764edf3163c77e073bdd74b6f54a87ffafdc5e7d9"
             ],
-            "version": "==4.3.5"
+            "version": "==4.4.0"
         },
         "requests": {
             "hashes": [
@@ -1782,7 +1782,7 @@
                 "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
                 "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
             ],
-            "markers": "python_version >= '3.6'",
+            "index": "pypi",
             "version": "==21.3"
         },
         "pbr": {
@@ -1811,19 +1811,19 @@
         },
         "pip-audit": {
             "hashes": [
-                "sha256:a99f825ee431a89b89981c4e9e6eaacff5af3233783f2f5d79fe03306dc378ce",
-                "sha256:f87b37b6db5317a3f5ecebc202b5d4401958b5e4bd05b39d7b230bdc6f63c34b"
+                "sha256:40876d6ad6adf5ac64fba1ecef034db9804c35c2c18f7c22cb7b11b382c10df9",
+                "sha256:a45540ab0c5a9311315ca42c78fa8f72cf3598d5968a67d883d2d6194eda598c"
             ],
             "index": "pypi",
-            "version": "==2.4.7"
+            "version": "==2.4.8"
         },
         "pip-requirements-parser": {
             "hashes": [
-                "sha256:22fa213a987913385b2484d5698ecfa1d9cf4154978cdf929085548af55355b0",
-                "sha256:8c2a6f8e091ac2693824a5ef4e3b250226e34f74a20a91a87b9ab0714b47788f"
+                "sha256:5159b8a9485a5a0d0254a29c9bd8b8ce66db2eb9a1a0244c64dfce43f7f2ac90",
+                "sha256:aa664781e69f82ecefda542d5210aa5b0f6533be95aea39a5453943db09359ed"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==31.2.0"
+            "version": "==32.0.0"
         },
         "pluggy": {
             "hashes": [

From d7cd23b5620ed6ac9791afa2e9d00567d6fe6610 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 8 Dec 2022 13:46:07 -0500
Subject: [PATCH 03/46] Update certifi

---
 Pipfile      |  2 +-
 Pipfile.lock | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Pipfile b/Pipfile
index 470d79012..c6c5e1f2a 100644
--- a/Pipfile
+++ b/Pipfile
@@ -20,7 +20,7 @@ boto3 = "==1.23.8"
 botocore = "==1.26.8"
 cachetools = "==5.1.0"
 celery = {version = "==5.2.7", extras = ["redis"]}
-certifi = "==2022.5.18.1"
+certifi = ">=2022.12.7"
 cffi = "==1.15.0"
 charset-normalizer = "==2.0.12"
 click = "==8.1.3"
diff --git a/Pipfile.lock b/Pipfile.lock
index c04ae2aa5..84dfc2b87 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "445280351af84a8b83d60a5f18715d7f19b9b221e618ff5084b243824ecec4e5"
+            "sha256": "85a9f8510c648bb198793c2f0228a9f7f4b9579de89d70f7363a44d9eb997d32"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -157,11 +157,11 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7",
-                "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"
+                "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
+                "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
             ],
             "index": "pypi",
-            "version": "==2022.5.18.1"
+            "version": "==2022.12.7"
         },
         "cffi": {
             "hashes": [
@@ -1355,11 +1355,11 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7",
-                "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"
+                "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3",
+                "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"
             ],
             "index": "pypi",
-            "version": "==2022.5.18.1"
+            "version": "==2022.12.7"
         },
         "cffi": {
             "hashes": [

From 79ac21c2df55d197b11c885a336623c1623c3255 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 8 Dec 2022 14:31:20 -0500
Subject: [PATCH 04/46] Track notifications_utils main instead of specific
 commit

---
 Pipfile      | 2 +-
 Pipfile.lock | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Pipfile b/Pipfile
index c6c5e1f2a..1d930bc14 100644
--- a/Pipfile
+++ b/Pipfile
@@ -58,8 +58,8 @@ werkzeug = "~=2.1.1"
 # gds metrics packages
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
-notifications-utils = {editable = true, ref = "3096f2937b5e3b59a2e7a4b4ffb05b7fa1ab72fa", git = "https://github.com/GSA/notifications-utils"}
 packaging = "==21.3"
+notifications-utils = {editable = true, branch = "main", git = "https://github.com/GSA/notifications-utils"}
 
 [dev-packages]
 flake8 = "==4.0.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index 84dfc2b87..07532380e 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "85a9f8510c648bb198793c2f0228a9f7f4b9579de89d70f7363a44d9eb997d32"
+            "sha256": "64cec0ebf1afc8708dcbfee70bfa3881152b2c2487ace24b785e107821b95917"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -708,6 +708,7 @@
             "version": "==6.3.0"
         },
         "notifications-utils": {
+            "branch": "main",
             "editable": true,
             "git": "https://github.com/GSA/notifications-utils",
             "ref": "3096f2937b5e3b59a2e7a4b4ffb05b7fa1ab72fa"

From c1c6fec28fdfb3454990bf5fd91b3c75076a5e34 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 8 Dec 2022 15:46:39 -0500
Subject: [PATCH 05/46] Get utils version with updated certifi

---
 Pipfile      | 2 +-
 Pipfile.lock | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Pipfile b/Pipfile
index 1d930bc14..e4d67bd3e 100644
--- a/Pipfile
+++ b/Pipfile
@@ -59,7 +59,7 @@ werkzeug = "~=2.1.1"
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
 packaging = "==21.3"
-notifications-utils = {editable = true, branch = "main", git = "https://github.com/GSA/notifications-utils"}
+notifications-utils = {editable = true, ref = "bcadde83385a5437215d71dcb6c40ff90d71acfa", git = "https://github.com/GSA/notifications-utils"}
 
 [dev-packages]
 flake8 = "==4.0.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index 6de6ba961..2a5710ced 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "64cec0ebf1afc8708dcbfee70bfa3881152b2c2487ace24b785e107821b95917"
+            "sha256": "866299b8c725ee7effa00c7ed5cb22cb9db902a134b3f959960cc185254d2265"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -740,10 +740,9 @@
             "version": "==6.3.0"
         },
         "notifications-utils": {
-            "branch": "main",
             "editable": true,
             "git": "https://github.com/GSA/notifications-utils",
-            "ref": "3096f2937b5e3b59a2e7a4b4ffb05b7fa1ab72fa"
+            "ref": "bcadde83385a5437215d71dcb6c40ff90d71acfa"
         },
         "orderedset": {
             "hashes": [

From 1f183b96b942367951ecb954cd9802f402897276 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Fri, 9 Dec 2022 10:44:55 -0500
Subject: [PATCH 06/46] Add migration to expand api_keys.secret column max
 length

---
 .../versions/0381_encrypted_column_types.py   | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 migrations/versions/0381_encrypted_column_types.py

diff --git a/migrations/versions/0381_encrypted_column_types.py b/migrations/versions/0381_encrypted_column_types.py
new file mode 100644
index 000000000..ba50b53d9
--- /dev/null
+++ b/migrations/versions/0381_encrypted_column_types.py
@@ -0,0 +1,21 @@
+"""
+
+Revision ID: 0381_encrypted_column_types
+Revises: 0380_bst_to_local
+Create Date: 2022-12-09 10:17:03.358405
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = '0381_encrypted_column_types'
+down_revision = '0380_bst_to_local'
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

From 17ee4c3f2b3ff7dd72854ac5c778c9551cfed2ac Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Fri, 9 Dec 2022 16:33:10 -0500
Subject: [PATCH 07/46] Use encrypt/decrypt methods in place of signing

---
 Pipfile                                       |   2 +-
 Pipfile.lock                                  | 125 +++++++++++-------
 app/celery/service_callback_tasks.py          |   8 +-
 app/celery/tasks.py                           |  10 +-
 app/config.py                                 |   2 +-
 app/models.py                                 |  20 +--
 app/v2/notifications/post_notifications.py    |   2 +-
 .../versions/0381_encrypted_column_types.py   |   4 +-
 .../celery/test_process_ses_receipts_tasks.py |  16 +--
 tests/app/celery/test_scheduled_tasks.py      |   6 +-
 .../app/celery/test_service_callback_tasks.py |   4 +-
 tests/app/celery/test_tasks.py                | 114 ++++++++--------
 .../app/dao/test_service_callback_api_dao.py  |   4 +-
 tests/app/dao/test_service_inbound_api_dao.py |   4 +-
 tests/app/test_model.py                       |   8 +-
 15 files changed, 179 insertions(+), 150 deletions(-)

diff --git a/Pipfile b/Pipfile
index e4d67bd3e..90e7121fc 100644
--- a/Pipfile
+++ b/Pipfile
@@ -59,7 +59,7 @@ werkzeug = "~=2.1.1"
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
 packaging = "==21.3"
-notifications-utils = {editable = true, ref = "bcadde83385a5437215d71dcb6c40ff90d71acfa", git = "https://github.com/GSA/notifications-utils"}
+notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils"}
 
 [dev-packages]
 flake8 = "==4.0.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index 2a5710ced..45efea2f3 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "866299b8c725ee7effa00c7ed5cb22cb9db902a134b3f959960cc185254d2265"
+            "sha256": "f5be0c6621dbbe40f5b5bf4f4b8642189087ec2609a8406c39702d2a892f2415"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -742,7 +742,41 @@
         "notifications-utils": {
             "editable": true,
             "git": "https://github.com/GSA/notifications-utils",
-            "ref": "bcadde83385a5437215d71dcb6c40ff90d71acfa"
+            "ref": "553f4affbef4d0a9e94e9bcf2ceb5c81d92bf032"
+        },
+        "numpy": {
+            "hashes": [
+                "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d",
+                "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07",
+                "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df",
+                "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9",
+                "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d",
+                "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a",
+                "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719",
+                "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2",
+                "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280",
+                "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa",
+                "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387",
+                "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1",
+                "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43",
+                "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f",
+                "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398",
+                "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63",
+                "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de",
+                "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8",
+                "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481",
+                "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0",
+                "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d",
+                "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e",
+                "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96",
+                "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb",
+                "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6",
+                "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d",
+                "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a",
+                "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.23.5"
         },
         "orderedset": {
             "hashes": [
@@ -897,11 +931,11 @@
         },
         "pypdf2": {
             "hashes": [
-                "sha256:7074034d199a817f3c0190c533a1a64ed4237cc060cdb1ebc3a6da796a2c71c0",
-                "sha256:f1b0df8cd686d2ed069088b903ec65fb0f53e9c7837adc8139eccfbb7eb9141f"
+                "sha256:41ff16ee122bad9790d57a4235281a838002d7f1cc8d631d91b6f65d709bd825",
+                "sha256:e03ef18abcc75da741a0acc1a7749253496887be38cd9887bcce1cee393da45e"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.11.2"
+            "version": "==2.12.1"
         },
         "pyproj": {
             "hashes": [
@@ -1088,48 +1122,47 @@
         },
         "shapely": {
             "hashes": [
-                "sha256:02dd5d7dc6e46515d88874134dc8fcdc65826bca93c3eecee59d1910c42c1b17",
-                "sha256:0b4ee3132ee90f07d63db3aea316c4c065ed7a26231458dda0874414a09d6ba3",
-                "sha256:0d885cb0cf670c1c834df3f371de8726efdf711f18e2a75da5cfa82843a7ab65",
-                "sha256:147066da0be41b147a61f8eb805dea3b13709dbc873a431ccd7306e24d712bc0",
-                "sha256:21776184516a16bf82a0c3d6d6a312b3cd15a4cabafc61ee01cf2714a82e8396",
-                "sha256:2e0a8c2e55f1be1312b51c92b06462ea89e6bb703fab4b114e7a846d941cfc40",
-                "sha256:2fd15397638df291c427a53d641d3e6fd60458128029c8c4f487190473a69a91",
-                "sha256:3480657460e939f45a7d359ef0e172a081f249312557fe9aa78c4fd3a362d993",
-                "sha256:370b574c78dc5af3a198a6da5d9b3d7c04654bd2ef7e80e80a3a0992dfb2d9cd",
-                "sha256:38f0fbbcb8ca20c16451c966c1f527cc43968e121c8a048af19ed3e339a921cd",
-                "sha256:4728666fff8cccc65a07448cae72c75a8773fea061c3f4f139c44adc429b18c3",
-                "sha256:48dcfffb9e225c0481120f4bdf622131c8c95f342b00b158cdbe220edbbe20b6",
-                "sha256:532a55ee2a6c52d23d6f7d1567c8f0473635f3b270262c44e1b0c88096827e22",
-                "sha256:5d7f85c2d35d39ff53c9216bc76b7641c52326f7e09aaad1789a3611a0f812f2",
-                "sha256:65b21243d8f6bcd421210daf1fabb9de84de2c04353c5b026173b88d17c1a581",
-                "sha256:66bdac74fbd1d3458fa787191a90fa0ae610f09e2a5ec398c36f968cc0ed743f",
-                "sha256:6d388c0c1bd878ed1af4583695690aa52234b02ed35f93a1c8486ff52a555838",
-                "sha256:6fe855e7d45685926b6ba00aaeb5eba5862611f7465775dacd527e081a8ced6d",
-                "sha256:753ed0e21ab108bd4282405b9b659f2e985e8502b1a72b978eaa51d3496dee19",
-                "sha256:783bad5f48e2708a0e2f695a34ed382e4162c795cb2f0368b39528ac1d6db7ed",
-                "sha256:78fb9d929b8ee15cfd424b6c10879ce1907f24e05fb83310fc47d2cd27088e40",
-                "sha256:84010db15eb364a52b74ea8804ef92a6a930dfc1981d17a369444b6ddec66efd",
-                "sha256:8d086591f744be483b34628b391d741e46f2645fe37594319e0a673cc2c26bcf",
-                "sha256:8e59817b0fe63d34baedaabba8c393c0090f061917d18fc0bcc2f621937a8f73",
-                "sha256:99a2f0da0109e81e0c101a2b4cd8412f73f5f299e7b5b2deaf64cd2a100ac118",
-                "sha256:99ab0ddc05e44acabdbe657c599fdb9b2d82e86c5493bdae216c0c4018a82dee",
-                "sha256:a23ef3882d6aa203dd3623a3d55d698f59bfbd9f8a3bfed52c2da05a7f0f8640",
-                "sha256:a354199219c8d836f280b88f2c5102c81bb044ccea45bd361dc38a79f3873714",
-                "sha256:a74631e511153366c6dbe3229fa93f877e3c87ea8369cd00f1d38c76b0ed9ace",
-                "sha256:ab38f7b5196ace05725e407cb8cab9ff66edb8e6f7bb36a398e8f73f52a7aaa2",
-                "sha256:adcf8a11b98af9375e32bff91de184f33a68dc48b9cb9becad4f132fa25cfa3c",
-                "sha256:b65f5d530ba91e49ffc7c589255e878d2506a8b96ffce69d3b7c4500a9a9eaf8",
-                "sha256:be9423d5a3577ac2e92c7e758bd8a2b205f5e51a012177a590bc46fc51eb4834",
-                "sha256:c2822111ddc5bcfb116e6c663e403579d0fe3f147d2a97426011a191c43a7458",
-                "sha256:c6a9a4a31cd6e86d0fbe8473ceed83d4fe760b19d949fb557ef668defafea0f6",
-                "sha256:d048f93e42ba578b82758c15d8ae037d08e69d91d9872bca5a1895b118f4e2b0",
-                "sha256:e9c30b311de2513555ab02464ebb76115d242842b29c412f5a9aa0cac57be9f6",
-                "sha256:ec14ceca36f67cb48b34d02d7f65a9acae15cd72b48e303531893ba4a960f3ea",
-                "sha256:ef3be705c3eac282a28058e6c6e5503419b250f482320df2172abcbea642c831"
+                "sha256:11f1b1231a6c04213fb1226c6968d1b1b3b369ec42d1e9655066af87631860ea",
+                "sha256:13a9f978cd287e0fa95f39904a2bb36deddab490e4fab8bf43eba01b7d9eb58f",
+                "sha256:17d0f89581aa15f7887052a6adf2753f9fe1c3fdbb6116653972e0d43e720e65",
+                "sha256:21ba32a6c45b7f8ab7d2d8d5cf339704e2d1dfdf3e2fb465b950a0c9bc894a4f",
+                "sha256:2287d0cb592c1814e9f48065888af7ee3f13e090e6f7fa3e208b06a83fb2f6af",
+                "sha256:292c22ff7806e3a25bc4324295e9204169c61a09165d4c9ee0a9784c1709c85e",
+                "sha256:40c397d67ba609a163d38b649eee2b06c5f9bdc86d244a8e4cd09c6e2791cf3c",
+                "sha256:44198fc188fe4b7dd39ef0fd325395d1d6ab0c29a7bbaa15663a16c362bf6f62",
+                "sha256:5477be8c11bf3109f7b804bb2d57536538b8d0a6118207f1020d71338f1a827c",
+                "sha256:550f110940d79931b6a12a17de07f6b158c9586c4b121f885af11458ae5626d7",
+                "sha256:56c0e70749f8c2956493e9333375d2e2264ce25c838fc49c3a2ececbf2d3ba92",
+                "sha256:5fe8649aafe6adcb4d90f7f735f06ca8ca02a16da273d901f1dd02afc0d3618e",
+                "sha256:6c71738702cf5c3fc60b3bbe869c321b053ea754f57addded540a71c78c2612e",
+                "sha256:7266080d39946395ba4b31fa35b9b7695e0a4e38ccabf0c67e2936caf9f9b054",
+                "sha256:73771b3f65c2949cce0b310b9b62b8ce069407ceb497a9dd4436f9a4d059f12c",
+                "sha256:73d605fcefd06ee997ba307ef363448d355f3c3e81b3f56ed332eaf6d506e1b5",
+                "sha256:7b2c41514ba985ea3772eee9b386d620784cccb7a459a270a072f3ef01fdd807",
+                "sha256:820bee508e4a0e564db22f8b55bb5e6e7f326d8d7c103639c42f5d3f378f4067",
+                "sha256:8a7ba97c97d85c1f07c57f9524c45128ef2bf8279061945d78052c78862b357f",
+                "sha256:8b9f780c3b79b4a6501e0e8833b1877841b7b0e0a243e77b529fda8f1030afc2",
+                "sha256:91bbca0378eb82f0808f0e59150ac0952086f4caaab87ad8515a5e55e896c21e",
+                "sha256:99420c89af78f371b96f0e2bad9afdebc6d0707d4275d157101483e4c4049fd6",
+                "sha256:a391cae931976fb6d8d15a4f4a92006358e93486454a812dde1d64184041a476",
+                "sha256:a9b6651812f2caa23e4d06bc06a2ed34450f82cb1c110c170a25b01bbb090895",
+                "sha256:b1def13ec2a74ebda2210d2fc1c53cecce5a079ec90f341101399427874507f1",
+                "sha256:b3d97f3ce6df47ca68c2d64b8c3cfa5c8ccc0fbc81ef8e15ff6004a6426e71b1",
+                "sha256:c47a61b1cd0c5b064c6d912bce7dba78c01f319f65ecccd6e61eecd21861a37a",
+                "sha256:c4b99a3456e06dc55482569669ece969cdab311f2ad2a1d5622fc770f68cf3cd",
+                "sha256:d28e19791c9be2ba1cb2fddefa86f73364bdf8334e88dbcd78a8e4494c0af66b",
+                "sha256:d486cab823f0a978964ae97ca10564ea2b2ced93e84a2ef0b7b62cbacec9d3d2",
+                "sha256:de3722c68e49fbde8cb6859695bbb8fb9a4d48bbdf34fcf38b7994d2bd9772e2",
+                "sha256:e4ed31658fd0799eaa3569982aab1a5bc8fcf25ec196606bf137ee4fa984be88",
+                "sha256:e991ad155783cd0830b895ec8f310fde9e79a7b283776b889a751fb1e7c819fc",
+                "sha256:eab24b60ae96b7375adceb1f120be818c59bd69db0f3540dc89527d8a371d253",
+                "sha256:eaea9ddee706654026a84aceb9a3156105917bab3de58fcf150343f847478202",
+                "sha256:ef98fec4a3aca6d33e3b9fdd680fe513cc7d1c6aedc65ada8a3965601d9d4bcf",
+                "sha256:f69c418f2040c8593e33b1aba8f2acf890804b073b817535b5d291139d152af5",
+                "sha256:f96b24da0242791cd6042f6caf074e7a4537a66ca2d1b57d423feb98ba901295"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.8.5.post1"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.0"
         },
         "six": {
             "hashes": [
diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py
index d06f1a173..0e81d38d0 100644
--- a/app/celery/service_callback_tasks.py
+++ b/app/celery/service_callback_tasks.py
@@ -12,7 +12,7 @@ from app.utils import DATETIME_FORMAT
 def send_delivery_status_to_service(
     self, notification_id, encrypted_status_update
 ):
-    status_update = encryption.verify_signature(encrypted_status_update)
+    status_update = encryption.decrypt(encrypted_status_update)
 
     data = {
         "id": str(notification_id),
@@ -38,7 +38,7 @@ def send_delivery_status_to_service(
 
 @notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
 def send_complaint_to_service(self, complaint_data):
-    complaint = encryption.verify_signature(complaint_data)
+    complaint = encryption.decrypt(complaint_data)
 
     data = {
         "notification_id": complaint['notification_id'],
@@ -125,7 +125,7 @@ def create_delivery_status_callback_data(notification, service_callback_api):
         "template_id": str(notification.template_id),
         "template_version": notification.template_version,
     }
-    return encryption.sign(data)
+    return encryption.encrypt(data)
 
 
 def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
@@ -138,4 +138,4 @@ def create_complaint_callback_data(complaint, notification, service_callback_api
         "service_callback_api_url": service_callback_api.url,
         "service_callback_api_bearer_token": service_callback_api.bearer_token,
     }
-    return encryption.sign(data)
+    return encryption.encrypt(data)
diff --git a/app/celery/tasks.py b/app/celery/tasks.py
index ca8ad8957..dd37229f1 100644
--- a/app/celery/tasks.py
+++ b/app/celery/tasks.py
@@ -125,7 +125,7 @@ def get_recipient_csv_and_template_and_sender_id(job):
 
 def process_row(row, template, job, service, sender_id=None):
     template_type = template.template_type
-    encrypted = encryption.sign({
+    encrypted = encryption.encrypt({
         'template': str(template.id),
         'template_version': job.template_version,
         'job': str(job.id),
@@ -183,7 +183,7 @@ def save_sms(self,
              notification_id,
              encrypted_notification,
              sender_id=None):
-    notification = encryption.verify_signature(encrypted_notification)
+    notification = encryption.decrypt(encrypted_notification)
     service = SerialisedService.from_id(service_id)
     template = SerialisedTemplate.from_id_and_service_id(
         notification['template'],
@@ -241,7 +241,7 @@ def save_email(self,
                notification_id,
                encrypted_notification,
                sender_id=None):
-    notification = encryption.verify_signature(encrypted_notification)
+    notification = encryption.decrypt(encrypted_notification)
 
     service = SerialisedService.from_id(service_id)
     template = SerialisedTemplate.from_id_and_service_id(
@@ -298,7 +298,7 @@ def save_api_sms(self, encrypted_notification):
 
 
 def save_api_email_or_sms(self, encrypted_notification):
-    notification = encryption.verify_signature(encrypted_notification)
+    notification = encryption.decrypt(encrypted_notification)
     service = SerialisedService.from_id(notification['service_id'])
     q = QueueNames.SEND_EMAIL if notification['notification_type'] == EMAIL_TYPE else QueueNames.SEND_SMS
     provider_task = provider_tasks.deliver_email if notification['notification_type'] == EMAIL_TYPE \
@@ -348,7 +348,7 @@ def save_letter(
         notification_id,
         encrypted_notification,
 ):
-    notification = encryption.verify_signature(encrypted_notification)
+    notification = encryption.decrypt(encrypted_notification)
 
     postal_address = PostalAddress.from_personalisation(
         InsensitiveDict(notification['personalisation'])
diff --git a/app/config.py b/app/config.py
index 2e73fa813..1344620fb 100644
--- a/app/config.py
+++ b/app/config.py
@@ -361,7 +361,7 @@ class Development(Config):
     CONTACT_LIST_BUCKET = _default_s3_credentials('local-contact-list')
 
     # credential overrides
-    DANGEROUS_SALT = 'dev-notify-salt'
+    DANGEROUS_SALT = 'development-notify-salt'
     SECRET_KEY = 'dev-notify-secret-key'  # nosec B105 - this is only used in development
     INTERNAL_CLIENT_API_KEYS = {Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key']}
     ALLOW_EXPIRED_API_TOKEN = getenv('ALLOW_EXPIRED_API_TOKEN', '0') == '1'
diff --git a/app/models.py b/app/models.py
index bb91bcc79..e2c5c5bed 100644
--- a/app/models.py
+++ b/app/models.py
@@ -738,13 +738,13 @@ class ServiceInboundApi(db.Model, Versioned):
     @property
     def bearer_token(self):
         if self._bearer_token:
-            return encryption.verify_signature(self._bearer_token)
+            return encryption.decrypt(self._bearer_token)
         return None
 
     @bearer_token.setter
     def bearer_token(self, bearer_token):
         if bearer_token:
-            self._bearer_token = encryption.sign(str(bearer_token))
+            self._bearer_token = encryption.encrypt(str(bearer_token))
 
     def serialize(self):
         return {
@@ -777,13 +777,13 @@ class ServiceCallbackApi(db.Model, Versioned):
     @property
     def bearer_token(self):
         if self._bearer_token:
-            return encryption.verify_signature(self._bearer_token)
+            return encryption.decrypt(self._bearer_token)
         return None
 
     @bearer_token.setter
     def bearer_token(self, bearer_token):
         if bearer_token:
-            self._bearer_token = encryption.sign(str(bearer_token))
+            self._bearer_token = encryption.encrypt(str(bearer_token))
 
     def serialize(self):
         return {
@@ -834,13 +834,13 @@ class ApiKey(db.Model, Versioned):
     @property
     def secret(self):
         if self._secret:
-            return encryption.verify_signature(self._secret)
+            return encryption.decrypt(self._secret)
         return None
 
     @secret.setter
     def secret(self, secret):
         if secret:
-            self._secret = encryption.sign(str(secret))
+            self._secret = encryption.encrypt(str(secret))
 
 
 KEY_TYPE_NORMAL = 'normal'
@@ -1512,12 +1512,12 @@ class Notification(db.Model):
     @property
     def personalisation(self):
         if self._personalisation:
-            return encryption.verify_signature(self._personalisation)
+            return encryption.decrypt(self._personalisation)
         return {}
 
     @personalisation.setter
     def personalisation(self, personalisation):
-        self._personalisation = encryption.sign(personalisation or {})
+        self._personalisation = encryption.encrypt(personalisation or {})
 
     def completed_at(self):
         if self.status in NOTIFICATION_STATUS_TYPES_COMPLETED:
@@ -1960,11 +1960,11 @@ class InboundSms(db.Model):
 
     @property
     def content(self):
-        return encryption.verify_signature(self._content)
+        return encryption.decrypt(self._content)
 
     @content.setter
     def content(self, content):
-        self._content = encryption.sign(content)
+        self._content = encryption.encrypt(content)
 
     def serialize(self):
         return {
diff --git a/app/v2/notifications/post_notifications.py b/app/v2/notifications/post_notifications.py
index c6402e735..756c4d846 100644
--- a/app/v2/notifications/post_notifications.py
+++ b/app/v2/notifications/post_notifications.py
@@ -300,7 +300,7 @@ def save_email_or_sms_to_queue(
         "status": NOTIFICATION_CREATED,
         "created_at": datetime.utcnow().strftime(DATETIME_FORMAT),
     }
-    encrypted = encryption.sign(
+    encrypted = encryption.encrypt(
         data
     )
 
diff --git a/migrations/versions/0381_encrypted_column_types.py b/migrations/versions/0381_encrypted_column_types.py
index ba50b53d9..64761fd8e 100644
--- a/migrations/versions/0381_encrypted_column_types.py
+++ b/migrations/versions/0381_encrypted_column_types.py
@@ -14,8 +14,8 @@ down_revision = '0380_bst_to_local'
 
 
 def upgrade():
-    pass
+    op.alter_column("api_keys", "secret", type_=sa.types.String())
 
 
 def downgrade():
-    pass
+    op.alter_column("api_keys", "secret", type_=sa.types.String(length=255))
diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py
index 7311f4171..1ec8afcea 100644
--- a/tests/app/celery/test_process_ses_receipts_tasks.py
+++ b/tests/app/celery/test_process_ses_receipts_tasks.py
@@ -1,5 +1,6 @@
 import json
 from datetime import datetime
+from unittest.mock import ANY
 
 from freezegun import freeze_time
 
@@ -14,11 +15,8 @@ from app.celery.research_mode_tasks import (
     ses_notification_callback,
     ses_soft_bounce_callback,
 )
-from app.celery.service_callback_tasks import (
-    create_delivery_status_callback_data,
-)
 from app.dao.notifications_dao import get_notification_by_id
-from app.models import Complaint, Notification
+from app.models import Complaint
 from tests.app.conftest import create_sample_notification
 from tests.app.db import (
     create_notification,
@@ -156,7 +154,7 @@ def test_ses_callback_should_update_notification_status(
             status='sending',
             sent_at=datetime.utcnow()
         )
-        callback_api = create_service_callback_api(
+        create_service_callback_api(
             service=sample_email_template.service,
             url="https://original_url.com"
         )
@@ -167,9 +165,9 @@ def test_ses_callback_should_update_notification_status(
             "callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at
         )
         statsd_client.incr.assert_any_call("callback.ses.delivered")
-        updated_notification = Notification.query.get(notification.id)
-        encrypted_data = create_delivery_status_callback_data(updated_notification, callback_api)
-        send_mock.assert_called_once_with([str(notification.id), encrypted_data], queue="service-callbacks")
+        send_mock.assert_called_once_with([str(notification.id), ANY], queue="service-callbacks")
+        # assert second arg is an encrypted string
+        assert isinstance(send_mock.call_args.args[0][1], str)
 
 
 def test_ses_callback_should_not_update_notification_status_if_already_delivered(sample_email_template, mocker):
@@ -331,7 +329,7 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email
     response = ses_complaint_callback()
     assert process_ses_results(response)
     assert send_mock.call_count == 1
-    assert encryption.verify_signature(send_mock.call_args[0][0][0]) == {
+    assert encryption.decrypt(send_mock.call_args[0][0][0]) == {
         'complaint_date': '2018-06-05T13:59:58.000000Z',
         'complaint_id': str(Complaint.query.one().id),
         'notification_id': str(notification.id),
diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py
index 36aa7e2db..55298af2c 100644
--- a/tests/app/celery/test_scheduled_tasks.py
+++ b/tests/app/celery/test_scheduled_tasks.py
@@ -465,7 +465,7 @@ def test_check_for_missing_rows_in_completed_jobs_ignores_old_and_new_jobs(
 ):
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_email'), {"sender_id": None}))
-    mocker.patch('app.encryption.sign', return_value="something_encrypted")
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     process_row = mocker.patch('app.celery.scheduled_tasks.process_row')
 
     job = create_job(
@@ -485,7 +485,7 @@ def test_check_for_missing_rows_in_completed_jobs_ignores_old_and_new_jobs(
 def test_check_for_missing_rows_in_completed_jobs(mocker, sample_email_template):
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_email'), {"sender_id": None}))
-    mocker.patch('app.encryption.sign', return_value="something_encrypted")
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     process_row = mocker.patch('app.celery.scheduled_tasks.process_row')
 
     job = create_job(template=sample_email_template,
@@ -506,7 +506,7 @@ def test_check_for_missing_rows_in_completed_jobs_calls_save_email(mocker, sampl
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_email'), {'sender_id': None}))
     save_email_task = mocker.patch('app.celery.tasks.save_email.apply_async')
-    mocker.patch('app.encryption.sign', return_value="something_encrypted")
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value='uuid')
 
     job = create_job(template=sample_email_template,
diff --git a/tests/app/celery/test_service_callback_tasks.py b/tests/app/celery/test_service_callback_tasks.py
index 1996d2239..d5249c678 100644
--- a/tests/app/celery/test_service_callback_tasks.py
+++ b/tests/app/celery/test_service_callback_tasks.py
@@ -191,7 +191,7 @@ def _set_up_data_for_status_update(callback_api, notification):
         "template_id": str(notification.template_id),
         "template_version": notification.template_version,
     }
-    encrypted_status_update = encryption.sign(data)
+    encrypted_status_update = encryption.encrypt(data)
     return encrypted_status_update
 
 
@@ -205,5 +205,5 @@ def _set_up_data_for_complaint(callback_api, complaint, notification):
         "service_callback_api_url": callback_api.url,
         "service_callback_api_bearer_token": callback_api.bearer_token,
     }
-    obscured_status_update = encryption.sign(data)
+    obscured_status_update = encryption.encrypt(data)
     return obscured_status_update
diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py
index 9556ac0ea..7fda9fd49 100644
--- a/tests/app/celery/test_tasks.py
+++ b/tests/app/celery/test_tasks.py
@@ -109,7 +109,7 @@ def test_should_process_sms_job(sample_job, mocker):
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('sms'), {'sender_id': None}))
     mocker.patch('app.celery.tasks.save_sms.apply_async')
-    mocker.patch('app.encryption.sign', return_value="something_encrypted")
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(sample_job.id)
@@ -117,11 +117,11 @@ def test_should_process_sms_job(sample_job, mocker):
         service_id=str(sample_job.service.id),
         job_id=str(sample_job.id)
     )
-    assert encryption.sign.call_args[0][0]['to'] == '+441234123123'
-    assert encryption.sign.call_args[0][0]['template'] == str(sample_job.template.id)
-    assert encryption.sign.call_args[0][0]['template_version'] == sample_job.template.version
-    assert encryption.sign.call_args[0][0]['personalisation'] == {'phonenumber': '+441234123123'}
-    assert encryption.sign.call_args[0][0]['row_number'] == 0
+    assert encryption.encrypt.call_args[0][0]['to'] == '+441234123123'
+    assert encryption.encrypt.call_args[0][0]['template'] == str(sample_job.template.id)
+    assert encryption.encrypt.call_args[0][0]['template_version'] == sample_job.template.version
+    assert encryption.encrypt.call_args[0][0]['personalisation'] == {'phonenumber': '+441234123123'}
+    assert encryption.encrypt.call_args[0][0]['row_number'] == 0
     tasks.save_sms.apply_async.assert_called_once_with(
         (str(sample_job.service_id),
          "uuid",
@@ -137,7 +137,7 @@ def test_should_process_sms_job_with_sender_id(sample_job, mocker, fake_uuid):
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('sms'), {'sender_id': fake_uuid}))
     mocker.patch('app.celery.tasks.save_sms.apply_async')
-    mocker.patch('app.encryption.sign', return_value="something_encrypted")
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(sample_job.id, sender_id=fake_uuid)
@@ -211,7 +211,7 @@ def test_should_process_job_if_send_limits_are_not_exceeded(
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_email'), {"sender_id": None}))
     mocker.patch('app.celery.tasks.save_email.apply_async')
-    mocker.patch('app.encryption.sign', return_value="something_encrypted")
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
     mocker.patch('app.celery.tasks.check_service_over_daily_message_limit', return_value=0)
     process_job(job.id)
@@ -255,7 +255,7 @@ def test_should_process_email_job(email_job_with_placeholders, mocker):
     """
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3', return_value=(email_csv, {"sender_id": None}))
     mocker.patch('app.celery.tasks.save_email.apply_async')
-    mocker.patch('app.encryption.sign', return_value="something_encrypted")
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(email_job_with_placeholders.id)
@@ -264,10 +264,10 @@ def test_should_process_email_job(email_job_with_placeholders, mocker):
         service_id=str(email_job_with_placeholders.service.id),
         job_id=str(email_job_with_placeholders.id)
     )
-    assert encryption.sign.call_args[0][0]['to'] == 'test@test.com'
-    assert encryption.sign.call_args[0][0]['template'] == str(email_job_with_placeholders.template.id)
-    assert encryption.sign.call_args[0][0]['template_version'] == email_job_with_placeholders.template.version
-    assert encryption.sign.call_args[0][0]['personalisation'] == {'emailaddress': 'test@test.com', 'name': 'foo'}
+    assert encryption.encrypt.call_args[0][0]['to'] == 'test@test.com'
+    assert encryption.encrypt.call_args[0][0]['template'] == str(email_job_with_placeholders.template.id)
+    assert encryption.encrypt.call_args[0][0]['template_version'] == email_job_with_placeholders.template.version
+    assert encryption.encrypt.call_args[0][0]['personalisation'] == {'emailaddress': 'test@test.com', 'name': 'foo'}
     tasks.save_email.apply_async.assert_called_once_with(
         (
             str(email_job_with_placeholders.service_id),
@@ -287,7 +287,7 @@ def test_should_process_email_job_with_sender_id(email_job_with_placeholders, mo
     """
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3', return_value=(email_csv, {"sender_id": fake_uuid}))
     mocker.patch('app.celery.tasks.save_email.apply_async')
-    mocker.patch('app.encryption.sign', return_value="something_encrypted")
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(email_job_with_placeholders.id, sender_id=fake_uuid)
@@ -341,7 +341,7 @@ def test_should_process_all_sms_job(sample_job_with_placeholdered_template,
     mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
                  return_value=(load_example_csv('multiple_sms'), {"sender_id": None}))
     mocker.patch('app.celery.tasks.save_sms.apply_async')
-    mocker.patch('app.encryption.sign', return_value="something_encrypted")
+    mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
     mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
 
     process_job(sample_job_with_placeholdered_template.id)
@@ -350,11 +350,11 @@ def test_should_process_all_sms_job(sample_job_with_placeholdered_template,
         service_id=str(sample_job_with_placeholdered_template.service.id),
         job_id=str(sample_job_with_placeholdered_template.id)
     )
-    assert encryption.sign.call_args[0][0]['to'] == '+441234123120'
-    assert encryption.sign.call_args[0][0]['template'] == str(sample_job_with_placeholdered_template.template.id)
-    assert encryption.sign.call_args[0][0][
+    assert encryption.encrypt.call_args[0][0]['to'] == '+441234123120'
+    assert encryption.encrypt.call_args[0][0]['template'] == str(sample_job_with_placeholdered_template.template.id)
+    assert encryption.encrypt.call_args[0][0][
                'template_version'] == sample_job_with_placeholdered_template.template.version  # noqa
-    assert encryption.sign.call_args[0][0]['personalisation'] == {'phonenumber': '+441234123120', 'name': 'chris'}
+    assert encryption.encrypt.call_args[0][0]['personalisation'] == {'phonenumber': '+441234123120', 'name': 'chris'}
     assert tasks.save_sms.apply_async.call_count == 10
     job = jobs_dao.dao_get_job_by_id(sample_job_with_placeholdered_template.id)
     assert job.job_status == 'finished'
@@ -374,7 +374,7 @@ def test_should_process_all_sms_job(sample_job_with_placeholdered_template,
 def test_process_row_sends_letter_task(template_type, research_mode, expected_function, expected_queue, mocker):
     mocker.patch('app.celery.tasks.create_uuid', return_value='noti_uuid')
     task_mock = mocker.patch('app.celery.tasks.{}.apply_async'.format(expected_function))
-    encrypt_mock = mocker.patch('app.celery.tasks.encryption.sign')
+    encrypt_mock = mocker.patch('app.celery.tasks.encryption.encrypt')
     template = Mock(id='template_id', template_type=template_type)
     job = Mock(id='job_id', template_version='temp_vers')
     service = Mock(id='service_id', research_mode=research_mode)
@@ -417,7 +417,7 @@ def test_process_row_sends_letter_task(template_type, research_mode, expected_fu
 def test_process_row_when_sender_id_is_provided(mocker, fake_uuid):
     mocker.patch('app.celery.tasks.create_uuid', return_value='noti_uuid')
     task_mock = mocker.patch('app.celery.tasks.save_sms.apply_async')
-    encrypt_mock = mocker.patch('app.celery.tasks.encryption.sign')
+    encrypt_mock = mocker.patch('app.celery.tasks.encryption.encrypt')
     template = Mock(id='template_id', template_type=SMS_TYPE)
     job = Mock(id='job_id', template_version='temp_vers')
     service = Mock(id='service_id', research_mode=False)
@@ -460,7 +460,7 @@ def test_should_send_template_to_correct_sms_task_and_persist(sample_template_wi
     save_sms(
         sample_template_with_placeholders.service_id,
         uuid.uuid4(),
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -473,7 +473,6 @@ def test_should_send_template_to_correct_sms_task_and_persist(sample_template_wi
     assert not persisted_notification.sent_by
     assert not persisted_notification.job_id
     assert persisted_notification.personalisation == {'name': 'Jo'}
-    assert persisted_notification._personalisation == encryption.sign({"name": "Jo"})
     assert persisted_notification.notification_type == 'sms'
     mocked_deliver_sms.assert_called_once_with(
         [str(persisted_notification.id)],
@@ -495,7 +494,7 @@ def test_should_put_save_sms_task_in_research_mode_queue_if_research_mode_servic
     save_sms(
         template.service_id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
     persisted_notification = Notification.query.one()
     provider_tasks.deliver_sms.apply_async.assert_called_once_with(
@@ -514,7 +513,7 @@ def test_should_save_sms_if_restricted_service_and_valid_number(notify_db_sessio
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     notification_id = uuid.uuid4()
-    encrypt_notification = encryption.sign(notification)
+    encrypt_notification = encryption.encrypt(notification)
     save_sms(
         service.id,
         notification_id,
@@ -550,7 +549,7 @@ def test_save_email_should_save_default_email_reply_to_text_on_notification(noti
     save_email(
         service.id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -568,7 +567,7 @@ def test_save_sms_should_save_default_smm_sender_notification_reply_to_text_on(n
     save_sms(
         service.id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -587,7 +586,7 @@ def test_should_not_save_sms_if_restricted_service_and_invalid_number(notify_db_
     save_sms(
         service.id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
     assert provider_tasks.deliver_sms.apply_async.called is False
     assert Notification.query.count() == 0
@@ -603,7 +602,7 @@ def test_should_not_save_email_if_restricted_service_and_invalid_email_address(n
     save_email(
         service.id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
 
     assert Notification.query.count() == 0
@@ -625,7 +624,7 @@ def test_should_put_save_email_task_in_research_mode_queue_if_research_mode_serv
     save_email(
         template.service_id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -648,7 +647,7 @@ def test_should_save_sms_template_to_and_persist_with_job_id(sample_job, mocker)
     save_sms(
         sample_job.service.id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
     persisted_notification = Notification.query.one()
     assert persisted_notification.to == '+447234123123'
@@ -685,7 +684,7 @@ def test_should_not_save_sms_if_team_key_and_recipient_not_in_team(notify_db_ses
     save_sms(
         service.id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
     assert provider_tasks.deliver_sms.apply_async.called is False
     assert Notification.query.count() == 0
@@ -708,7 +707,7 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh
         save_email(
             sample_email_template_with_placeholders.service_id,
             notification_id,
-            encryption.sign(notification),
+            encryption.encrypt(notification),
         )
 
     persisted_notification = Notification.query.one()
@@ -721,7 +720,6 @@ def test_should_use_email_template_and_persist(sample_email_template_with_placeh
     assert not persisted_notification.sent_by
     assert persisted_notification.job_row_number == 1
     assert persisted_notification.personalisation == {'name': 'Jo'}
-    assert persisted_notification._personalisation == encryption.sign({"name": "Jo"})
     assert persisted_notification.api_key_id is None
     assert persisted_notification.key_type == KEY_TYPE_NORMAL
     assert persisted_notification.notification_type == 'email'
@@ -747,7 +745,7 @@ def test_save_email_should_use_template_version_from_job_not_latest(sample_email
     save_email(
         sample_email_template.service_id,
         uuid.uuid4(),
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -773,7 +771,7 @@ def test_should_use_email_template_subject_placeholders(sample_email_template_wi
     save_email(
         sample_email_template_with_placeholders.service_id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
     persisted_notification = Notification.query.one()
     assert persisted_notification.to == 'my_email@my_email.com'
@@ -802,7 +800,7 @@ def test_save_email_uses_the_reply_to_text_when_provided(sample_email_template,
     save_email(
         sample_email_template.service_id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
         sender_id=other_email_reply_to.id,
     )
     persisted_notification = Notification.query.one()
@@ -821,7 +819,7 @@ def test_save_email_uses_the_default_reply_to_text_if_sender_id_is_none(sample_e
     save_email(
         sample_email_template.service_id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
         sender_id=None,
     )
     persisted_notification = Notification.query.one()
@@ -839,7 +837,7 @@ def test_should_use_email_template_and_persist_without_personalisation(sample_em
     save_email(
         sample_email_template.service_id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
     persisted_notification = Notification.query.one()
     assert persisted_notification.to == 'my_email@my_email.com'
@@ -870,7 +868,7 @@ def test_save_sms_should_go_to_retry_queue_if_database_errors(sample_template, m
         save_sms(
             sample_template.service_id,
             notification_id,
-            encryption.sign(notification),
+            encryption.encrypt(notification),
         )
     assert provider_tasks.deliver_sms.apply_async.called is False
     tasks.save_sms.retry.assert_called_with(exc=expected_exception, queue="retry-tasks")
@@ -893,7 +891,7 @@ def test_save_email_should_go_to_retry_queue_if_database_errors(sample_email_tem
         save_email(
             sample_email_template.service_id,
             notification_id,
-            encryption.sign(notification),
+            encryption.encrypt(notification),
         )
     assert not provider_tasks.deliver_email.apply_async.called
     tasks.save_email.retry.assert_called_with(exc=expected_exception, queue="retry-tasks")
@@ -911,7 +909,7 @@ def test_save_email_does_not_send_duplicate_and_does_not_put_in_retry_queue(samp
     save_email(
         sample_notification.service_id,
         notification_id,
-        encryption.sign(json),
+        encryption.encrypt(json),
     )
     assert Notification.query.count() == 1
     assert not deliver_email.called
@@ -928,7 +926,7 @@ def test_save_sms_does_not_send_duplicate_and_does_not_put_in_retry_queue(sample
     save_sms(
         sample_notification.service_id,
         notification_id,
-        encryption.sign(json),
+        encryption.encrypt(json),
     )
     assert Notification.query.count() == 1
     assert not deliver_sms.called
@@ -998,7 +996,7 @@ def test_save_letter_saves_letter_to_database(
     save_letter(
         job.service_id,
         notification_id,
-        encryption.sign(notification_json),
+        encryption.encrypt(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1042,7 +1040,7 @@ def test_save_letter_saves_letter_to_database_with_correct_postage(
     save_letter(
         letter_job.service_id,
         notification_id,
-        encryption.sign(notification_json),
+        encryption.encrypt(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1075,7 +1073,7 @@ def test_save_letter_saves_letter_to_database_with_correct_client_reference(
     save_letter(
         letter_job.service_id,
         notification_id,
-        encryption.sign(notification_json),
+        encryption.encrypt(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1100,7 +1098,7 @@ def test_save_letter_saves_letter_to_database_with_formatted_postcode(mocker, no
     save_letter(
         letter_job.service_id,
         notification_id,
-        encryption.sign(notification_json),
+        encryption.encrypt(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1139,7 +1137,7 @@ def test_save_letter_saves_letter_to_database_right_reply_to(mocker, notify_db_s
     save_letter(
         job.service_id,
         notification_id,
-        encryption.sign(notification_json),
+        encryption.encrypt(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1201,7 +1199,7 @@ def test_save_letter_uses_template_reply_to_text(mocker, notify_db_session):
     save_letter(
         job.service_id,
         uuid.uuid4(),
-        encryption.sign(notification_json),
+        encryption.encrypt(notification_json),
     )
 
     notification_db = Notification.query.one()
@@ -1219,7 +1217,7 @@ def test_save_sms_uses_sms_sender_reply_to_text(mocker, notify_db_session):
     save_sms(
         service.id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
     )
 
     persisted_notification = Notification.query.one()
@@ -1238,7 +1236,7 @@ def test_save_sms_uses_non_default_sms_sender_reply_to_text_if_provided(mocker,
     save_sms(
         service.id,
         notification_id,
-        encryption.sign(notification),
+        encryption.encrypt(notification),
         sender_id=new_sender.id,
     )
 
@@ -1276,7 +1274,7 @@ def test_save_letter_sets_delivered_letters_as_pdf_permission_in_research_mode_i
         save_letter(
             sample_letter_job.service_id,
             notification_id,
-            encryption.sign(notification_json),
+            encryption.encrypt(notification_json),
         )
 
     notification = Notification.query.filter(Notification.id == notification_id).one()
@@ -1314,7 +1312,7 @@ def test_save_letter_calls_create_fake_response_for_letters_in_research_mode_on_
         save_letter(
             sample_letter_job.service_id,
             notification_id,
-            encryption.sign(notification_json),
+            encryption.encrypt(notification_json),
         )
 
     mock_create_fake_letter_response_file.assert_called_once_with(
@@ -1345,7 +1343,7 @@ def test_save_letter_calls_get_pdf_for_templated_letter_task_not_in_research(
     save_letter(
         sample_letter_job.service_id,
         notification_id,
-        encryption.sign(notification_json),
+        encryption.encrypt(notification_json),
     )
 
     assert mock_create_letters_pdf.called
@@ -1843,7 +1841,7 @@ def test_save_api_email_or_sms(mocker, sample_service, notification_type):
         data.update({"to": "+447700900855"})
         expected_queue = QueueNames.SEND_SMS
 
-    encrypted = encryption.sign(
+    encrypted = encryption.encrypt(
         data
     )
 
@@ -1890,7 +1888,7 @@ def test_save_api_email_dont_retry_if_notification_already_exists(sample_service
         data.update({"to": "+447700900855"})
         expected_queue = QueueNames.SEND_SMS
 
-    encrypted = encryption.sign(
+    encrypted = encryption.encrypt(
         data
     )
     assert len(Notification.query.all()) == 0
@@ -1953,7 +1951,7 @@ def test_save_tasks_use_cached_service_and_template(
         task_function(
             service.id,
             uuid.uuid4(),
-            encryption.sign(notification),
+            encryption.encrypt(notification),
         )
 
     # We talk to the database once for the service and once for the
@@ -1995,7 +1993,7 @@ def test_save_api_tasks_use_cache(
     api_key = create_api_key(service=template.service)
 
     def create_encrypted_notification():
-        return encryption.sign({
+        return encryption.encrypt({
             "to": recipient,
             "id": str(uuid.uuid4()),
             "template_id": str(template.id),
diff --git a/tests/app/dao/test_service_callback_api_dao.py b/tests/app/dao/test_service_callback_api_dao.py
index 573cf5b72..504e470d8 100644
--- a/tests/app/dao/test_service_callback_api_dao.py
+++ b/tests/app/dao/test_service_callback_api_dao.py
@@ -40,7 +40,7 @@ def test_save_service_callback_api(sample_service):
     assert versioned.service_id == sample_service.id
     assert versioned.updated_by_id == sample_service.users[0].id
     assert versioned.url == "https://some_service/callback_endpoint"
-    assert encryption.verify_signature(versioned._bearer_token) == "some_unique_string"
+    assert encryption.decrypt(versioned._bearer_token) == "some_unique_string"
     assert versioned.updated_at is None
     assert versioned.version == 1
 
@@ -140,7 +140,7 @@ def test_update_service_callback_api(sample_service):
         assert x.id is not None
         assert x.service_id == sample_service.id
         assert x.updated_by_id == sample_service.users[0].id
-        assert encryption.verify_signature(x._bearer_token) == "some_unique_string"
+        assert encryption.decrypt(x._bearer_token) == "some_unique_string"
 
 
 def test_get_service_callback_api(sample_service):
diff --git a/tests/app/dao/test_service_inbound_api_dao.py b/tests/app/dao/test_service_inbound_api_dao.py
index 6f7f57af5..26a750533 100644
--- a/tests/app/dao/test_service_inbound_api_dao.py
+++ b/tests/app/dao/test_service_inbound_api_dao.py
@@ -40,7 +40,7 @@ def test_save_service_inbound_api(sample_service):
     assert versioned.service_id == sample_service.id
     assert versioned.updated_by_id == sample_service.users[0].id
     assert versioned.url == "https://some_service/inbound_messages"
-    assert encryption.verify_signature(versioned._bearer_token) == "some_unique_string"
+    assert encryption.decrypt(versioned._bearer_token) == "some_unique_string"
     assert versioned.updated_at is None
     assert versioned.version == 1
 
@@ -97,7 +97,7 @@ def test_update_service_inbound_api(sample_service):
         assert x.id is not None
         assert x.service_id == sample_service.id
         assert x.updated_by_id == sample_service.users[0].id
-        assert encryption.verify_signature(x._bearer_token) == "some_unique_string"
+        assert encryption.decrypt(x._bearer_token) == "some_unique_string"
 
 
 def test_get_service_inbound_api(sample_service):
diff --git a/tests/app/test_model.py b/tests/app/test_model.py
index 57c4b40cd..a7f394db9 100644
--- a/tests/app/test_model.py
+++ b/tests/app/test_model.py
@@ -154,9 +154,9 @@ def test_notification_personalisation_getter_returns_empty_dict_from_None():
     assert noti.personalisation == {}
 
 
-def test_notification_personalisation_getter_always_returns_empty_dict():
+def test_notification_personalisation_getter_always_returns_empty_dict(notify_api):
     noti = Notification()
-    noti._personalisation = encryption.sign({})
+    noti._personalisation = encryption.encrypt({})
     assert noti.personalisation == {}
 
 
@@ -164,11 +164,11 @@ def test_notification_personalisation_getter_always_returns_empty_dict():
     None,
     {}
 ])
-def test_notification_personalisation_setter_always_sets_empty_dict(input_value):
+def test_notification_personalisation_setter_always_sets_empty_dict(notify_api, input_value):
     noti = Notification()
     noti.personalisation = input_value
 
-    assert noti._personalisation == encryption.sign({})
+    assert noti.personalisation == {}
 
 
 def test_notification_subject_is_none_for_sms(sample_service):

From 0d0db5f6f6ad115e751de1fd5acdc240713fe10e Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Mon, 12 Dec 2022 16:46:31 -0500
Subject: [PATCH 08/46] Document potential PII and data storage policies

---
 docs/data-storage.md | 64 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 docs/data-storage.md

diff --git a/docs/data-storage.md b/docs/data-storage.md
new file mode 100644
index 000000000..54045c742
--- /dev/null
+++ b/docs/data-storage.md
@@ -0,0 +1,64 @@
+Data Storage Policies & Procedures
+==================================
+
+
+Potential PII Locations
+-----------------------
+
+### Tables
+
+#### users<sup>1</sup>
+
+* name
+* email_address
+* mobile_number
+
+#### invited_users<sup>1</sup>
+
+* email_address
+
+#### invited_organization_users<sup>1</sup>
+
+* email_address
+
+#### jobs
+
+No db data is PII, but each job has a csv file in s3 containing phone numbers and personalization data.
+
+#### notifications
+
+* to
+* _personalization<sup>2</sup>
+* phone_prefix (?)
+
+#### notification_history
+
+* phone_prefix (?)
+
+#### inbound_sms
+
+* content<sup>2</sup>
+* user_number
+
+#### events
+
+* data (contains user IP addresses)
+
+### Notes
+
+#### Note 1.
+
+Users and invited users are Federal, State, or Local government employees or contractors. Members of the general public are _not_ users of the system
+
+#### Note 2.
+
+Field-level encryption is used on these fields.
+
+Details on encryption schemes and algorithms can be found in [SC-28(1)](https://github.com/GSA/us-notify-compliance/blob/main/dist/system-security-plans/lato/sc-28.1.md)
+
+Data Retention Policy
+---------------------
+
+Seven (7) days by default. Each service can be set with a custom policy via `ServiceDataRetention` by a Platform Admin. The `ServiceDataRetention` setting applies per-service and per-message type and controls both entries in the `notifications` table as well as `csv` contact files uploaded to s3
+
+Data cleanup is controlled by several tasks in the `nightly_tasks.py` file, kicked off by Celery Beat.

From 6aa46da91680d8b93ef4562b6c8be92ee82d3a07 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 13 Dec 2022 08:53:41 -0500
Subject: [PATCH 09/46] Create fixture that just calls create_app

---
 tests/app/test_model.py |  4 ++--
 tests/conftest.py       | 12 ++++++++----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/tests/app/test_model.py b/tests/app/test_model.py
index a7f394db9..3a98831da 100644
--- a/tests/app/test_model.py
+++ b/tests/app/test_model.py
@@ -154,7 +154,7 @@ def test_notification_personalisation_getter_returns_empty_dict_from_None():
     assert noti.personalisation == {}
 
 
-def test_notification_personalisation_getter_always_returns_empty_dict(notify_api):
+def test_notification_personalisation_getter_always_returns_empty_dict(notify_app):
     noti = Notification()
     noti._personalisation = encryption.encrypt({})
     assert noti.personalisation == {}
@@ -164,7 +164,7 @@ def test_notification_personalisation_getter_always_returns_empty_dict(notify_ap
     None,
     {}
 ])
-def test_notification_personalisation_setter_always_sets_empty_dict(notify_api, input_value):
+def test_notification_personalisation_setter_always_sets_empty_dict(notify_app, input_value):
     noti = Notification()
     noti.personalisation = input_value
 
diff --git a/tests/conftest.py b/tests/conftest.py
index dabd3eb33..12dfb3710 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -12,16 +12,20 @@ from app.dao.provider_details_dao import get_provider_details_by_identifier
 
 
 @pytest.fixture(scope='session')
-def notify_api():
+def notify_app():
     app = Flask('test')
     create_app(app)
+    return app
 
+
+@pytest.fixture(scope='session')
+def notify_api(notify_app):
     # deattach server-error error handlers - error_handler_spec looks like:
     #   {'blueprint_name': {
     #       status_code: [error_handlers],
     #       None: { ExceptionClass: error_handler }
     # }}
-    for error_handlers in app.error_handler_spec.values():
+    for error_handlers in notify_app.error_handler_spec.values():
         error_handlers.pop(500, None)
         if None in error_handlers:
             error_handlers[None] = {
@@ -32,10 +36,10 @@ def notify_api():
             if error_handlers[None] == []:
                 error_handlers.pop(None)
 
-    ctx = app.app_context()
+    ctx = notify_app.app_context()
     ctx.push()
 
-    yield app
+    yield notify_app
 
     ctx.pop()
 

From 7860ab9bb0c761d0948000d2ec95c424369a279e Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 13 Dec 2022 11:53:07 -0500
Subject: [PATCH 10/46] Update notifications-utils repo address

---
 Pipfile      |  2 +-
 Pipfile.lock | 71 ++++++++++++++++++++++++++++------------------------
 2 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/Pipfile b/Pipfile
index 90e7121fc..1b9ef1a27 100644
--- a/Pipfile
+++ b/Pipfile
@@ -59,7 +59,7 @@ werkzeug = "~=2.1.1"
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
 packaging = "==21.3"
-notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils"}
+notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils.git"}
 
 [dev-packages]
 flake8 = "==4.0.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index 45efea2f3..d08abf43d 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f5be0c6621dbbe40f5b5bf4f4b8642189087ec2609a8406c39702d2a892f2415"
+            "sha256": "d0a90dd082bf51ae47e04412c0848f00bc5e4b538804d1333e9264ca0ad2d4e9"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -741,7 +741,7 @@
         },
         "notifications-utils": {
             "editable": true,
-            "git": "https://github.com/GSA/notifications-utils",
+            "git": "https://github.com/GSA/notifications-utils.git",
             "ref": "553f4affbef4d0a9e94e9bcf2ceb5c81d92bf032"
         },
         "numpy": {
@@ -939,39 +939,44 @@
         },
         "pyproj": {
             "hashes": [
-                "sha256:0fff9c3a991508f16027be27d153f6c5583d03799443639d13c681e60f49e2d7",
-                "sha256:12f62c20656ac9b6076ebb213e9a635d52f4f01fef95310121d337e62e910cb6",
-                "sha256:14ad113b5753c6057f9b2f3c85a6497cef7fa237c4328f2943c0223e98c1dde6",
-                "sha256:1f9c100fd0fd80edbc7e4daa303600a8cbef6f0de43d005617acb38276b88dc0",
-                "sha256:221d8939685e0c43ee594c9f04b6a73a10e8e1cc0e85f28be0b4eb2f1bc8777d",
-                "sha256:25a36e297f3e0524694d40259e3e895edc1a47492a0e30608268ffc1328e3f5d",
-                "sha256:2cb8592259ea54e7557523b079d3f2304081680bdb48bfbf0fd879ee6156129c",
-                "sha256:3b85acf09e5a9e35cd9ee72989793adb7089b4e611be02a43d3d0bda50ad116b",
-                "sha256:45554f47d1a12a84b0620e4abc08a2a1b5d9f273a4759eaef75e74788ec7162a",
-                "sha256:4688b4cd62cbd86b5e855f9e27d90fbb53f2b4c2ea1cd394a46919e1a4151b89",
-                "sha256:47ad53452ae1dc8b0bf1df920a210bb5616989085aa646592f8681f1d741a754",
-                "sha256:48787962232109bad8b72e27949037a9b03591228a6955f25dbe451233e8648a",
-                "sha256:4a23d84c5ffc383c7d9f0bde3a06fc1f6697b1b96725597f8f01e7b4bef0a2b5",
-                "sha256:4e161114bc92701647a83c4bbce79489984f12d980cabb365516e953d1450885",
-                "sha256:4fd425ee8b6781c249c7adb7daa2e6c41ce573afabe4f380f5eecd913b56a3be",
-                "sha256:52e54796e2d9554a5eb8f11df4748af1fbbc47f76aa234d6faf09216a84554c5",
-                "sha256:5816807ca0bdc7256558770c6206a6783a3f02bcf844f94ee245f197bb5f7285",
-                "sha256:65a0bcdbad95b3c00b419e5d75b1f7e450ec17349b5ea16bf7438ac1d50a12a2",
-                "sha256:77d5f519f3cdb94b026ecca626f78db4f041afe201cf082079c8c0092a30b087",
-                "sha256:82200b4569d68b421c079d2973475b58d5959306fe758b43366e79fe96facfe5",
-                "sha256:954b068136518b3174d0a99448056e97af62b63392a95c420894f7de2229dae6",
-                "sha256:9a496d9057b2128db9d733e66b206f2d5954bbae6b800d412f562d780561478c",
-                "sha256:a454a7c4423faa2a14e939d08ef293ee347fa529c9df79022b0585a6e1d8310c",
-                "sha256:a708445927ace9857f52c3ba67d2915da7b41a8fdcd9b8f99a4c9ed60a75eb33",
-                "sha256:aa5171f700f174777a9e9ed8f4655583243967c0f9cf2c90e3f54e54ff740134",
-                "sha256:ccb4b70ad25218027f77e0c8934d10f9b7cdf91d5e64080147743d58fddbc3c0",
-                "sha256:d94afed99f31673d3d19fe750283621e193e2a53ca9e0443bf9d092c3905833b",
-                "sha256:e7e609903572a56cca758bbaee5c1663c3e829ddce5eec4f368e68277e37022b",
-                "sha256:f343725566267a296b09ee7e591894f1fdc90f84f8ad5ec476aeb53bd4479c07",
-                "sha256:f80adda8c54b84271a93829477a01aa57bc178c834362e9f74e1de1b5033c74c"
+                "sha256:0189fdd7aa789542a7a623010dfff066c5849b24397f81f860ec3ee085cbf55c",
+                "sha256:0406f64ff59eb3342efb102c9f31536430aa5cde5ef0bfabd5aaccb73dd8cd5a",
+                "sha256:0c7b32382ae22a9bf5b690d24c7b4c0fb89ba313c3a91ef1a8c54b50baf10954",
+                "sha256:129234afa179c8293b010ea4f73655ff7b20b5afdf7fac170f223bcf0ed6defd",
+                "sha256:19f5de1a7c3b81b676d846350d4bdf2ae6af13b9a450d1881706f088ecad0e2c",
+                "sha256:231c038c6b65395c41ae3362320f03ce8054cb54dc63556e605695e5d461a27e",
+                "sha256:25a5425cd2a0b16f5f944d49165196eebaa60b898a08c404a644c29e6a7a04b3",
+                "sha256:261eb29b1d55b1eb7f336127344d9b31284d950a9446d1e0d1c2411f7dd8e3ac",
+                "sha256:2bd633f3b8ca6eb09135dfaf06f09e2869deb139985aab26d728e8a60c9938b9",
+                "sha256:2d1e7f42da205e0534831ae9aa9cee0353ab8c1aab2c369474adbb060294d98a",
+                "sha256:2f87f16b902c8b2af007295c63a435f043db9e40bd45e6f96962c7b8cd08fdb5",
+                "sha256:321b82210dc5271558573d0874b9967c5a25872a28d0168049ddabe8bfecffce",
+                "sha256:3d70ca5933cddbe6f51396006fb9fc78bc2b1f9d28775922453c4b04625a7efb",
+                "sha256:57ec7d2b7f2773d877927abc72e2229ef8530c09181be0e28217742bae1bc4f5",
+                "sha256:5a53acbde511a7a9e1873c7f93c68f35b8c3653467b77195fe18e847555dcb7a",
+                "sha256:5f3f75b030cf811f040c90a8758a20115e8746063e4cad0d0e941a4954d1219b",
+                "sha256:6bdac3bc1899fcc4021be06d303b342923fb8311fe06f8d862c348a1a0e78b41",
+                "sha256:7aef19d5a0a3b2d6b17f7dc9a87af722e71139cd1eea7eb82ed062a8a4b0e272",
+                "sha256:8078c90cea07d53e3406c7c84cbf76a2ac0ffc580c365f13801575486b9d558c",
+                "sha256:97065fe82e80f7e2740e7897a0e36e8defc0a3614927f0276b4f1d1ea1ef66fa",
+                "sha256:a30d78e619dae5cd1bb69addae2f1e5f8ee1b4a8ab4f3d954e9eaf41948db506",
+                "sha256:a32e1d12340ad93232b7ea4dc1a4f4b21fa9fa9efa4b293adad45be7af6b51ec",
+                "sha256:a5eada965e8ac24e783f2493d1d9bcd11c5c93959bd43558224dd31d9faebd1c",
+                "sha256:a98fe3e53be428e67ae6a9ee9affff92346622e0e3ea0cbc15dce939b318d395",
+                "sha256:c240fe6bcb5c325b50fc967d5458d708412633f4f05fefc7fb14c14254ebf421",
+                "sha256:c60d112d8f1621a606b7f2adb0b1582f80498e663413d2ba9f5df1c93d99f432",
+                "sha256:cd9f9c409f465834988ce0aa8c1ed496081c6957f2e5ef40ed28de04397d3c0b",
+                "sha256:ce50126dad7cd4749ab86fc4c8b54ec0898149ce6710ab5c93c76a54a4afa249",
+                "sha256:da96319b137cfd66f0bae0e300cdc77dd17af4785b9360a9bdddb1d7176a0bbb",
+                "sha256:e463c687007861a9949909211986850cfc2e72930deda0d06449ef2e315db534",
+                "sha256:e8c0d1ac9ef5a4d2e6501a4b30136c55f1e1db049d1626cc313855c4f97d196d",
+                "sha256:e9d82df555cf19001bac40e1de0e40fb762dec785685b77edd6993286c01b7f7",
+                "sha256:ef76abfee1a0676ef973470abe11e22998750f2bd944afaf76d44ad70b538c06",
+                "sha256:ef8c30c62fe4e386e523e14e1e83bd460f745bd2c8dfd0d0c327f9460c4d3c0c",
+                "sha256:f38dea459e22e86326b1c7d47718a3e10c7a27910cf5eb86ea2679b8084d0c4e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.4.0"
+            "version": "==3.4.1"
         },
         "pyrsistent": {
             "hashes": [

From 431f7aeb3cde534b4d7970bc506ca932a905fd42 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 13 Dec 2022 14:18:34 -0500
Subject: [PATCH 11/46] Gracefully handle decryption errors in past-7-days
 notification reports

---
 app/models.py           | 8 +++++++-
 tests/app/test_model.py | 7 +++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/app/models.py b/app/models.py
index e2c5c5bed..15b1ecb51 100644
--- a/app/models.py
+++ b/app/models.py
@@ -3,6 +3,9 @@ import itertools
 import uuid
 
 from flask import current_app, url_for
+from notifications_utils.clients.encryption.encryption_client import (
+    EncryptionError,
+)
 from notifications_utils.insensitive_dict import InsensitiveDict
 from notifications_utils.letter_timings import get_letter_timings
 from notifications_utils.postal_address import (
@@ -1512,7 +1515,10 @@ class Notification(db.Model):
     @property
     def personalisation(self):
         if self._personalisation:
-            return encryption.decrypt(self._personalisation)
+            try:
+                return encryption.decrypt(self._personalisation)
+            except EncryptionError:
+                current_app.logger.error("Error decrypting notification.personalisation, returning empty dict")
         return {}
 
     @personalisation.setter
diff --git a/tests/app/test_model.py b/tests/app/test_model.py
index 3a98831da..3798c358b 100644
--- a/tests/app/test_model.py
+++ b/tests/app/test_model.py
@@ -160,6 +160,13 @@ def test_notification_personalisation_getter_always_returns_empty_dict(notify_ap
     assert noti.personalisation == {}
 
 
+def test_notification_personalisation_getter_returns_empty_dict_for_encryption_errors(notify_app):
+    noti = Notification()
+    # old _personalisation values were created with encryption.sign, which will trigger a decryption error
+    noti._personalisation = encryption.sign({"value": "PII"})
+    assert noti.personalisation == {}
+
+
 @pytest.mark.parametrize('input_value', [
     None,
     {}

From 469e2a2fc0fbc7afd857c92f48b0ef013681b742 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 22 Dec 2022 08:51:15 -0500
Subject: [PATCH 12/46] Specify terraform module versions

---
 terraform/demo/main.tf       |  8 ++++----
 terraform/production/main.tf | 10 +++++-----
 terraform/sandbox/main.tf    |  8 ++++----
 terraform/staging/main.tf    |  8 ++++----
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf
index cb703cf50..e609c8c34 100644
--- a/terraform/demo/main.tf
+++ b/terraform/demo/main.tf
@@ -7,7 +7,7 @@ locals {
 }
 
 module "database" {
-  source = "github.com/18f/terraform-cloudgov//database"
+  source = "github.com/18f/terraform-cloudgov//database?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -20,7 +20,7 @@ module "database" {
 }
 
 module "redis" {
-  source = "github.com/18f/terraform-cloudgov//redis"
+  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -33,7 +33,7 @@ module "redis" {
 }
 
 module "csv_upload_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -44,7 +44,7 @@ module "csv_upload_bucket" {
 }
 
 module "contact_list_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
diff --git a/terraform/production/main.tf b/terraform/production/main.tf
index e32610a37..cd9fb7126 100644
--- a/terraform/production/main.tf
+++ b/terraform/production/main.tf
@@ -7,7 +7,7 @@ locals {
 }
 
 module "database" {
-  source = "github.com/18f/terraform-cloudgov//database"
+  source = "github.com/18f/terraform-cloudgov//database?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -20,7 +20,7 @@ module "database" {
 }
 
 module "redis" {
-  source = "github.com/18f/terraform-cloudgov//redis"
+  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -33,7 +33,7 @@ module "redis" {
 }
 
 module "csv_upload_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -44,7 +44,7 @@ module "csv_upload_bucket" {
 }
 
 module "contact_list_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -74,7 +74,7 @@ module "egress-space" {
 #     `cf create-domain TKTK-org-name TKTK-production-domain-name`
 ###########################################################################
 # module "domain" {
-#   source = "github.com/18f/terraform-cloudgov//domain"
+#   source = "github.com/18f/terraform-cloudgov//domain?ref=v0.1.0"
 #
 #   cf_user          = var.cf_user
 #   cf_password      = var.cf_password
diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf
index 96e0338ab..1a692080a 100644
--- a/terraform/sandbox/main.tf
+++ b/terraform/sandbox/main.tf
@@ -7,7 +7,7 @@ locals {
 }
 
 module "database" {
-  source = "github.com/18f/terraform-cloudgov//database"
+  source = "github.com/18f/terraform-cloudgov//database?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -20,7 +20,7 @@ module "database" {
 }
 
 module "redis" {
-  source = "github.com/18f/terraform-cloudgov//redis"
+  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -33,7 +33,7 @@ module "redis" {
 }
 
 module "csv_upload_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -44,7 +44,7 @@ module "csv_upload_bucket" {
 }
 
 module "contact_list_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf
index 911e9d8b9..5a2143090 100644
--- a/terraform/staging/main.tf
+++ b/terraform/staging/main.tf
@@ -7,7 +7,7 @@ locals {
 }
 
 module "database" {
-  source = "github.com/18f/terraform-cloudgov//database"
+  source = "github.com/18f/terraform-cloudgov//database?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -20,7 +20,7 @@ module "database" {
 }
 
 module "redis" {
-  source = "github.com/18f/terraform-cloudgov//redis"
+  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -33,7 +33,7 @@ module "redis" {
 }
 
 module "csv_upload_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password
@@ -44,7 +44,7 @@ module "csv_upload_bucket" {
 }
 
 module "contact_list_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
 
   cf_user          = var.cf_user
   cf_password      = var.cf_password

From f967d2a4788da99a710275b08b9f54989f4d1483 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 22 Dec 2022 08:57:16 -0500
Subject: [PATCH 13/46] Update terraform module to v0.2.0

---
 terraform/demo/main.tf                     | 28 +++++------------
 terraform/demo/providers.tf                |  7 +++++
 terraform/production/main.tf               | 36 +++++++---------------
 terraform/production/providers.tf          |  7 +++++
 terraform/sandbox/main.tf                  | 28 +++++------------
 terraform/sandbox/providers.tf             |  7 +++++
 terraform/shared/egress_space/main.tf      | 10 +++---
 terraform/shared/egress_space/providers.tf |  9 +-----
 terraform/shared/egress_space/variables.tf |  5 ---
 terraform/staging/main.tf                  | 28 +++++------------
 terraform/staging/providers.tf             |  7 +++++
 11 files changed, 69 insertions(+), 103 deletions(-)

diff --git a/terraform/demo/main.tf b/terraform/demo/main.tf
index e609c8c34..86ce9ba53 100644
--- a/terraform/demo/main.tf
+++ b/terraform/demo/main.tf
@@ -7,58 +7,46 @@ locals {
 }
 
 module "database" {
-  source = "github.com/18f/terraform-cloudgov//database?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//database?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
-  env              = local.env
-  app_name         = local.app_name
+  name             = "${local.app_name}-rds-${local.env}"
   recursive_delete = local.recursive_delete
   rds_plan_name    = "micro-psql"
 }
 
 module "redis" {
-  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
-  env              = local.env
-  app_name         = local.app_name
+  name             = "${local.app_name}-redis-${local.env}"
   recursive_delete = local.recursive_delete
   redis_plan_name  = "redis-dev"
 }
 
 module "csv_upload_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
   recursive_delete = local.recursive_delete
-  s3_service_name  = "${local.app_name}-csv-upload-bucket-${local.env}"
+  name             = "${local.app_name}-csv-upload-bucket-${local.env}"
 }
 
 module "contact_list_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
   recursive_delete = local.recursive_delete
-  s3_service_name  = "${local.app_name}-contact-list-bucket-${local.env}"
+  name             = "${local.app_name}-contact-list-bucket-${local.env}"
 }
 
 module "egress-space" {
   source = "../shared/egress_space"
 
-  cf_user                  = var.cf_user
-  cf_password              = var.cf_password
   cf_org_name              = local.cf_org_name
   cf_restricted_space_name = local.cf_space_name
   deployers = [
diff --git a/terraform/demo/providers.tf b/terraform/demo/providers.tf
index 11d4c4457..180959761 100644
--- a/terraform/demo/providers.tf
+++ b/terraform/demo/providers.tf
@@ -15,3 +15,10 @@ terraform {
     profile = "notify-terraform-backend"
   }
 }
+
+provider "cloudfoundry" {
+  api_url      = "https://api.fr.cloud.gov"
+  user         = var.cf_user
+  password     = var.cf_password
+  app_logs_max = 30
+}
diff --git a/terraform/production/main.tf b/terraform/production/main.tf
index cd9fb7126..2bcae487b 100644
--- a/terraform/production/main.tf
+++ b/terraform/production/main.tf
@@ -7,58 +7,46 @@ locals {
 }
 
 module "database" {
-  source = "github.com/18f/terraform-cloudgov//database?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//database?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
-  env              = local.env
-  app_name         = local.app_name
+  name             = "${local.app_name}-rds-${local.env}"
   recursive_delete = local.recursive_delete
   rds_plan_name    = "TKTK-production-rds-plan"
 }
 
 module "redis" {
-  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
-  env              = local.env
-  app_name         = local.app_name
+  name             = "${local.app_name}-redis-${local.env}"
   recursive_delete = local.recursive_delete
   redis_plan_name  = "TKTK-production-redis-plan"
 }
 
 module "csv_upload_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
   recursive_delete = local.recursive_delete
-  s3_service_name  = "${local.app_name}-csv-upload-bucket-${local.env}"
+  name             = "${local.app_name}-csv-upload-bucket-${local.env}"
 }
 
 module "contact_list_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
   recursive_delete = local.recursive_delete
-  s3_service_name  = "${local.app_name}-contact-list-bucket-${local.env}"
+  name             = "${local.app_name}-contact-list-bucket-${local.env}"
 }
 
 module "egress-space" {
   source = "../shared/egress_space"
 
-  cf_user                  = var.cf_user
-  cf_password              = var.cf_password
   cf_org_name              = local.cf_org_name
   cf_restricted_space_name = local.cf_space_name
   deployers = [
@@ -74,14 +62,12 @@ module "egress-space" {
 #     `cf create-domain TKTK-org-name TKTK-production-domain-name`
 ###########################################################################
 # module "domain" {
-#   source = "github.com/18f/terraform-cloudgov//domain?ref=v0.1.0"
+#   source = "github.com/18f/terraform-cloudgov//domain?ref=v0.2.0"
 #
-#   cf_user          = var.cf_user
-#   cf_password      = var.cf_password
 #   cf_org_name      = local.cf_org_name
 #   cf_space_name    = local.cf_space_name
-#   env              = local.env
-#   app_name         = local.app_name
+#   app_name_or_id   = "${local.app_name}-${local.env}"
+#   name             = "${local.app_name}-domain-${local.env}"
 #   recursive_delete = local.recursive_delete
 #   cdn_plan_name    = "domain"
 #   domain_name      = "TKTK-production-domain-name"
diff --git a/terraform/production/providers.tf b/terraform/production/providers.tf
index f4cfe869a..360f648f4 100644
--- a/terraform/production/providers.tf
+++ b/terraform/production/providers.tf
@@ -15,3 +15,10 @@ terraform {
     profile = "notify-terraform-backend"
   }
 }
+
+provider "cloudfoundry" {
+  api_url      = "https://api.fr.cloud.gov"
+  user         = var.cf_user
+  password     = var.cf_password
+  app_logs_max = 30
+}
diff --git a/terraform/sandbox/main.tf b/terraform/sandbox/main.tf
index 1a692080a..6e8bb11f2 100644
--- a/terraform/sandbox/main.tf
+++ b/terraform/sandbox/main.tf
@@ -7,58 +7,46 @@ locals {
 }
 
 module "database" {
-  source = "github.com/18f/terraform-cloudgov//database?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//database?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
-  env              = local.env
-  app_name         = local.app_name
+  name             = "${local.app_name}-rds-${local.env}"
   recursive_delete = local.recursive_delete
   rds_plan_name    = "micro-psql"
 }
 
 module "redis" {
-  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
-  env              = local.env
-  app_name         = local.app_name
+  name             = "${local.app_name}-redis-${local.env}"
   recursive_delete = local.recursive_delete
   redis_plan_name  = "redis-dev"
 }
 
 module "csv_upload_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
   recursive_delete = local.recursive_delete
-  s3_service_name  = "${local.app_name}-csv-upload-bucket-${local.env}"
+  name             = "${local.app_name}-csv-upload-bucket-${local.env}"
 }
 
 module "contact_list_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
   recursive_delete = local.recursive_delete
-  s3_service_name  = "${local.app_name}-contact-list-bucket-${local.env}"
+  name             = "${local.app_name}-contact-list-bucket-${local.env}"
 }
 
 module "egress-space" {
   source = "../shared/egress_space"
 
-  cf_user                  = var.cf_user
-  cf_password              = var.cf_password
   cf_org_name              = local.cf_org_name
   cf_restricted_space_name = local.cf_space_name
   deployers = [
diff --git a/terraform/sandbox/providers.tf b/terraform/sandbox/providers.tf
index 0cab42f6f..00ea2888d 100644
--- a/terraform/sandbox/providers.tf
+++ b/terraform/sandbox/providers.tf
@@ -15,3 +15,10 @@ terraform {
     profile = "notify-terraform-backend"
   }
 }
+
+provider "cloudfoundry" {
+  api_url      = "https://api.fr.cloud.gov"
+  user         = var.cf_user
+  password     = var.cf_password
+  app_logs_max = 30
+}
diff --git a/terraform/shared/egress_space/main.tf b/terraform/shared/egress_space/main.tf
index 4a65b74e9..4b841ad14 100644
--- a/terraform/shared/egress_space/main.tf
+++ b/terraform/shared/egress_space/main.tf
@@ -12,7 +12,7 @@ data "cloudfoundry_org" "org" {
 
 resource "cloudfoundry_space" "public_egress" {
   name = "${var.cf_restricted_space_name}-egress"
-  org = data.cloudfoundry_org.org.id
+  org  = data.cloudfoundry_org.org.id
 }
 
 ###
@@ -21,8 +21,8 @@ resource "cloudfoundry_space" "public_egress" {
 
 data "cloudfoundry_user" "users" {
   for_each = var.deployers
-  name = each.key
-  org_id = data.cloudfoundry_org.org.id
+  name     = each.key
+  org_id   = data.cloudfoundry_org.org.id
 }
 
 locals {
@@ -30,7 +30,7 @@ locals {
 }
 
 resource "cloudfoundry_space_users" "deployers" {
-  space = cloudfoundry_space.public_egress.id
-  managers = local.user_ids
+  space      = cloudfoundry_space.public_egress.id
+  managers   = local.user_ids
   developers = local.user_ids
 }
diff --git a/terraform/shared/egress_space/providers.tf b/terraform/shared/egress_space/providers.tf
index ad8addecd..8db86ca90 100644
--- a/terraform/shared/egress_space/providers.tf
+++ b/terraform/shared/egress_space/providers.tf
@@ -2,15 +2,8 @@ terraform {
   required_version = "~> 1.0"
   required_providers {
     cloudfoundry = {
-      source = "cloudfoundry-community/cloudfoundry"
+      source  = "cloudfoundry-community/cloudfoundry"
       version = "~> 0.15"
     }
   }
 }
-
-provider "cloudfoundry" {
-  api_url = "https://api.fr.cloud.gov"
-  user = var.cf_user
-  password = var.cf_password
-  app_logs_max = 30
-}
diff --git a/terraform/shared/egress_space/variables.tf b/terraform/shared/egress_space/variables.tf
index 1ab080dea..45bcc717d 100644
--- a/terraform/shared/egress_space/variables.tf
+++ b/terraform/shared/egress_space/variables.tf
@@ -1,8 +1,3 @@
-variable "cf_password" {
-  type = string
-  sensitive = true
-}
-variable "cf_user" {}
 variable "cf_org_name" {}
 variable "cf_restricted_space_name" {}
 variable "deployers" {
diff --git a/terraform/staging/main.tf b/terraform/staging/main.tf
index 5a2143090..a2ea6d0d8 100644
--- a/terraform/staging/main.tf
+++ b/terraform/staging/main.tf
@@ -7,58 +7,46 @@ locals {
 }
 
 module "database" {
-  source = "github.com/18f/terraform-cloudgov//database?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//database?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
-  env              = local.env
-  app_name         = local.app_name
+  name             = "${local.app_name}-rds-${local.env}"
   recursive_delete = local.recursive_delete
   rds_plan_name    = "micro-psql"
 }
 
 module "redis" {
-  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//redis?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
-  env              = local.env
-  app_name         = local.app_name
+  name             = "${local.app_name}-redis-${local.env}"
   recursive_delete = local.recursive_delete
   redis_plan_name  = "redis-dev"
 }
 
 module "csv_upload_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
   recursive_delete = local.recursive_delete
-  s3_service_name  = "${local.app_name}-csv-upload-bucket-${local.env}"
+  name             = "${local.app_name}-csv-upload-bucket-${local.env}"
 }
 
 module "contact_list_bucket" {
-  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.1.0"
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.2.0"
 
-  cf_user          = var.cf_user
-  cf_password      = var.cf_password
   cf_org_name      = local.cf_org_name
   cf_space_name    = local.cf_space_name
   recursive_delete = local.recursive_delete
-  s3_service_name  = "${local.app_name}-contact-list-bucket-${local.env}"
+  name             = "${local.app_name}-contact-list-bucket-${local.env}"
 }
 
 module "egress-space" {
   source = "../shared/egress_space"
 
-  cf_user                  = var.cf_user
-  cf_password              = var.cf_password
   cf_org_name              = local.cf_org_name
   cf_restricted_space_name = local.cf_space_name
   deployers = [
diff --git a/terraform/staging/providers.tf b/terraform/staging/providers.tf
index 72a7c30f6..181b5fd69 100644
--- a/terraform/staging/providers.tf
+++ b/terraform/staging/providers.tf
@@ -15,3 +15,10 @@ terraform {
     profile = "notify-terraform-backend"
   }
 }
+
+provider "cloudfoundry" {
+  api_url      = "https://api.fr.cloud.gov"
+  user         = var.cf_user
+  password     = var.cf_password
+  app_logs_max = 30
+}

From 6a04be03708bbad1a0c7eff59d179c91eae12980 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Fri, 16 Dec 2022 12:21:40 -0500
Subject: [PATCH 14/46] Remove firetext and mmg from inbound SMS options

---
 app/config.py                                 |  17 --
 app/models.py                                 |   4 +-
 app/notifications/receive_notifications.py    | 120 +-------
 pytest.ini                                    |   2 -
 .../test_receive_notification.py              | 267 +++---------------
 5 files changed, 53 insertions(+), 357 deletions(-)

diff --git a/app/config.py b/app/config.py
index 1344620fb..8d9d58d8f 100644
--- a/app/config.py
+++ b/app/config.py
@@ -114,23 +114,10 @@ class Config(object):
     VALIDATE_SNS_TOPICS = True
     VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound']
 
-    # SMS config to be cleaned up during https://github.com/GSA/notifications-api/issues/7
-    # MMG API Key
-    MMG_API_KEY = getenv('MMG_API_KEY', 'placeholder')
-    # Firetext API Key
-    FIRETEXT_API_KEY = getenv("FIRETEXT_API_KEY", "placeholder")
-    FIRETEXT_INTERNATIONAL_API_KEY = getenv("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder")
     # these should always add up to 100%
     SMS_PROVIDER_RESTING_POINTS = {
         'sns': 100,
-        'mmg': 0,
-        'firetext': 0
     }
-    FIRETEXT_INBOUND_SMS_AUTH = json.loads(getenv('FIRETEXT_INBOUND_SMS_AUTH', '[]'))
-    MMG_INBOUND_SMS_AUTH = json.loads(getenv('MMG_INBOUND_SMS_AUTH', '[]'))
-    MMG_INBOUND_SMS_USERNAME = json.loads(getenv('MMG_INBOUND_SMS_USERNAME', '[]'))
-    MMG_URL = getenv("MMG_URL", "https://api.mmg.co.uk/jsonv2a/api.php")
-    FIRETEXT_URL = getenv("FIRETEXT_URL", "https://www.firetext.co.uk/api/sendsms/json")
 
     # Zendesk
     ZENDESK_API_KEY = getenv('ZENDESK_API_KEY')
@@ -373,10 +360,6 @@ class Test(Development):
     ANTIVIRUS_ENABLED = True
     DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com']
 
-    FIRETEXT_INBOUND_SMS_AUTH = ['testkey']
-    MMG_INBOUND_SMS_AUTH = ['testkey']
-    MMG_INBOUND_SMS_USERNAME = ['username']
-
     HIGH_VOLUME_SERVICE = [
         '941b6f9a-50d7-4742-8d50-f365ca74bf27',
         '63f95b86-2d19-4497-b8b2-ccf25457df4e',
diff --git a/app/models.py b/app/models.py
index 15b1ecb51..c0399430a 100644
--- a/app/models.py
+++ b/app/models.py
@@ -1116,12 +1116,10 @@ class TemplateHistory(TemplateBase):
         )
 
 
-MMG_PROVIDER = "mmg"
-FIRETEXT_PROVIDER = "firetext"
 SNS_PROVIDER = 'sns'
 SES_PROVIDER = 'ses'
 
-SMS_PROVIDERS = [MMG_PROVIDER, FIRETEXT_PROVIDER, SNS_PROVIDER]
+SMS_PROVIDERS = [SNS_PROVIDER]
 EMAIL_PROVIDERS = [SES_PROVIDER]
 PROVIDERS = SMS_PROVIDERS + EMAIL_PROVIDERS
 
diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py
index 03009a791..9343aa1d5 100644
--- a/app/notifications/receive_notifications.py
+++ b/app/notifications/receive_notifications.py
@@ -1,8 +1,4 @@
-from datetime import datetime
-from urllib.parse import unquote
-
-import iso8601
-from flask import Blueprint, abort, current_app, json, jsonify, request
+from flask import Blueprint, current_app, json, jsonify, request
 from gds_metrics.metrics import Counter
 from notifications_utils.recipients import try_validate_and_format_phone_number
 
@@ -93,113 +89,10 @@ def receive_sns_sms():
     ), 200
 
 
-@receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST'])
-def receive_mmg_sms():
-    """
-    {
-        'MSISDN': '447123456789'
-        'Number': '40604',
-        'Message': 'some+uri+encoded+message%3A',
-        'ID': 'SOME-MMG-SPECIFIC-ID',
-        'DateRecieved': '2017-05-21+11%3A56%3A11'
-    }
-    """
-    post_data = request.get_json()
-
-    auth = request.authorization
-
-    if not auth:
-        current_app.logger.warning("Inbound sms (MMG) no auth header")
-        abort(401)
-    elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \
-            or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']:
-        current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username))
-        abort(403)
-
-    inbound_number = strip_leading_forty_four(post_data['Number'])
-
-    service = fetch_potential_service(inbound_number, 'mmg')
-    if not service:
-        # since this is an issue with our service <-> number mapping, or no inbound_sms service permission
-        # we should still tell MMG that we received it successfully
-        return 'RECEIVED', 200
-
-    INBOUND_SMS_COUNTER.labels("mmg").inc()
-
-    inbound = create_inbound_sms_object(service,
-                                        content=format_mmg_message(post_data["Message"]),
-                                        from_number=post_data['MSISDN'],
-                                        provider_ref=post_data["ID"],
-                                        date_received=post_data.get('DateRecieved'),
-                                        provider_name="mmg")
-
-    tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
-
-    current_app.logger.debug(
-        '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference))
-    return jsonify({
-        "status": "ok"
-    }), 200
-
-
-@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST'])
-def receive_firetext_sms():
-    post_data = request.form
-
-    auth = request.authorization
-    if not auth:
-        current_app.logger.warning("Inbound sms (Firetext) no auth header")
-        abort(401)
-    elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']:
-        current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username))
-        abort(403)
-
-    inbound_number = strip_leading_forty_four(post_data['destination'])
-
-    service = fetch_potential_service(inbound_number, 'firetext')
-    if not service:
-        return jsonify({
-            "status": "ok"
-        }), 200
-
-    inbound = create_inbound_sms_object(service=service,
-                                        content=post_data["message"],
-                                        from_number=post_data['source'],
-                                        provider_ref=None,
-                                        date_received=post_data['time'],
-                                        provider_name="firetext")
-
-    INBOUND_SMS_COUNTER.labels("firetext").inc()
-
-    tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
-    current_app.logger.debug(
-        '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference))
-    return jsonify({
-        "status": "ok"
-    }), 200
-
-
-def format_mmg_message(message):
-    return unescape_string(unquote(message.replace('+', ' ')))
-
-
 def unescape_string(string):
     return string.encode('raw_unicode_escape').decode('unicode_escape')
 
 
-def format_mmg_datetime(date):
-    """
-    We expect datetimes in format 2017-05-21+11%3A56%3A11 - ie, spaces replaced with pluses, and URI encoded
-    and in UTC
-    """
-    try:
-        orig_date = format_mmg_message(date)
-        parsed_datetime = iso8601.parse_date(orig_date).replace(tzinfo=None)
-        return parsed_datetime
-    except iso8601.ParseError:
-        return datetime.utcnow()
-
-
 def create_inbound_sms_object(service, content, from_number, provider_ref, date_received, provider_name):
     user_number = try_validate_and_format_phone_number(
         from_number,
@@ -208,9 +101,6 @@ def create_inbound_sms_object(service, content, from_number, provider_ref, date_
     )
 
     provider_date = date_received
-    if provider_date:
-        provider_date = format_mmg_datetime(provider_date)
-
     inbound = InboundSms(
         service=service,
         notify_number=service.get_inbound_number(),
@@ -246,7 +136,9 @@ def has_inbound_sms_permissions(permissions):
     return set([INBOUND_SMS_TYPE, SMS_TYPE]).issubset(set(str_permissions))
 
 
-def strip_leading_forty_four(number):
-    if number.startswith('44'):
-        return number.replace('44', '0', 1)
+def strip_leading_plus_one(number):
+    if number.startswith('+'):
+        number = number.replace('+', '', 1)
+    if number.startswith('1'):
+        number = number.replace('1', '', 1)
     return number
diff --git a/pytest.ini b/pytest.ini
index 06dd1145f..6578fbca8 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -2,8 +2,6 @@
 testpaths = tests
 env =
     NOTIFY_ENVIRONMENT=test
-    MMG_API_KEY=mmg-secret-key
-    FIRETEXT_API_KEY=Firetext
     REDIS_ENABLED=0
 addopts = -p no:warnings
 xfail_strict = true
diff --git a/tests/app/notifications/test_receive_notification.py b/tests/app/notifications/test_receive_notification.py
index 56efffdfd..61a967d58 100644
--- a/tests/app/notifications/test_receive_notification.py
+++ b/tests/app/notifications/test_receive_notification.py
@@ -1,17 +1,14 @@
-import base64
+from base64 import b64encode
 from datetime import datetime
 
 import pytest
 from flask import json
-from freezegun import freeze_time
 
 from app.models import EMAIL_TYPE, INBOUND_SMS_TYPE, SMS_TYPE, InboundSms
 from app.notifications.receive_notifications import (
     create_inbound_sms_object,
-    format_mmg_datetime,
-    format_mmg_message,
     has_inbound_sms_permissions,
-    strip_leading_forty_four,
+    strip_leading_plus_one,
     unescape_string,
 )
 from tests.app.db import (
@@ -22,57 +19,41 @@ from tests.app.db import (
 from tests.conftest import set_config
 
 
-def firetext_post(client, data, auth=True, password='testkey'):
-    headers = [
-        ('Content-Type', 'application/x-www-form-urlencoded'),
-    ]
-
-    if auth:
-        auth_value = base64.b64encode("notify:{}".format(password).encode('utf-8')).decode('utf-8')
-        headers.append(('Authorization', 'Basic ' + auth_value))
-
-    return client.post(
-        path='/notifications/sms/receive/firetext',
-        data=data,
-        headers=headers
-    )
-
-
-def mmg_post(client, data, auth=True, password='testkey'):
+def sns_post(client, data, auth=True, password='testkey'):
     headers = [
         ('Content-Type', 'application/json'),
     ]
 
     if auth:
-        auth_value = base64.b64encode("username:{}".format(password).encode('utf-8')).decode('utf-8')
-        headers.append(('Authorization', 'Basic ' + auth_value))
+        auth_value = b64encode(f"notify:{password}".encode())
+        headers.append(('Authorization', f"Basic {auth_value}"))
 
     return client.post(
-        path='/notifications/sms/receive/mmg',
-        data=json.dumps(data),
+        path='/notifications/sms/receive/sns',
+        data={"Message": data},
         headers=headers
     )
 
 
-def test_receive_notification_returns_received_to_mmg(client, mocker, sample_service_full_permissions):
+@pytest.mark.skip(reason="Need to implement SNS tests. Body here mostly from MMG")
+def test_receive_notification_returns_received_to_sns(client, mocker, sample_service_full_permissions):
     mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
     prom_counter_labels_mock = mocker.patch('app.notifications.receive_notifications.INBOUND_SMS_COUNTER.labels')
     data = {
-        "ID": "1234",
-        "MSISDN": "447700900855",
-        "Message": "Some message to notify",
-        "Trigger": "Trigger?",
-        "Number": sample_service_full_permissions.get_inbound_number(),
-        "Channel": "SMS",
-        "DateRecieved": "2012-06-27 12:33:00"
+        "originationNumber": "+15558675309",
+        "destinationNumber": sample_service_full_permissions.get_inbound_number(),
+        "messageKeyword": "JOIN",
+        "messageBody": "EXAMPLE",
+        "inboundMessageId": "cae173d2-66b9-564c-8309-21f858e9fb84",
+        "previousPublishedMessageId": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
     }
-    response = mmg_post(client, data)
+    response = sns_post(client, data)
 
     assert response.status_code == 200
     result = json.loads(response.get_data(as_text=True))
-    assert result['status'] == 'ok'
+    assert result['result'] == 'success'
 
-    prom_counter_labels_mock.assert_called_once_with("mmg")
+    prom_counter_labels_mock.assert_called_once_with("sns")
     prom_counter_labels_mock.return_value.inc.assert_called_once_with()
 
     inbound_sms_id = InboundSms.query.all()[0].id
@@ -84,7 +65,7 @@ def test_receive_notification_returns_received_to_mmg(client, mocker, sample_ser
     [SMS_TYPE],
     [INBOUND_SMS_TYPE],
 ])
-def test_receive_notification_from_mmg_without_permissions_does_not_persist(
+def test_receive_notification_from_sns_without_permissions_does_not_persist(
     client,
     mocker,
     notify_db_session,
@@ -101,42 +82,17 @@ def test_receive_notification_from_mmg_without_permissions_does_not_persist(
         "Channel": "SMS",
         "DateRecieved": "2012-06-27 12:33:00"
     }
-    response = mmg_post(client, data)
-
+    response = sns_post(client, data)
     assert response.status_code == 200
-    assert response.get_data(as_text=True) == 'RECEIVED'
+
+    parsed_response = json.loads(response.get_data(as_text=True))
+    assert parsed_response['result'] == 'success'
+
     assert InboundSms.query.count() == 0
     assert mocked.called is False
 
 
-@pytest.mark.parametrize('permissions', [
-    [SMS_TYPE],
-    [INBOUND_SMS_TYPE],
-])
-def test_receive_notification_from_firetext_without_permissions_does_not_persist(
-    client,
-    mocker,
-    notify_db_session,
-    permissions
-):
-    service = create_service_with_inbound_number(inbound_number='07111111111', service_permissions=permissions)
-    mocker.patch("app.notifications.receive_notifications.dao_fetch_service_by_inbound_number",
-                 return_value=service)
-    mocked_send_inbound_sms = mocker.patch(
-        "app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
-    mocker.patch("app.notifications.receive_notifications.has_inbound_sms_permissions", return_value=False)
-
-    data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"
-    response = firetext_post(client, data)
-
-    assert response.status_code == 200
-    result = json.loads(response.get_data(as_text=True))
-
-    assert result['status'] == 'ok'
-    assert InboundSms.query.count() == 0
-    assert not mocked_send_inbound_sms.called
-
-
+@pytest.mark.skip(reason="Need to implement inbound SNS tests. Body here from MMG")
 def test_receive_notification_without_permissions_does_not_create_inbound_even_with_inbound_number_set(
         client, mocker, sample_service):
     inbound_number = create_inbound_number('1', service_id=sample_service.id, active=True)
@@ -156,7 +112,7 @@ def test_receive_notification_without_permissions_does_not_create_inbound_even_w
         "DateRecieved": "2012-06-27 12:33:00"
     }
 
-    response = mmg_post(client, data)
+    response = sns_post(client, data)
 
     assert response.status_code == 200
     assert len(InboundSms.query.all()) == 0
@@ -174,17 +130,6 @@ def test_check_permissions_for_inbound_sms(notify_db_session, permissions, expec
     assert has_inbound_sms_permissions(service.permissions) is expected_response
 
 
-@pytest.mark.parametrize('message, expected_output', [
-    ('abc', 'abc'),
-    ('', ''),
-    ('lots+of+words', 'lots of words'),
-    ('%F0%9F%93%A9+%F0%9F%93%A9+%F0%9F%93%A9', '📩 📩 📩'),
-    ('x+%2B+y', 'x + y')
-])
-def test_format_mmg_message(message, expected_output):
-    assert format_mmg_message(message) == expected_output
-
-
 @pytest.mark.parametrize('raw, expected', [
     (
         '😬',
@@ -215,20 +160,8 @@ def test_unescape_string(raw, expected):
     assert unescape_string(raw) == expected
 
 
-@pytest.mark.parametrize('provider_date, expected_output', [
-    ('2017-01-21+11%3A56%3A11', datetime(2017, 1, 21, 11, 56, 11)),
-    ('2017-05-21+11%3A56%3A11', datetime(2017, 5, 21, 11, 56, 11))
-])
-def test_format_mmg_datetime(provider_date, expected_output):
-    assert format_mmg_datetime(provider_date) == expected_output
-
-
-@freeze_time('2020-05-14 14:30:00')
-def test_format_mmg_datetime_returns_now_if_cannot_parse_date():
-    assert format_mmg_datetime('13-05-2020 08%3A37%3A43') == datetime.utcnow()
-
-
-def test_create_inbound_mmg_sms_object(sample_service_full_permissions):
+@pytest.mark.skip(reason="Need to implement inbound SNS tests. Body here from MMG")
+def test_create_inbound_sns_sms_object(sample_service_full_permissions):
     data = {
         'Message': 'hello+there+%F0%9F%93%A9',
         'Number': sample_service_full_permissions.get_inbound_number(),
@@ -237,8 +170,8 @@ def test_create_inbound_mmg_sms_object(sample_service_full_permissions):
         'ID': 'bar',
     }
 
-    inbound_sms = create_inbound_sms_object(sample_service_full_permissions, format_mmg_message(data["Message"]),
-                                            data["MSISDN"], data["ID"], data["DateRecieved"], "mmg")
+    inbound_sms = create_inbound_sms_object(sample_service_full_permissions, data["Message"],
+                                            data["MSISDN"], data["ID"], data["DateRecieved"], "sns")
 
     assert inbound_sms.service_id == sample_service_full_permissions.id
     assert inbound_sms.notify_number == sample_service_full_permissions.get_inbound_number()
@@ -247,10 +180,11 @@ def test_create_inbound_mmg_sms_object(sample_service_full_permissions):
     assert inbound_sms.provider_reference == 'bar'
     assert inbound_sms._content != 'hello there 📩'
     assert inbound_sms.content == 'hello there 📩'
-    assert inbound_sms.provider == 'mmg'
+    assert inbound_sms.provider == 'sns'
 
 
-def test_create_inbound_mmg_sms_object_uses_inbound_number_if_set(sample_service_full_permissions):
+@pytest.mark.skip(reason="Need to implement inbound SNS tests. Body here from MMG")
+def test_create_inbound_sns_sms_object_uses_inbound_number_if_set(sample_service_full_permissions):
     sample_service_full_permissions.sms_sender = 'foo'
     inbound_number = sample_service_full_permissions.get_inbound_number()
 
@@ -264,17 +198,18 @@ def test_create_inbound_mmg_sms_object_uses_inbound_number_if_set(sample_service
 
     inbound_sms = create_inbound_sms_object(
         sample_service_full_permissions,
-        format_mmg_message(data["Message"]),
+        data["Message"],
         data["MSISDN"],
         data["ID"],
         data["DateRecieved"],
-        "mmg"
+        "sns"
     )
 
     assert inbound_sms.service_id == sample_service_full_permissions.id
     assert inbound_sms.notify_number == inbound_number
 
 
+@pytest.mark.skip(reason="Need to implement inbound SNS tests. Body here from MMG")
 @pytest.mark.parametrize('notify_number', ['foo', 'baz'], ids=['two_matching_services', 'no_matching_services'])
 def test_receive_notification_error_if_not_single_matching_service(client, notify_db_session, notify_number):
     create_service_with_inbound_number(
@@ -295,7 +230,7 @@ def test_receive_notification_error_if_not_single_matching_service(client, notif
         'DateRecieved': '2017-01-02 03:04:05',
         'ID': 'bar',
     }
-    response = mmg_post(client, data)
+    response = sns_post(client, data)
 
     # we still return 'RECEIVED' to MMG
     assert response.status_code == 200
@@ -303,131 +238,20 @@ def test_receive_notification_error_if_not_single_matching_service(client, notif
     assert InboundSms.query.count() == 0
 
 
-def test_receive_notification_returns_received_to_firetext(notify_db_session, client, mocker):
-    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
-    prom_counter_labels_mock = mocker.patch('app.notifications.receive_notifications.INBOUND_SMS_COUNTER.labels')
-
-    service = create_service_with_inbound_number(
-        service_name='b', inbound_number='07111111111', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])
-
-    data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"
-
-    response = firetext_post(client, data)
-
-    assert response.status_code == 200
-    result = json.loads(response.get_data(as_text=True))
-
-    prom_counter_labels_mock.assert_called_once_with("firetext")
-    prom_counter_labels_mock.return_value.inc.assert_called_once_with()
-
-    assert result['status'] == 'ok'
-    inbound_sms_id = InboundSms.query.all()[0].id
-    mocked.assert_called_once_with([str(inbound_sms_id), str(service.id)], queue="notify-internal-tasks")
-
-
-def test_receive_notification_from_firetext_persists_message(notify_db_session, client, mocker):
-    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
-    mocker.patch('app.notifications.receive_notifications.INBOUND_SMS_COUNTER')
-
-    service = create_service_with_inbound_number(
-        inbound_number='07111111111',
-        service_name='b',
-        service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])
-
-    data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"
-
-    response = firetext_post(client, data)
-
-    assert response.status_code == 200
-    result = json.loads(response.get_data(as_text=True))
-
-    persisted = InboundSms.query.first()
-    assert result['status'] == 'ok'
-    assert persisted.notify_number == '07111111111'
-    assert persisted.user_number == '447999999999'
-    assert persisted.service == service
-    assert persisted.content == 'this is a message'
-    assert persisted.provider == 'firetext'
-    assert persisted.provider_date == datetime(2017, 1, 1, 12, 0, 0, 0)
-    mocked.assert_called_once_with([str(persisted.id), str(service.id)], queue="notify-internal-tasks")
-
-
-def test_receive_notification_from_firetext_persists_message_with_normalized_phone(notify_db_session, client, mocker):
-    mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
-    mocker.patch('app.notifications.receive_notifications.INBOUND_SMS_COUNTER')
-
-    create_service_with_inbound_number(
-        inbound_number='07111111111', service_name='b', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])
-
-    data = "source=(+44)7999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"
-
-    response = firetext_post(client, data)
-
-    assert response.status_code == 200
-    result = json.loads(response.get_data(as_text=True))
-
-    persisted = InboundSms.query.first()
-
-    assert result['status'] == 'ok'
-    assert persisted.user_number == '447999999999'
-
-
-def test_returns_ok_to_firetext_if_mismatched_sms_sender(notify_db_session, client, mocker):
-    mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
-    mocker.patch('app.notifications.receive_notifications.INBOUND_SMS_COUNTER')
-
-    create_service_with_inbound_number(
-        inbound_number='07111111199', service_name='b', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE])
-
-    data = "source=(+44)7999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"
-
-    response = firetext_post(client, data)
-
-    assert response.status_code == 200
-    result = json.loads(response.get_data(as_text=True))
-
-    assert not InboundSms.query.all()
-    assert result['status'] == 'ok'
-    assert mocked.call_count == 0
-
-
 @pytest.mark.parametrize(
     'number, expected',
     [
-        ('447123123123', '07123123123'),
-        ('447123123144', '07123123144'),
-        ('07123123123', '07123123123'),
-        ('447444444444', '07444444444')
+        ('15558675309', '5558675309'),
+        ('+15558675309', '5558675309'),
+        ('5558675309', '5558675309'),
+        ('15111111111', '5111111111')
     ]
 )
 def test_strip_leading_country_code(number, expected):
-    assert strip_leading_forty_four(number) == expected
-
-
-@pytest.mark.parametrize("auth, keys, status_code", [
-    ["testkey", ["testkey"], 200],
-    ["", ["testkey"], 401],
-    ["wrong", ["testkey"], 403],
-    ["testkey1", ["testkey1", "testkey2"], 200],
-    ["testkey2", ["testkey1", "testkey2"], 200],
-    ["wrong", ["testkey1", "testkey2"], 403],
-    ["", [], 401],
-    ["testkey", [], 403],
-])
-def test_firetext_inbound_sms_auth(notify_db_session, notify_api, client, mocker, auth, keys, status_code):
-    mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
-
-    create_service_with_inbound_number(
-        service_name='b', inbound_number='07111111111', service_permissions=[EMAIL_TYPE, SMS_TYPE, INBOUND_SMS_TYPE]
-    )
-
-    data = "source=07999999999&destination=07111111111&message=this is a message&time=2017-01-01 12:00:00"
-
-    with set_config(notify_api, 'FIRETEXT_INBOUND_SMS_AUTH', keys):
-        response = firetext_post(client, data, auth=bool(auth), password=auth)
-        assert response.status_code == status_code
+    assert strip_leading_plus_one(number) == expected
 
 
+@pytest.mark.skip(reason="Need to implement inbound SNS tests. Body here from MMG")
 @pytest.mark.parametrize("auth, keys, status_code", [
     ["testkey", ["testkey"], 200],
     ["", ["testkey"], 401],
@@ -456,10 +280,11 @@ def test_mmg_inbound_sms_auth(notify_db_session, notify_api, client, mocker, aut
     }
 
     with set_config(notify_api, 'MMG_INBOUND_SMS_AUTH', keys):
-        response = mmg_post(client, data, auth=bool(auth), password=auth)
+        response = sns_post(client, data, auth=bool(auth), password=auth)
         assert response.status_code == status_code
 
 
+@pytest.mark.skip(reason="Need to implement inbound SNS tests. Body here from MMG")
 def test_create_inbound_sms_object_works_with_alphanumeric_sender(sample_service_full_permissions):
     data = {
         'Message': 'hello',
@@ -471,7 +296,7 @@ def test_create_inbound_sms_object_works_with_alphanumeric_sender(sample_service
 
     inbound_sms = create_inbound_sms_object(
         service=sample_service_full_permissions,
-        content=format_mmg_message(data["Message"]),
+        content=data["Message"],
         from_number='ALPHANUM3R1C',
         provider_ref='foo',
         date_received=None,

From 041cd0809736d2fe4715b4995d808b2057f26994 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Mon, 19 Dec 2022 10:48:53 -0500
Subject: [PATCH 15/46] Clean up more mmg and firetext references

---
 .../process_sms_client_response_tasks.py      |   3 +-
 app/commands.py                               |   5 +-
 app/dao/notifications_dao.py                  |   5 +-
 app/dao/provider_details_dao.py               |   6 +-
 app/delivery/send_to_providers.py             |   6 +-
 .../0382_remove_old_sms_providers_.py         |  22 ++++
 tests/app/celery/test_reporting_tasks.py      |  12 +-
 tests/app/celery/test_tasks.py                |   3 -
 tests/app/conftest.py                         |   9 +-
 .../notification_dao/test_notification_dao.py |  25 ++---
 tests/app/dao/test_fact_billing_dao.py        |  10 +-
 tests/app/dao/test_provider_details_dao.py    | 105 +++++++-----------
 tests/app/inbound_number/test_rest.py         |   4 +-
 tests/app/provider_details/test_rest.py       |   4 +-
 tests/app/test_model.py                       |   6 +-
 .../notifications/test_get_notifications.py   |   4 +-
 tests/conftest.py                             |   5 +-
 17 files changed, 106 insertions(+), 128 deletions(-)
 create mode 100644 migrations/versions/0382_remove_old_sms_providers_.py

diff --git a/app/celery/process_sms_client_response_tasks.py b/app/celery/process_sms_client_response_tasks.py
index 8bdf84301..2a1cc4dd4 100644
--- a/app/celery/process_sms_client_response_tasks.py
+++ b/app/celery/process_sms_client_response_tasks.py
@@ -12,8 +12,7 @@ from app.dao.templates_dao import dao_get_template_by_id
 from app.models import NOTIFICATION_PENDING
 
 sms_response_mapper = {
-    # 'MMG': get_mmg_responses,
-    # 'Firetext': get_firetext_responses,
+    # 'SNS': get_sns_responses,
 }
 
 
diff --git a/app/commands.py b/app/commands.py
index 11439fb98..1718ea82d 100644
--- a/app/commands.py
+++ b/app/commands.py
@@ -147,12 +147,11 @@ def purge_functional_test_data(user_email_prefix):
 
 @notify_command(name='insert-inbound-numbers')
 @click.option('-f', '--file_name', required=True,
-              help="""Full path of the file to upload, file is a contains inbound numbers,
-              one number per line. The number must have the format of 07... not 447....""")
+              help="""Full path of the file to upload, file is a contains inbound numbers, one number per line.""")
 def insert_inbound_numbers_from_file(file_name):
     print("Inserting inbound numbers from {}".format(file_name))
     with open(file_name) as file:
-        sql = "insert into inbound_numbers values('{}', '{}', 'mmg', null, True, now(), null);"
+        sql = "insert into inbound_numbers values('{}', '{}', 'sns', null, True, now(), null);"
 
         for line in file:
             line = line.strip()
diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py
index 47db5aba7..22f04592c 100644
--- a/app/dao/notifications_dao.py
+++ b/app/dao/notifications_dao.py
@@ -104,7 +104,7 @@ def _update_notification_status(notification, status, provider_response=None):
 
 
 @autocommit
-def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None):
+def update_notification_status_by_id(notification_id, status, sent_by=None):
     notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()
 
     if not notification:
@@ -134,8 +134,7 @@ def update_notification_status_by_id(notification_id, status, sent_by=None, deta
         notification.sent_by = sent_by
     return _update_notification_status(
         notification=notification,
-        status=status,
-        detailed_status_code=detailed_status_code
+        status=status
     )
 
 
diff --git a/app/dao/provider_details_dao.py b/app/dao/provider_details_dao.py
index 15ed99505..2fc03d424 100644
--- a/app/dao/provider_details_dao.py
+++ b/app/dao/provider_details_dao.py
@@ -24,10 +24,8 @@ def get_provider_details_by_identifier(identifier):
 
 
 def get_alternative_sms_provider(identifier):
-    if identifier == 'firetext':
-        return 'mmg'
-    elif identifier == 'mmg':
-        return 'firetext'
+    if identifier == 'sns':
+        raise Exception("No alternative SMS providers currently available")
     raise ValueError('Unrecognised sms provider {}'.format(identifier))
 
 
diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py
index c1532e83e..84feb52cc 100644
--- a/app/delivery/send_to_providers.py
+++ b/app/delivery/send_to_providers.py
@@ -111,7 +111,7 @@ def send_email_to_provider(notification):
         technical_failure(notification=notification)
         return
     if notification.status == 'created':
-        provider = provider_to_use(EMAIL_TYPE)
+        provider = provider_to_use(EMAIL_TYPE, False)
 
         template_dict = SerialisedTemplate.from_id_and_service_id(
             template_id=notification.template_id, service_id=service.id, version=notification.template_version
@@ -175,12 +175,10 @@ provider_cache = TTLCache(maxsize=8, ttl=10)
 
 @cached(cache=provider_cache)
 def provider_to_use(notification_type, international=True):
-    international = False  # TODO: remove or resolve the functionality of this flag
-    # TODO rip firetext and mmg out of early migrations and clean up the expression below
     active_providers = [
         p for p in get_provider_details_by_notification_type(
             notification_type, international
-        ) if p.active and p.identifier not in ['firetext', 'mmg']
+        ) if p.active
     ]
 
     if not active_providers:
diff --git a/migrations/versions/0382_remove_old_sms_providers_.py b/migrations/versions/0382_remove_old_sms_providers_.py
new file mode 100644
index 000000000..f33f1b0d8
--- /dev/null
+++ b/migrations/versions/0382_remove_old_sms_providers_.py
@@ -0,0 +1,22 @@
+"""
+
+Revision ID: 0382_remove_old_sms_providers
+Revises: 0381_encrypted_column_types
+Create Date: 2022-12-16 12:52:14.182717
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = '0382_remove_old_providers'
+down_revision = '0381_encrypted_column_types'
+
+
+def upgrade():
+    op.execute("DELETE FROM provider_details WHERE identifier IN ('mmg', 'firetext')")
+    op.execute("DELETE FROM provider_details_history WHERE identifier IN ('mmg', 'firetext')")
+
+
+def downgrade():
+    raise Exception("Irreversible migration")
diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py
index 6c1e8ded2..5cdc55733 100644
--- a/tests/app/celery/test_reporting_tasks.py
+++ b/tests/app/celery/test_reporting_tasks.py
@@ -170,7 +170,7 @@ def test_create_nightly_billing_for_day_sms_rate_multiplier(
         created_at=yesterday,
         template=sample_template,
         status='delivered',
-        sent_by='mmg',
+        sent_by='sns',
         international=False,
         rate_multiplier=1.0,
         billable_units=1,
@@ -179,7 +179,7 @@ def test_create_nightly_billing_for_day_sms_rate_multiplier(
         created_at=yesterday,
         template=sample_template,
         status='delivered',
-        sent_by='mmg',
+        sent_by='sns',
         international=False,
         rate_multiplier=second_rate,
         billable_units=1,
@@ -214,7 +214,7 @@ def test_create_nightly_billing_for_day_different_templates(
         created_at=yesterday,
         template=sample_template,
         status='delivered',
-        sent_by='mmg',
+        sent_by='sns',
         international=False,
         rate_multiplier=1.0,
         billable_units=1,
@@ -223,7 +223,7 @@ def test_create_nightly_billing_for_day_different_templates(
         created_at=yesterday,
         template=sample_email_template,
         status='delivered',
-        sent_by='ses',
+        sent_by='sns',
         international=False,
         rate_multiplier=0,
         billable_units=0,
@@ -262,7 +262,7 @@ def test_create_nightly_billing_for_day_different_sent_by(
         created_at=yesterday,
         template=sample_template,
         status='delivered',
-        sent_by='mmg',
+        sent_by='sns',
         international=False,
         rate_multiplier=1.0,
         billable_units=1,
@@ -271,7 +271,7 @@ def test_create_nightly_billing_for_day_different_sent_by(
         created_at=yesterday,
         template=sample_template,
         status='delivered',
-        sent_by='firetext',
+        sent_by='sns',
         international=False,
         rate_multiplier=1.0,
         billable_units=1,
diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py
index 7fda9fd49..5c55a439d 100644
--- a/tests/app/celery/test_tasks.py
+++ b/tests/app/celery/test_tasks.py
@@ -75,9 +75,6 @@ class AnyStringWith(str):
         return self in other
 
 
-mmg_error = {'Error': '40', 'Description': 'error'}
-
-
 def _notification_json(template, to, personalisation=None, job_id=None, row_number=0):
     return {
         "template": str(template.id),
diff --git a/tests/app/conftest.py b/tests/app/conftest.py
index 7cd374b72..ba4a6419b 100644
--- a/tests/app/conftest.py
+++ b/tests/app/conftest.py
@@ -626,8 +626,8 @@ def ses_provider():
 
 
 @pytest.fixture(scope='function')
-def mmg_provider():
-    return ProviderDetails.query.filter_by(identifier='mmg').one()
+def sns_provider():
+    return ProviderDetails.query.filter_by(identifier='sns').one()
 
 
 @pytest.fixture(scope='function')
@@ -888,9 +888,8 @@ def sample_service_guest_list(notify_db_session):
 def sample_inbound_numbers(sample_service):
     service = create_service(service_name='sample service 2', check_if_service_exists=True)
     inbound_numbers = list()
-    inbound_numbers.append(create_inbound_number(number='1', provider='mmg'))
-    inbound_numbers.append(create_inbound_number(number='2', provider='mmg', active=False, service_id=service.id))
-    inbound_numbers.append(create_inbound_number(number='3', provider='firetext', service_id=sample_service.id))
+    inbound_numbers.append(create_inbound_number(number='1', provider='sns'))
+    inbound_numbers.append(create_inbound_number(number='2', provider='sns', active=False, service_id=service.id))
     return inbound_numbers
 
 
diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py
index b0dfd8482..49e9b92c0 100644
--- a/tests/app/dao/notification_dao/test_notification_dao.py
+++ b/tests/app/dao/notification_dao/test_notification_dao.py
@@ -53,7 +53,6 @@ from tests.app.db import (
 )
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_by_able_to_update_status_by_reference(sample_email_template, ses_provider):
     data = _notification_json(sample_email_template, status='sending')
 
@@ -69,8 +68,7 @@ def test_should_by_able_to_update_status_by_reference(sample_email_template, ses
     assert Notification.query.get(notification.id).status == 'delivered'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
-def test_should_by_able_to_update_status_by_id(sample_template, sample_job, mmg_provider):
+def test_should_by_able_to_update_status_by_id(sample_template, sample_job, sns_provider):
     with freeze_time('2000-01-01 12:00:00'):
         data = _notification_json(sample_template, job_id=sample_job.id, status='sending')
         notification = Notification(**data)
@@ -107,7 +105,6 @@ def test_should_not_update_status_by_reference_if_not_sending_and_does_not_updat
     assert sample_job == Job.query.get(notification.job_id)
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_update_status_by_id_if_created(sample_template, sample_notification):
     assert Notification.query.get(sample_notification.id).status == 'created'
     updated = update_notification_status_by_id(sample_notification.id, 'failed')
@@ -115,7 +112,6 @@ def test_should_update_status_by_id_if_created(sample_template, sample_notificat
     assert updated.status == 'failed'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Remove letters")
 def test_should_update_status_by_id_if_pending_virus_check(sample_letter_template):
     notification = create_notification(template=sample_letter_template, status='pending-virus-check')
     assert Notification.query.get(notification.id).status == 'pending-virus-check'
@@ -124,7 +120,6 @@ def test_should_update_status_by_id_if_pending_virus_check(sample_letter_templat
     assert updated.status == 'cancelled'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Remove letters")
 def test_should_update_status_of_international_letter_to_cancelled(sample_letter_template):
     notification = create_notification(
         template=sample_letter_template,
@@ -136,13 +131,12 @@ def test_should_update_status_of_international_letter_to_cancelled(sample_letter
     assert Notification.query.get(notification.id).status == 'cancelled'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_update_status_by_id_and_set_sent_by(sample_template):
     notification = create_notification(template=sample_template, status='sending')
 
-    updated = update_notification_status_by_id(notification.id, 'delivered', sent_by='mmg')
+    updated = update_notification_status_by_id(notification.id, 'delivered', sent_by='sns')
     assert updated.status == 'delivered'
-    assert updated.sent_by == 'mmg'
+    assert updated.sent_by == 'sns'
 
 
 def test_should_not_update_status_by_reference_if_from_country_with_no_delivery_receipts(sample_template):
@@ -186,7 +180,6 @@ def test_should_not_update_status_by_id_if_sent_to_country_with_carrier_delivery
     assert notification.status == NOTIFICATION_SENT
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_not_update_status_by_id_if_sent_to_country_with_delivery_receipts(sample_template):
     notification = create_notification(
         sample_template,
@@ -209,7 +202,6 @@ def test_should_not_update_status_by_reference_if_not_sending(sample_template):
     assert not updated
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_by_able_to_update_status_by_id_from_pending_to_delivered(sample_template, sample_job):
     notification = create_notification(template=sample_template, job=sample_job, status='sending')
 
@@ -220,7 +212,6 @@ def test_should_by_able_to_update_status_by_id_from_pending_to_delivered(sample_
     assert Notification.query.get(notification.id).status == 'delivered'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_by_able_to_update_status_by_id_from_pending_to_temporary_failure(sample_template, sample_job):
     notification = create_notification(template=sample_template, job=sample_job, status='sending', sent_by='sns')
 
@@ -232,7 +223,6 @@ def test_should_by_able_to_update_status_by_id_from_pending_to_temporary_failure
     assert Notification.query.get(notification.id).status == 'temporary-failure'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_by_able_to_update_status_by_id_from_sending_to_permanent_failure(sample_template, sample_job):
     data = _notification_json(sample_template, job_id=sample_job.id, status='sending')
     notification = Notification(**data)
@@ -246,7 +236,6 @@ def test_should_by_able_to_update_status_by_id_from_sending_to_permanent_failure
     assert Notification.query.get(notification.id).status == 'permanent-failure'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_not_update_status_once_notification_status_is_delivered(
         sample_email_template):
     notification = create_notification(template=sample_email_template, status='sending')
@@ -372,7 +361,7 @@ def test_update_notification_with_research_mode_service_does_not_create_or_updat
     assert NotificationHistory.query.count() == 0
 
 
-def test_not_save_notification_and_not_create_stats_on_commit_error(sample_template, sample_job, mmg_provider):
+def test_not_save_notification_and_not_create_stats_on_commit_error(sample_template, sample_job, sns_provider):
     random_id = str(uuid.uuid4())
 
     assert Notification.query.count() == 0
@@ -386,7 +375,7 @@ def test_not_save_notification_and_not_create_stats_on_commit_error(sample_templ
     assert Job.query.get(sample_job.id).notifications_sent == 0
 
 
-def test_save_notification_and_increment_job(sample_template, sample_job, mmg_provider):
+def test_save_notification_and_increment_job(sample_template, sample_job, sns_provider):
     assert Notification.query.count() == 0
     data = _notification_json(sample_template, job_id=sample_job.id)
 
@@ -409,7 +398,7 @@ def test_save_notification_and_increment_job(sample_template, sample_job, mmg_pr
     assert Notification.query.count() == 2
 
 
-def test_save_notification_and_increment_correct_job(sample_template, mmg_provider):
+def test_save_notification_and_increment_correct_job(sample_template, sns_provider):
     job_1 = create_job(sample_template)
     job_2 = create_job(sample_template)
 
@@ -432,7 +421,7 @@ def test_save_notification_and_increment_correct_job(sample_template, mmg_provid
     assert job_1.id != job_2.id
 
 
-def test_save_notification_with_no_job(sample_template, mmg_provider):
+def test_save_notification_with_no_job(sample_template, sns_provider):
     assert Notification.query.count() == 0
     data = _notification_json(sample_template)
 
diff --git a/tests/app/dao/test_fact_billing_dao.py b/tests/app/dao/test_fact_billing_dao.py
index 1a50649bf..f9383cd39 100644
--- a/tests/app/dao/test_fact_billing_dao.py
+++ b/tests/app/dao/test_fact_billing_dao.py
@@ -174,14 +174,14 @@ def test_fetch_billing_data_for_day_is_grouped_by_service(notify_db_session):
 def test_fetch_billing_data_for_day_is_grouped_by_provider(notify_db_session):
     service = create_service()
     template = create_template(service=service)
-    create_notification(template=template, status='delivered', sent_by='mmg')
-    create_notification(template=template, status='delivered', sent_by='firetext')
+    create_notification(template=template, status='delivered', sent_by='sns')
+    create_notification(template=template, status='delivered', sent_by='sns')
 
     today = convert_utc_to_local_timezone(datetime.utcnow())
     results = fetch_billing_data_for_day(today.date())
-    assert len(results) == 2
-    assert results[0].notifications_sent == 1
-    assert results[1].notifications_sent == 1
+    assert len(results) == 1
+    assert results[0].notifications_sent == 2
+    # assert results[1].notifications_sent == 1
 
 
 def test_fetch_billing_data_for_day_is_grouped_by_rate_mulitplier(notify_db_session):
diff --git a/tests/app/dao/test_provider_details_dao.py b/tests/app/dao/test_provider_details_dao.py
index 939f8b9d8..0e41991ba 100644
--- a/tests/app/dao/test_provider_details_dao.py
+++ b/tests/app/dao/test_provider_details_dao.py
@@ -23,7 +23,7 @@ from tests.conftest import set_config
 
 @pytest.fixture(autouse=True)
 def set_provider_resting_points(notify_api):
-    with set_config(notify_api, 'SMS_PROVIDER_RESTING_POINTS', {'mmg': 60, 'firetext': 40}):
+    with set_config(notify_api, 'SMS_PROVIDER_RESTING_POINTS', {'sns': 100}):
         yield
 
 
@@ -69,11 +69,10 @@ def test_can_get_email_providers(notify_db_session):
     assert all('email' == notification_type for notification_type in types)
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_not_error_if_any_provider_in_code_not_in_database(restore_provider_details):
-    ProviderDetails.query.filter_by(identifier='mmg').delete()
+    ProviderDetails.query.filter_by(identifier='sns').delete()
 
-    assert notification_provider_clients.get_sms_client('mmg')
+    assert notification_provider_clients.get_sms_client('sns')
 
 
 @freeze_time('2000-01-01T00:00:00')
@@ -108,20 +107,22 @@ def test_update_adds_history(restore_provider_details):
 
 
 def test_update_sms_provider_to_inactive_sets_inactive(restore_provider_details):
-    mmg = get_provider_details_by_identifier('mmg')
+    sns = get_provider_details_by_identifier('sns')
 
-    mmg.active = False
-    dao_update_provider_details(mmg)
+    sns.active = False
+    dao_update_provider_details(sns)
 
-    assert not mmg.active
+    assert not sns.active
 
 
 @pytest.mark.parametrize('identifier, expected', [
-    ('firetext', 'mmg'),
-    ('mmg', 'firetext'),
+    ('sns', 'other')
 ])
 def test_get_alternative_sms_provider_returns_expected_provider(identifier, expected):
-    assert get_alternative_sms_provider(identifier) == expected
+    """Currently always raises, as we only have SNS configured"""
+    with pytest.raises(Exception):
+        get_alternative_sms_provider(identifier)
+    # assert get_alternative_sms_provider(identifier) == expected
 
 
 def test_get_alternative_sms_provider_fails_if_unrecognised():
@@ -133,38 +134,37 @@ def test_get_alternative_sms_provider_fails_if_unrecognised():
 def test_adjust_provider_priority_sets_priority(
     restore_provider_details,
     notify_user,
-    mmg_provider,
+    sns_provider,
 ):
     # need to update these manually to avoid triggering the `onupdate` clause of the updated_at column
-    ProviderDetails.query.filter(ProviderDetails.identifier == 'mmg').update({'updated_at': datetime.min})
+    ProviderDetails.query.filter(ProviderDetails.identifier == 'sns').update({'updated_at': datetime.min})
 
-    _adjust_provider_priority(mmg_provider, 50)
+    _adjust_provider_priority(sns_provider, 50)
 
-    assert mmg_provider.updated_at == datetime.utcnow()
-    assert mmg_provider.created_by.id == notify_user.id
-    assert mmg_provider.priority == 50
+    assert sns_provider.updated_at == datetime.utcnow()
+    assert sns_provider.created_by.id == notify_user.id
+    assert sns_provider.priority == 50
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
 @freeze_time('2016-01-01 00:30')
 def test_adjust_provider_priority_adds_history(
     restore_provider_details,
     notify_user,
-    mmg_provider,
+    sns_provider,
 ):
     # need to update these manually to avoid triggering the `onupdate` clause of the updated_at column
-    ProviderDetails.query.filter(ProviderDetails.identifier == 'mmg').update({'updated_at': datetime.min})
+    ProviderDetails.query.filter(ProviderDetails.identifier == 'sns').update({'updated_at': datetime.min})
 
     old_provider_history_rows = ProviderDetailsHistory.query.filter(
-        ProviderDetailsHistory.id == mmg_provider.id
+        ProviderDetailsHistory.id == sns_provider.id
     ).order_by(
         desc(ProviderDetailsHistory.version)
     ).all()
 
-    _adjust_provider_priority(mmg_provider, 50)
+    _adjust_provider_priority(sns_provider, 50)
 
     updated_provider_history_rows = ProviderDetailsHistory.query.filter(
-        ProviderDetailsHistory.id == mmg_provider.id
+        ProviderDetailsHistory.id == sns_provider.id
     ).order_by(
         desc(ProviderDetailsHistory.version)
     ).all()
@@ -174,30 +174,26 @@ def test_adjust_provider_priority_adds_history(
     assert updated_provider_history_rows[0].priority == 50
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
 @freeze_time('2016-01-01 01:00')
 def test_get_sms_providers_for_update_returns_providers(restore_provider_details):
-    sixty_one_minutes_ago = datetime(2015, 12, 31, 23, 59)
-    ProviderDetails.query.filter(ProviderDetails.identifier == 'mmg').update({'updated_at': sixty_one_minutes_ago})
-    ProviderDetails.query.filter(ProviderDetails.identifier == 'firetext').update({'updated_at': None})
+    ProviderDetails.query.filter(ProviderDetails.identifier == 'sns').update({'updated_at': None})
 
     resp = _get_sms_providers_for_update(timedelta(hours=1))
 
-    assert {p.identifier for p in resp} == {'mmg', 'firetext'}
+    assert {p.identifier for p in resp} == {'sns'}
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
 @freeze_time('2016-01-01 01:00')
 def test_get_sms_providers_for_update_returns_nothing_if_recent_updates(restore_provider_details):
     fifty_nine_minutes_ago = datetime(2016, 1, 1, 0, 1)
-    ProviderDetails.query.filter(ProviderDetails.identifier == 'mmg').update({'updated_at': fifty_nine_minutes_ago})
+    ProviderDetails.query.filter(ProviderDetails.identifier == 'sns').update({'updated_at': fifty_nine_minutes_ago})
 
     resp = _get_sms_providers_for_update(timedelta(hours=1))
 
     assert not resp
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
+@pytest.mark.skip(reason="Reenable if/when we add a second SMS provider")
 @pytest.mark.parametrize(['starting_priorities', 'expected_priorities'], [
     ({'mmg': 50, 'firetext': 50}, {'mmg': 40, 'firetext': 60}),
     ({'mmg': 0, 'firetext': 20}, {'mmg': 0, 'firetext': 30}),  # lower bound respected
@@ -242,27 +238,24 @@ def test_reduce_sms_provider_priority_does_nothing_if_providers_have_recently_ch
     mock_get_providers = mocker.patch('app.dao.provider_details_dao._get_sms_providers_for_update', return_value=[])
     mock_adjust = mocker.patch('app.dao.provider_details_dao._adjust_provider_priority')
 
-    dao_reduce_sms_provider_priority('firetext', time_threshold=timedelta(minutes=5))
+    dao_reduce_sms_provider_priority('sns', time_threshold=timedelta(minutes=5))
 
     mock_get_providers.assert_called_once_with(timedelta(minutes=5))
     assert mock_adjust.called is False
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_reduce_sms_provider_priority_does_nothing_if_there_is_only_one_active_provider(
     mocker,
     restore_provider_details,
 ):
-    firetext = get_provider_details_by_identifier('firetext')
-    firetext.active = False
-
     mock_adjust = mocker.patch('app.dao.provider_details_dao._adjust_provider_priority')
 
-    dao_reduce_sms_provider_priority('firetext', time_threshold=timedelta(minutes=5))
+    dao_reduce_sms_provider_priority('sns', time_threshold=timedelta(minutes=5))
 
     assert mock_adjust.called is False
 
 
+@pytest.mark.skip(reason="Reenable if/when we add a second SMS provider")
 @pytest.mark.parametrize('existing_mmg, existing_firetext, new_mmg, new_firetext', [
     (50, 50, 60, 40),  # not just 50/50 - 60/40 specifically
     (65, 35, 60, 40),  # doesn't overshoot if there's less than 10 difference
@@ -298,13 +291,11 @@ def test_adjust_provider_priority_back_to_resting_points_does_nothing_if_theyre_
     restore_provider_details,
     mocker,
 ):
-    mmg = get_provider_details_by_identifier('mmg')
-    firetext = get_provider_details_by_identifier('firetext')
-    mmg.priority = 60
-    firetext.priority = 40
+    sns = get_provider_details_by_identifier('sns')
+    sns.priority = 100
 
     mock_adjust = mocker.patch('app.dao.provider_details_dao._adjust_provider_priority')
-    mocker.patch('app.dao.provider_details_dao._get_sms_providers_for_update', return_value=[mmg, firetext])
+    mocker.patch('app.dao.provider_details_dao._get_sms_providers_for_update', return_value=[sns])
 
     dao_adjust_provider_priority_back_to_resting_points()
 
@@ -323,7 +314,6 @@ def test_adjust_provider_priority_back_to_resting_points_does_nothing_if_no_prov
     assert mock_adjust.called is False
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: New SMS provider")
 @freeze_time('2018-06-28 12:00')
 def test_dao_get_provider_stats(notify_db_session):
     service_1 = create_service(service_name='1')
@@ -331,34 +321,23 @@ def test_dao_get_provider_stats(notify_db_session):
     sms_template_1 = create_template(service_1, 'sms')
     sms_template_2 = create_template(service_2, 'sms')
 
-    create_ft_billing('2017-06-05', sms_template_2, provider='firetext', billable_unit=4)
-    create_ft_billing('2018-05-31', sms_template_1, provider='mmg', billable_unit=1)
-    create_ft_billing('2018-06-01', sms_template_1, provider='mmg',
-                      rate_multiplier=2, billable_unit=1)
-    create_ft_billing('2018-06-03', sms_template_2, provider='firetext', billable_unit=4)
-    create_ft_billing('2018-06-15', sms_template_1, provider='firetext', billable_unit=1)
-    create_ft_billing('2018-06-28', sms_template_2, provider='mmg', billable_unit=2)
+    create_ft_billing('2017-06-05', sms_template_2, provider='sns', billable_unit=4)
+    create_ft_billing('2018-06-03', sms_template_2, provider='sns', billable_unit=4)
+    create_ft_billing('2018-06-15', sms_template_1, provider='sns', billable_unit=1)
 
     results = dao_get_provider_stats()
 
     assert len(results) > 0
 
     ses = next(result for result in results if result.identifier == 'ses')
-    firetext = next(result for result in results if result.identifier == 'firetext')
-    mmg = next(result for result in results if result.identifier == 'mmg')
+    sns = next(result for result in results if result.identifier == 'sns')
 
     assert ses.display_name == 'AWS SES'
     assert ses.created_by_name is None
     assert ses.current_month_billable_sms == 0
 
-    assert firetext.display_name == 'Firetext'
-    assert firetext.notification_type == 'sms'
-    assert firetext.supports_international is False
-    assert firetext.active is True
-    assert firetext.current_month_billable_sms == 5
-
-    assert mmg.identifier == 'mmg'
-    assert mmg.display_name == 'MMG'
-    assert mmg.supports_international is True
-    assert mmg.active is True
-    assert mmg.current_month_billable_sms == 4
+    assert sns.display_name == 'AWS SNS'
+    assert sns.notification_type == 'sms'
+    assert sns.supports_international is True
+    assert sns.active is True
+    assert sns.current_month_billable_sms == 5
diff --git a/tests/app/inbound_number/test_rest.py b/tests/app/inbound_number/test_rest.py
index 62e66b51a..90f975271 100644
--- a/tests/app/inbound_number/test_rest.py
+++ b/tests/app/inbound_number/test_rest.py
@@ -16,7 +16,7 @@ def test_rest_get_inbound_numbers(admin_request, sample_inbound_numbers):
 
 
 def test_rest_get_inbound_number(admin_request, notify_db_session, sample_service):
-    inbound_number = create_inbound_number(number='1', provider='mmg', active=False, service_id=sample_service.id)
+    inbound_number = create_inbound_number(number='1', provider='sns', active=False, service_id=sample_service.id)
 
     result = admin_request.get(
         'inbound_number.get_inbound_number_for_service',
@@ -37,7 +37,7 @@ def test_rest_get_inbound_number_when_service_is_not_assigned_returns_empty_dict
 def test_rest_set_inbound_number_active_flag_off(
         admin_request, notify_db_session):
     service = create_service(service_name='test service 1')
-    create_inbound_number(number='1', provider='mmg', active=True, service_id=service.id)
+    create_inbound_number(number='1', provider='sns', active=True, service_id=service.id)
 
     admin_request.post(
         'inbound_number.post_set_inbound_number_off',
diff --git a/tests/app/provider_details/test_rest.py b/tests/app/provider_details/test_rest.py
index 2b347fbba..4e2c001e9 100644
--- a/tests/app/provider_details/test_rest.py
+++ b/tests/app/provider_details/test_rest.py
@@ -11,7 +11,7 @@ def test_get_provider_details_returns_all_providers(admin_request, notify_db_ses
     json_resp = admin_request.get('provider_details.get_providers')['provider_details']
 
     assert len(json_resp) > 0
-    assert {'ses', 'firetext', 'mmg', 'dvla'} <= {x['identifier'] for x in json_resp}
+    assert {'ses', 'sns', 'dvla'} == {x['identifier'] for x in json_resp}
 
 
 def test_get_provider_details_by_id(client, notify_db_session):
@@ -32,7 +32,7 @@ def test_get_provider_details_by_id(client, notify_db_session):
 
 @freeze_time('2018-06-28 12:00')
 def test_get_provider_contains_correct_fields(client, sample_template):
-    create_ft_billing('2018-06-01', sample_template, provider='mmg', billable_unit=1)
+    create_ft_billing('2018-06-01', sample_template, provider='sns', billable_unit=1)
 
     response = client.get(
         '/provider-details',
diff --git a/tests/app/test_model.py b/tests/app/test_model.py
index 3798c358b..ae38ff48a 100644
--- a/tests/app/test_model.py
+++ b/tests/app/test_model.py
@@ -32,8 +32,8 @@ from tests.app.db import (
 
 
 @pytest.mark.parametrize('mobile_number', [
-    '07700 900678',
-    '+44 7700 900678'
+    '5558675309',
+    '+15558675309'
 ])
 def test_should_build_service_guest_list_from_mobile_number(mobile_number):
     service_guest_list = ServiceGuestList.from_string('service_id', MOBILE_TYPE, mobile_number)
@@ -93,7 +93,7 @@ def test_status_conversion(initial_statuses, expected_statuses):
 
 @freeze_time("2016-01-01 11:09:00.000000")
 @pytest.mark.parametrize('template_type, recipient', [
-    ('sms', '+447700900855'),
+    ('sms', '+15558675309'),
     ('email', 'foo@bar.com'),
 ])
 def test_notification_for_csv_returns_correct_type(sample_service, template_type, recipient):
diff --git a/tests/app/v2/notifications/test_get_notifications.py b/tests/app/v2/notifications/test_get_notifications.py
index 7c93f9aad..2129b558a 100644
--- a/tests/app/v2/notifications/test_get_notifications.py
+++ b/tests/app/v2/notifications/test_get_notifications.py
@@ -9,8 +9,8 @@ from tests.app.db import create_notification, create_template
 
 
 @pytest.mark.parametrize('billable_units, provider', [
-    (1, 'mmg'),
-    (0, 'mmg'),
+    (1, 'sns'),
+    (0, 'sns'),
     (1, None)
 ])
 def test_get_notification_by_id_returns_200(
diff --git a/tests/conftest.py b/tests/conftest.py
index 12dfb3710..6d65314e3 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -102,11 +102,10 @@ def _notify_db(notify_api, worker_id):
 def sms_providers(_notify_db):
     """
     In production we randomly choose which provider to use based on their priority. To guarantee tests run the same each
-    time, make sure we always choose mmg. You'll need to override them in your tests if you wish to do something
+    time, make sure we always choose sns. You'll need to override them in your tests if you wish to do something
     different.
     """
-    get_provider_details_by_identifier('mmg').priority = 100
-    get_provider_details_by_identifier('firetext').priority = 0
+    get_provider_details_by_identifier('sns').priority = 100
 
 
 @pytest.fixture(scope='function')

From b4a05d6ae7d8891fa57d1a255ac32341d0eecddd Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 20 Dec 2022 10:36:58 -0500
Subject: [PATCH 16/46] Remove mmg & firetext from test_send_to_providers

---
 tests/app/delivery/test_send_to_providers.py | 569 ++++++++-----------
 1 file changed, 250 insertions(+), 319 deletions(-)

diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py
index eda4edae3..107c841d5 100644
--- a/tests/app/delivery/test_send_to_providers.py
+++ b/tests/app/delivery/test_send_to_providers.py
@@ -6,9 +6,11 @@ from unittest.mock import ANY
 
 import pytest
 from flask import current_app
+# from notifications_utils.recipients import validate_and_format_phone_number
+from requests import HTTPError
 
 import app
-# from app import firetext_client, mmg_client, notification_provider_clients
+from app import aws_sns_client, notification_provider_clients
 from app.dao import notifications_dao
 from app.dao.provider_details_dao import get_provider_details_by_identifier
 from app.delivery import send_to_providers
@@ -25,16 +27,16 @@ from app.models import (
     Notification,
 )
 from app.serialised_models import SerialisedService
-from tests.app.db import (  # create_service,; create_service_with_defined_sms_sender,; create_template,
+from tests.app.db import (
     create_email_branding,
     create_notification,
     create_reply_to_email,
+    create_service,
     create_service_sms_sender,
+    create_service_with_defined_sms_sender,
+    create_template,
 )
 
-# from notifications_utils.recipients import validate_and_format_phone_number
-# from requests import HTTPError
-
 
 def setup_function(_function):
     # pytest will run this function before each test. It makes sure the
@@ -42,21 +44,21 @@ def setup_function(_function):
     send_to_providers.provider_cache.clear()
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
+@pytest.mark.skip(reason="Reenable when we have more than 1 SMS provider")
 def test_provider_to_use_should_return_random_provider(mocker, notify_db_session):
-    mmg = get_provider_details_by_identifier('mmg')
-    firetext = get_provider_details_by_identifier('firetext')
-    mmg.priority = 25
-    firetext.priority = 75
-    mock_choices = mocker.patch('app.delivery.send_to_providers.random.choices', return_value=[mmg])
+    sns = get_provider_details_by_identifier('sns')
+    other = get_provider_details_by_identifier('other')
+    sns.priority = 60
+    other.priority = 40
+    mock_choices = mocker.patch('app.delivery.send_to_providers.random.choices', return_value=[sns])
 
-    ret = send_to_providers.provider_to_use('sms', international=False)
+    ret = send_to_providers.provider_to_use('sms', international=True)
 
-    mock_choices.assert_called_once_with([mmg, firetext], weights=[25, 75])
-    assert ret.name == 'mmg'
+    mock_choices.assert_called_once_with([sns, other], weights=[60, 40])
+    assert ret.name == 'sns'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
+@pytest.mark.skip(reason="Reenable when we have more than 1 SMS provider")
 def test_provider_to_use_should_cache_repeated_calls(mocker, notify_db_session):
     mock_choices = mocker.patch(
         'app.delivery.send_to_providers.random.choices',
@@ -72,85 +74,78 @@ def test_provider_to_use_should_cache_repeated_calls(mocker, notify_db_session):
     assert len(mock_choices.call_args_list) == 1
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 @pytest.mark.parametrize('international_provider_priority', (
     # Since there’s only one international provider it should always
     # be used, no matter what its priority is set to
     0, 50, 100,
 ))
-def test_provider_to_use_should_only_return_mmg_for_international(
+def test_provider_to_use_should_only_return_sns_for_international(
     mocker,
     notify_db_session,
     international_provider_priority,
 ):
-    mmg = get_provider_details_by_identifier('mmg')
-    mmg.priority = international_provider_priority
-    mock_choices = mocker.patch('app.delivery.send_to_providers.random.choices', return_value=[mmg])
+    sns = get_provider_details_by_identifier('sns')
+    sns.priority = international_provider_priority
 
     ret = send_to_providers.provider_to_use('sms', international=True)
 
-    mock_choices.assert_called_once_with([mmg], weights=[100])
-    assert ret.name == 'mmg'
+    assert ret.name == 'sns'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
+@pytest.mark.skip(reason="Reenable when we have more than 1 SMS provider")
 def test_provider_to_use_should_only_return_active_providers(mocker, restore_provider_details):
-    mmg = get_provider_details_by_identifier('mmg')
-    firetext = get_provider_details_by_identifier('firetext')
-    mmg.active = False
-    mock_choices = mocker.patch('app.delivery.send_to_providers.random.choices', return_value=[firetext])
+    sns = get_provider_details_by_identifier('sns')
+    other = get_provider_details_by_identifier('other')
+    sns.active = False
+    other.active = True
 
     ret = send_to_providers.provider_to_use('sms')
 
-    mock_choices.assert_called_once_with([firetext], weights=[100])
-    assert ret.name == 'firetext'
+    assert ret.name == 'other'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_provider_to_use_raises_if_no_active_providers(mocker, restore_provider_details):
-    mmg = get_provider_details_by_identifier('mmg')
-    mmg.active = False
+    sns = get_provider_details_by_identifier('sns')
+    sns.active = False
 
     with pytest.raises(Exception):
-        send_to_providers.provider_to_use('sms', international=True)
+        send_to_providers.provider_to_use('sms')
 
 
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# def test_should_send_personalised_template_to_correct_sms_provider_and_persist(
-#     sample_sms_template_with_html,
-#     mocker
-# ):
-#     db_notification = create_notification(template=sample_sms_template_with_html,
-#                                           to_field="+447234123123", personalisation={"name": "Jo"},
-#                                           status='created',
-#                                           reply_to_text=sample_sms_template_with_html.service.get_default_sms_sender(),
-#                                           normalised_to="447234123123"
-#                                           )
+def test_should_send_personalised_template_to_correct_sms_provider_and_persist(
+    sample_sms_template_with_html,
+    mocker
+):
+    db_notification = create_notification(template=sample_sms_template_with_html,
+                                          to_field="5558675309", personalisation={"name": "Jo"},
+                                          status='created',
+                                          reply_to_text=sample_sms_template_with_html.service.get_default_sms_sender(),
+                                          normalised_to="5558675309"
+                                          )
 
-#     mocker.patch('app.mmg_client.send_sms')
+    mocker.patch('app.aws_sns_client.send_sms')
 
-#     send_to_providers.send_sms_to_provider(
-#         db_notification
-#     )
+    send_to_providers.send_sms_to_provider(
+        db_notification
+    )
 
-#     mmg_client.send_sms.assert_called_once_with(
-#         to="447234123123",
-#         content="Sample service: Hello Jo\nHere is <em>some HTML</em> & entities",
-#         reference=str(db_notification.id),
-#         sender=current_app.config['FROM_NUMBER'],
-#         international=False
-#     )
+    aws_sns_client.send_sms.assert_called_once_with(
+        to="5558675309",
+        content="Sample service: Hello Jo\nHere is <em>some HTML</em> & entities",
+        reference=str(db_notification.id),
+        sender=current_app.config['FROM_NUMBER'],
+        international=False
+    )
 
-#     notification = Notification.query.filter_by(id=db_notification.id).one()
+    notification = Notification.query.filter_by(id=db_notification.id).one()
 
-#     assert notification.status == 'sending'
-#     assert notification.sent_at <= datetime.utcnow()
-#     assert notification.sent_by == 'mmg'
-#     assert notification.billable_units == 1
-#     assert notification.personalisation == {"name": "Jo"}
+    assert notification.status == 'sent'
+    assert notification.sent_at <= datetime.utcnow()
+    assert notification.sent_by == 'sns'
+    assert notification.billable_units == 1
+    assert notification.personalisation == {"name": "Jo"}
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_should_send_personalised_template_to_correct_email_provider_and_persist(
     sample_email_template_with_html,
     mocker
@@ -169,7 +164,7 @@ def test_should_send_personalised_template_to_correct_email_provider_and_persist
     )
 
     app.aws_ses_client.send_email.assert_called_once_with(
-        '"Sample service" <sample.service@test.notify.com>',
+        '"Sample service" <sample.service@notify.sandbox.10x.gsa.gov>',
         'jo.smith@example.com',
         'Jo <em>some HTML</em>',
         body='Hello Jo\nThis is an email from GOV.\u200bUK with <em>some HTML</em>\n',
@@ -187,7 +182,6 @@ def test_should_send_personalised_template_to_correct_email_provider_and_persist
     assert notification.personalisation == {"name": "Jo"}
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_should_not_send_email_message_when_service_is_inactive_notifcation_is_in_tech_failure(
         sample_service, sample_notification, mocker
 ):
@@ -201,12 +195,10 @@ def test_should_not_send_email_message_when_service_is_inactive_notifcation_is_i
     assert Notification.query.get(sample_notification.id).status == 'technical-failure'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-@pytest.mark.parametrize("client_send", ["app.mmg_client.send_sms", "app.firetext_client.send_sms"])
 def test_should_not_send_sms_message_when_service_is_inactive_notification_is_in_tech_failure(
-        sample_service, sample_notification, mocker, client_send):
+        sample_service, sample_notification, mocker):
     sample_service.active = False
-    send_mock = mocker.patch(client_send, return_value='reference')
+    send_mock = mocker.patch("app.aws_sns_client.send_sms", return_value='reference')
 
     with pytest.raises(NotificationTechnicalFailureException) as e:
         send_to_providers.send_sms_to_provider(sample_notification)
@@ -215,153 +207,148 @@ def test_should_not_send_sms_message_when_service_is_inactive_notification_is_in
     assert Notification.query.get(sample_notification.id).status == 'technical-failure'
 
 
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# def test_send_sms_should_use_template_version_from_notification_not_latest(
-#         sample_template,
-#         mocker):
-#     db_notification = create_notification(template=sample_template, to_field='+447234123123', status='created',
-#                                           reply_to_text=sample_template.service.get_default_sms_sender(),
-#                                           normalised_to='447234123123')
+def test_send_sms_should_use_template_version_from_notification_not_latest(
+        sample_template,
+        mocker):
+    db_notification = create_notification(template=sample_template, to_field='5558675309', status='created',
+                                          reply_to_text=sample_template.service.get_default_sms_sender(),
+                                          normalised_to='5558675309')
 
-#     mocker.patch('app.mmg_client.send_sms')
+    mocker.patch('app.aws_sns_client.send_sms')
 
-#     version_on_notification = sample_template.version
-#     expected_template_id = sample_template.id
+    version_on_notification = sample_template.version
+    expected_template_id = sample_template.id
 
-#     # Change the template
-#     from app.dao.templates_dao import (
-#         dao_get_template_by_id,
-#         dao_update_template,
-#     )
-#     sample_template.content = sample_template.content + " another version of the template"
-#     dao_update_template(sample_template)
-#     t = dao_get_template_by_id(sample_template.id)
-#     assert t.version > version_on_notification
+    # Change the template
+    from app.dao.templates_dao import (
+        dao_get_template_by_id,
+        dao_update_template,
+    )
+    sample_template.content = sample_template.content + " another version of the template"
+    dao_update_template(sample_template)
+    t = dao_get_template_by_id(sample_template.id)
+    assert t.version > version_on_notification
 
-#     send_to_providers.send_sms_to_provider(
-#         db_notification
-#     )
+    send_to_providers.send_sms_to_provider(
+        db_notification
+    )
 
-#     mmg_client.send_sms.assert_called_once_with(
-#         to=validate_and_format_phone_number("+447234123123"),
-#         content="Sample service: This is a template:\nwith a newline",
-#         reference=str(db_notification.id),
-#         sender=current_app.config['FROM_NUMBER'],
-#         international=False
-#     )
+    aws_sns_client.send_sms.assert_called_once_with(
+        to="5558675309",
+        content="Sample service: This is a template:\nwith a newline",
+        reference=str(db_notification.id),
+        sender=current_app.config['FROM_NUMBER'],
+        international=False
+    )
 
-#     t = dao_get_template_by_id(expected_template_id)
+    t = dao_get_template_by_id(expected_template_id)
 
-#     persisted_notification = notifications_dao.get_notification_by_id(db_notification.id)
-#     assert persisted_notification.to == db_notification.to
-#     assert persisted_notification.template_id == expected_template_id
-#     assert persisted_notification.template_version == version_on_notification
-#     assert persisted_notification.template_version != t.version
-#     assert persisted_notification.status == 'sending'
-#     assert not persisted_notification.personalisation
+    persisted_notification = notifications_dao.get_notification_by_id(db_notification.id)
+    assert persisted_notification.to == db_notification.to
+    assert persisted_notification.template_id == expected_template_id
+    assert persisted_notification.template_version == version_on_notification
+    assert persisted_notification.template_version != t.version
+    assert persisted_notification.status == 'sent'
+    assert not persisted_notification.personalisation
 
 
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# @pytest.mark.parametrize('research_mode,key_type', [
-#     (True, KEY_TYPE_NORMAL),
-#     (False, KEY_TYPE_TEST)
-# ])
-# def test_should_call_send_sms_response_task_if_research_mode(
-#         notify_db_session, sample_service, sample_notification, mocker, research_mode, key_type
-# ):
-#     mocker.patch('app.mmg_client.send_sms')
-#     mocker.patch('app.delivery.send_to_providers.send_sms_response')
+@pytest.mark.parametrize('research_mode,key_type', [
+    (True, KEY_TYPE_NORMAL),
+    (False, KEY_TYPE_TEST)
+])
+def test_should_call_send_sms_response_task_if_research_mode(
+        notify_db_session, sample_service, sample_notification, mocker, research_mode, key_type
+):
+    mocker.patch('app.aws_sns_client.send_sms')
+    mocker.patch('app.delivery.send_to_providers.send_sms_response')
 
-#     if research_mode:
-#         sample_service.research_mode = True
-#         notify_db_session.add(sample_service)
-#         notify_db_session.commit()
+    if research_mode:
+        sample_service.research_mode = True
+        notify_db_session.add(sample_service)
+        notify_db_session.commit()
 
-#     sample_notification.key_type = key_type
+    sample_notification.key_type = key_type
 
-#     send_to_providers.send_sms_to_provider(
-#         sample_notification
-#     )
-#     assert not mmg_client.send_sms.called
+    send_to_providers.send_sms_to_provider(
+        sample_notification
+    )
+    assert not aws_sns_client.send_sms.called
 
-#     app.delivery.send_to_providers.send_sms_response.assert_called_once_with(
-#         'mmg', str(sample_notification.id), sample_notification.to
-#     )
+    app.delivery.send_to_providers.send_sms_response.assert_called_once_with(
+        'sns', str(sample_notification.id), sample_notification.to
+    )
 
-#     persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id)
-#     assert persisted_notification.to == sample_notification.to
-#     assert persisted_notification.template_id == sample_notification.template_id
-#     assert persisted_notification.status == 'sending'
-#     assert persisted_notification.sent_at <= datetime.utcnow()
-#     assert persisted_notification.sent_by == 'mmg'
-#     assert not persisted_notification.personalisation
+    persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id)
+    assert persisted_notification.to == sample_notification.to
+    assert persisted_notification.template_id == sample_notification.template_id
+    assert persisted_notification.status == 'sent'
+    assert persisted_notification.sent_at <= datetime.utcnow()
+    assert persisted_notification.sent_by == 'sns'
+    assert not persisted_notification.personalisation
 
 
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# def test_should_have_sending_status_if_fake_callback_function_fails(sample_notification, mocker):
-#     mocker.patch('app.delivery.send_to_providers.send_sms_response', side_effect=HTTPError)
+@pytest.mark.skip(reason="Needs updating when we get SMS delivery receipts done")
+def test_should_have_sending_status_if_fake_callback_function_fails(sample_notification, mocker):
+    mocker.patch('app.delivery.send_to_providers.send_sms_response', side_effect=HTTPError)
 
-#     sample_notification.key_type = KEY_TYPE_TEST
+    sample_notification.key_type = KEY_TYPE_TEST
 
-#     with pytest.raises(HTTPError):
-#         send_to_providers.send_sms_to_provider(
-#             sample_notification
-#         )
-#     assert sample_notification.status == 'sending'
-#     assert sample_notification.sent_by == 'mmg'
+    with pytest.raises(HTTPError):
+        send_to_providers.send_sms_to_provider(
+            sample_notification
+        )
+    assert sample_notification.status == 'sending'
+    assert sample_notification.sent_by == 'sns'
 
 
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# def test_should_not_send_to_provider_when_status_is_not_created(
-#     sample_template,
-#     mocker
-# ):
-#     notification = create_notification(template=sample_template, status='sending')
-#     mocker.patch('app.mmg_client.send_sms')
-#     response_mock = mocker.patch('app.delivery.send_to_providers.send_sms_response')
+def test_should_not_send_to_provider_when_status_is_not_created(
+    sample_template,
+    mocker
+):
+    notification = create_notification(template=sample_template, status='sending')
+    mocker.patch('app.aws_sns_client.send_sms')
+    response_mock = mocker.patch('app.delivery.send_to_providers.send_sms_response')
 
-#     send_to_providers.send_sms_to_provider(
-#         notification
-#     )
+    send_to_providers.send_sms_to_provider(
+        notification
+    )
 
-#     app.mmg_client.send_sms.assert_not_called()
-#     response_mock.assert_not_called()
+    app.aws_sns_client.send_sms.assert_not_called()
+    response_mock.assert_not_called()
 
 
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# def test_should_send_sms_with_downgraded_content(notify_db_session, mocker):
-#     # é, o, and u are in GSM.
-#     # ī, grapes, tabs, zero width space and ellipsis are not
-#     # ó isn't in GSM, but it is in the welsh alphabet so will still be sent
-#     msg = "a é ī o u 🍇 foo\tbar\u200bbaz((misc))…"
-#     placeholder = '∆∆∆abc'
-#     gsm_message = "?ódz Housing Service: a é i o u ? foo barbaz???abc..."
-#     service = create_service(service_name='Łódź Housing Service')
-#     template = create_template(service, content=msg)
-#     db_notification = create_notification(
-#         template=template,
-#         personalisation={'misc': placeholder}
-#     )
+def test_should_send_sms_with_downgraded_content(notify_db_session, mocker):
+    # é, o, and u are in GSM.
+    # ī, grapes, tabs, zero width space and ellipsis are not
+    # ó isn't in GSM, but it is in the welsh alphabet so will still be sent
+    msg = "a é ī o u 🍇 foo\tbar\u200bbaz((misc))…"
+    placeholder = '∆∆∆abc'
+    gsm_message = "?ódz Housing Service: a é i o u ? foo barbaz???abc..."
+    service = create_service(service_name='Łódź Housing Service')
+    template = create_template(service, content=msg)
+    db_notification = create_notification(
+        template=template,
+        personalisation={'misc': placeholder}
+    )
 
-#     mocker.patch('app.mmg_client.send_sms')
+    mocker.patch('app.aws_sns_client.send_sms')
 
-#     send_to_providers.send_sms_to_provider(db_notification)
+    send_to_providers.send_sms_to_provider(db_notification)
 
-#     mmg_client.send_sms.assert_called_once_with(
-#         to=ANY,
-#         content=gsm_message,
-#         reference=ANY,
-#         sender=ANY,
-#         international=False
-#     )
+    aws_sns_client.send_sms.assert_called_once_with(
+        to=ANY,
+        content=gsm_message,
+        reference=ANY,
+        sender=ANY,
+        international=False
+    )
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_send_sms_should_use_service_sms_sender(
         sample_service,
         sample_template,
         mocker):
-    mocker.patch('app.mmg_client.send_sms')
+    mocker.patch('app.aws_sns_client.send_sms')
 
     sms_sender = create_service_sms_sender(service=sample_service, sms_sender='123456', is_default=False)
     db_notification = create_notification(template=sample_template, reply_to_text=sms_sender.sms_sender)
@@ -371,7 +358,7 @@ def test_send_sms_should_use_service_sms_sender(
         db_notification,
     )
 
-    app.mmg_client.send_sms.assert_called_once_with(
+    app.aws_sns_client.send_sms.assert_called_once_with(
         to=ANY,
         content=ANY,
         reference=ANY,
@@ -380,7 +367,6 @@ def test_send_sms_should_use_service_sms_sender(
     )
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 @pytest.mark.parametrize('research_mode,key_type', [
     (True, KEY_TYPE_NORMAL),
     (False, KEY_TYPE_TEST)
@@ -421,7 +407,6 @@ def test_send_email_to_provider_should_call_research_mode_task_response_task_if_
     assert persisted_notification.billable_units == 0
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_send_email_to_provider_should_not_send_to_provider_when_status_is_not_created(
     sample_email_template,
     mocker
@@ -437,7 +422,6 @@ def test_send_email_to_provider_should_not_send_to_provider_when_status_is_not_c
     app.delivery.send_to_providers.send_email_response.assert_not_called()
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_send_email_should_use_service_reply_to_email(
         sample_service,
         sample_email_template,
@@ -461,7 +445,6 @@ def test_send_email_should_use_service_reply_to_email(
     )
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_get_html_email_renderer_should_return_for_normal_service(sample_service):
     options = send_to_providers.get_html_email_options(sample_service)
     assert options['govuk_banner'] is True
@@ -512,7 +495,6 @@ def test_get_html_email_renderer_with_branding_details_and_render_govuk_banner_o
     assert options == {'govuk_banner': True, 'brand_banner': False}
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_get_html_email_renderer_prepends_logo_path(notify_api):
     Service = namedtuple('Service', ['email_branding'])
     EmailBranding = namedtuple('EmailBranding', ['brand_type', 'colour', 'name', 'logo', 'text'])
@@ -593,22 +575,21 @@ def test_should_not_update_notification_if_research_mode_on_exception(
     assert update_mock.called
 
 
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# @pytest.mark.parametrize("starting_status, expected_status", [
-#     ("delivered", "delivered"),
-#     ("created", "sending"),
-#     ("technical-failure", "technical-failure"),
-# ])
-# def test_update_notification_to_sending_does_not_update_status_from_a_final_status(
-#     sample_service, notify_db_session, starting_status, expected_status
-# ):
-#     template = create_template(sample_service)
-#     notification = create_notification(template=template, status=starting_status)
-#     send_to_providers.update_notification_to_sending(
-#         notification,
-#         notification_provider_clients.get_client_by_name_and_type("mmg", "sms")
-#     )
-#     assert notification.status == expected_status
+@pytest.mark.parametrize("starting_status, expected_status", [
+    ("delivered", "delivered"),
+    ("created", "sent"),
+    ("technical-failure", "technical-failure"),
+])
+def test_update_notification_to_sending_does_not_update_status_from_a_final_status(
+    sample_service, notify_db_session, starting_status, expected_status
+):
+    template = create_template(sample_service)
+    notification = create_notification(template=template, status=starting_status)
+    send_to_providers.update_notification_to_sending(
+        notification,
+        notification_provider_clients.get_client_by_name_and_type("sns", "sms")
+    )
+    assert notification.status == expected_status
 
 
 def __update_notification(notification_to_update, research_mode, expected_status):
@@ -616,14 +597,13 @@ def __update_notification(notification_to_update, research_mode, expected_status
         notification_to_update.status = expected_status
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 @pytest.mark.parametrize('research_mode,key_type, billable_units, expected_status', [
     (True, KEY_TYPE_NORMAL, 0, 'delivered'),
-    (False, KEY_TYPE_NORMAL, 1, 'sending'),
+    (False, KEY_TYPE_NORMAL, 1, 'sent'),
     (False, KEY_TYPE_TEST, 0, 'sending'),
     (True, KEY_TYPE_TEST, 0, 'sending'),
     (True, KEY_TYPE_TEAM, 0, 'delivered'),
-    (False, KEY_TYPE_TEAM, 1, 'sending')
+    (False, KEY_TYPE_TEAM, 1, 'sent')
 ])
 def test_should_update_billable_units_and_status_according_to_research_mode_and_key_type(
     sample_template,
@@ -634,7 +614,7 @@ def test_should_update_billable_units_and_status_according_to_research_mode_and_
     expected_status
 ):
     notification = create_notification(template=sample_template, billable_units=0, status='created', key_type=key_type)
-    mocker.patch('app.mmg_client.send_sms')
+    mocker.patch('app.aws_sns_client.send_sms')
     mocker.patch('app.delivery.send_to_providers.send_sms_response',
                  side_effect=__update_notification(notification, research_mode, expected_status))
 
@@ -648,12 +628,11 @@ def test_should_update_billable_units_and_status_according_to_research_mode_and_
     assert notification.status == expected_status
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_should_set_notification_billable_units_and_reduces_provider_priority_if_sending_to_provider_fails(
     sample_notification,
     mocker,
 ):
-    mocker.patch('app.mmg_client.send_sms', side_effect=Exception())
+    mocker.patch('app.aws_sns_client.send_sms', side_effect=Exception())
     mock_reduce = mocker.patch('app.delivery.send_to_providers.dao_reduce_sms_provider_priority')
 
     sample_notification.billable_units = 0
@@ -663,119 +642,73 @@ def test_should_set_notification_billable_units_and_reduces_provider_priority_if
         send_to_providers.send_sms_to_provider(sample_notification)
 
     assert sample_notification.billable_units == 1
-    mock_reduce.assert_called_once_with('mmg', time_threshold=timedelta(minutes=1))
+    mock_reduce.assert_called_once_with('sns', time_threshold=timedelta(minutes=1))
 
 
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# def test_should_send_sms_to_international_providers(
-#     sample_template,
-#     sample_user,
-#     mocker
-# ):
-#     mocker.patch('app.mmg_client.send_sms')
-#     mocker.patch('app.firetext_client.send_sms')
+def test_should_send_sms_to_international_providers(
+    sample_template,
+    sample_user,
+    mocker
+):
+    mocker.patch('app.aws_sns_client.send_sms')
 
-#     # set firetext to active
-#     get_provider_details_by_identifier('firetext').priority = 100
-#     get_provider_details_by_identifier('mmg').priority = 0
+    notification_international = create_notification(
+        template=sample_template,
+        to_field="+6011-17224412",
+        personalisation={"name": "Jo"},
+        status='created',
+        international=True,
+        reply_to_text=sample_template.service.get_default_sms_sender(),
+        normalised_to='601117224412'
+    )
 
-#     notification_international = create_notification(
-#         template=sample_template,
-#         to_field="+6011-17224412",
-#         personalisation={"name": "Jo"},
-#         status='created',
-#         international=True,
-#         reply_to_text=sample_template.service.get_default_sms_sender(),
-#         normalised_to='601117224412'
-#     )
+    send_to_providers.send_sms_to_provider(
+        notification_international
+    )
 
-#     send_to_providers.send_sms_to_provider(
-#         notification_international
-#     )
+    aws_sns_client.send_sms.assert_called_once_with(
+        to="601117224412",
+        content=ANY,
+        reference=str(notification_international.id),
+        sender=current_app.config['FROM_NUMBER'],
+        international=True
+    )
 
-#     mmg_client.send_sms.assert_called_once_with(
-#         to="601117224412",
-#         content=ANY,
-#         reference=str(notification_international.id),
-#         sender=current_app.config['FROM_NUMBER'],
-#         international=True
-#     )
-
-#     assert notification_international.status == 'sent'
-#     assert notification_international.sent_by == 'mmg'
+    assert notification_international.status == 'sent'
+    assert notification_international.sent_by == 'sns'
 
 
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# def test_should_send_non_international_sms_to_default_provider(
-#     sample_template,
-#     sample_user,
-#     mocker
-# ):
-#     mocker.patch('app.mmg_client.send_sms')
-#     mocker.patch('app.firetext_client.send_sms')
+@pytest.mark.parametrize('sms_sender, expected_sender, prefix_sms, expected_content', [
+    ('foo', 'foo', False, 'bar'),
+    ('foo', 'foo', True, 'Sample service: bar'),
+    # if 40604 is actually in DB then treat that as if entered manually
+    ('40604', '40604', False, 'bar'),
+    # 'testing' is the FROM_NUMBER during unit tests
+    ('testing', 'testing', True, 'Sample service: bar'),
+    ('testing', 'testing', False, 'bar'),
+])
+def test_should_handle_sms_sender_and_prefix_message(
+    mocker,
+    sms_sender,
+    prefix_sms,
+    expected_sender,
+    expected_content,
+    notify_db_session
+):
+    mocker.patch('app.aws_sns_client.send_sms')
+    service = create_service_with_defined_sms_sender(sms_sender_value=sms_sender, prefix_sms=prefix_sms)
+    template = create_template(service, content='bar')
+    notification = create_notification(template, reply_to_text=sms_sender)
 
-#     # set firetext to active
-#     get_provider_details_by_identifier('firetext').priority = 100
-#     get_provider_details_by_identifier('mmg').priority = 0
+    send_to_providers.send_sms_to_provider(notification)
 
-#     notification_uk = create_notification(
-#         template=sample_template,
-#         to_field="+447234123999",
-#         personalisation={"name": "Jo"},
-#         status='created',
-#         international=False,
-#         reply_to_text=sample_template.service.get_default_sms_sender(),
-#         normalised_to="447234123999"
-#     )
-
-#     send_to_providers.send_sms_to_provider(
-#         notification_uk
-#     )
-
-#     firetext_client.send_sms.assert_called_once_with(
-#         to="447234123999",
-#         content=ANY,
-#         reference=str(notification_uk.id),
-#         sender=current_app.config['FROM_NUMBER'],
-#         international=False
-#     )
-
-#     assert notification_uk.status == 'sending'
-#     assert notification_uk.sent_by == 'firetext'
-
-
-# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-# @pytest.mark.parametrize('sms_sender, expected_sender, prefix_sms, expected_content', [
-#     ('foo', 'foo', False, 'bar'),
-#     ('foo', 'foo', True, 'Sample service: bar'),
-#     # if 40604 is actually in DB then treat that as if entered manually
-#     ('40604', '40604', False, 'bar'),
-#     # 'testing' is the FROM_NUMBER during unit tests
-#     ('testing', 'testing', True, 'Sample service: bar'),
-#     ('testing', 'testing', False, 'bar'),
-# ])
-# def test_should_handle_sms_sender_and_prefix_message(
-#     mocker,
-#     sms_sender,
-#     prefix_sms,
-#     expected_sender,
-#     expected_content,
-#     notify_db_session
-# ):
-#     mocker.patch('app.mmg_client.send_sms')
-#     service = create_service_with_defined_sms_sender(sms_sender_value=sms_sender, prefix_sms=prefix_sms)
-#     template = create_template(service, content='bar')
-#     notification = create_notification(template, reply_to_text=sms_sender)
-
-#     send_to_providers.send_sms_to_provider(notification)
-
-#     mmg_client.send_sms.assert_called_once_with(
-#         content=expected_content,
-#         sender=expected_sender,
-#         to=ANY,
-#         reference=ANY,
-#         international=False
-#     )
+    aws_sns_client.send_sms.assert_called_once_with(
+        content=expected_content,
+        sender=expected_sender,
+        to=ANY,
+        reference=ANY,
+        international=False
+    )
 
 
 def test_send_email_to_provider_uses_reply_to_from_notification(
@@ -799,14 +732,13 @@ def test_send_email_to_provider_uses_reply_to_from_notification(
     )
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_send_sms_to_provider_should_use_normalised_to(
         mocker, client, sample_template
 ):
-    send_mock = mocker.patch('app.mmg_client.send_sms')
+    send_mock = mocker.patch('app.aws_sns_client.send_sms')
     notification = create_notification(template=sample_template,
-                                       to_field='+447700900855',
-                                       normalised_to='447700900855')
+                                       to_field='+15558675309',
+                                       normalised_to='5558675309')
     send_to_providers.send_sms_to_provider(notification)
     send_mock.assert_called_once_with(to=notification.normalised_to,
                                       content=ANY,
@@ -832,7 +764,6 @@ def test_send_email_to_provider_should_user_normalised_to(
                                       reply_to_address=notification.reply_to_text)
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
 def test_send_sms_to_provider_should_return_template_if_found_in_redis(
         mocker, client, sample_template
 ):
@@ -854,7 +785,7 @@ def test_send_sms_to_provider_should_return_template_if_found_in_redis(
         'app.dao.services_dao.dao_fetch_service_by_id'
     )
 
-    send_mock = mocker.patch('app.mmg_client.send_sms')
+    send_mock = mocker.patch('app.aws_sns_client.send_sms')
     notification = create_notification(template=sample_template,
                                        to_field='+447700900855',
                                        normalised_to='447700900855')

From db75efe78960d243913faffe80063b27d96e4358 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 20 Dec 2022 10:41:26 -0500
Subject: [PATCH 17/46] Remove mmg & firetext from test_provider_details_dao

---
 tests/app/dao/test_provider_details_dao.py | 54 +++++++++++-----------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/tests/app/dao/test_provider_details_dao.py b/tests/app/dao/test_provider_details_dao.py
index 0e41991ba..1ca40cedd 100644
--- a/tests/app/dao/test_provider_details_dao.py
+++ b/tests/app/dao/test_provider_details_dao.py
@@ -195,17 +195,17 @@ def test_get_sms_providers_for_update_returns_nothing_if_recent_updates(restore_
 
 @pytest.mark.skip(reason="Reenable if/when we add a second SMS provider")
 @pytest.mark.parametrize(['starting_priorities', 'expected_priorities'], [
-    ({'mmg': 50, 'firetext': 50}, {'mmg': 40, 'firetext': 60}),
-    ({'mmg': 0, 'firetext': 20}, {'mmg': 0, 'firetext': 30}),  # lower bound respected
-    ({'mmg': 50, 'firetext': 100}, {'mmg': 40, 'firetext': 100}),  # upper bound respected
+    ({'sns': 50, 'other': 50}, {'sns': 40, 'other': 60}),
+    ({'sns': 0, 'other': 20}, {'sns': 0, 'other': 30}),  # lower bound respected
+    ({'sns': 50, 'other': 100}, {'sns': 40, 'other': 100}),  # upper bound respected
 
     # document what happens if they have unexpected values outside of the 0 - 100 range (due to manual setting from
     # the admin app). the code never causes further issues, but sometimes doesn't actively reset the vaues to 0-100.
-    ({'mmg': 150, 'firetext': 50}, {'mmg': 140, 'firetext': 60}),
-    ({'mmg': 50, 'firetext': 150}, {'mmg': 40, 'firetext': 100}),
+    ({'sns': 150, 'other': 50}, {'sns': 140, 'other': 60}),
+    ({'sns': 50, 'other': 150}, {'sns': 40, 'other': 100}),
 
-    ({'mmg': -100, 'firetext': 50}, {'mmg': 0, 'firetext': 60}),
-    ({'mmg': 50, 'firetext': -100}, {'mmg': 40, 'firetext': -90}),
+    ({'sns': -100, 'other': 50}, {'sns': 0, 'other': 60}),
+    ({'sns': 50, 'other': -100}, {'sns': 40, 'other': -90}),
 ])
 def test_reduce_sms_provider_priority_adjusts_provider_priorities(
     mocker,
@@ -216,19 +216,19 @@ def test_reduce_sms_provider_priority_adjusts_provider_priorities(
 ):
     mock_adjust = mocker.patch('app.dao.provider_details_dao._adjust_provider_priority')
 
-    mmg = get_provider_details_by_identifier('mmg')
-    firetext = get_provider_details_by_identifier('firetext')
+    sns = get_provider_details_by_identifier('sns')
+    other = get_provider_details_by_identifier('other')
 
-    mmg.priority = starting_priorities['mmg']
-    firetext.priority = starting_priorities['firetext']
+    sns.priority = starting_priorities['sns']
+    other.priority = starting_priorities['other']
     # need to update these manually to avoid triggering the `onupdate` clause of the updated_at column
     ProviderDetails.query.filter(ProviderDetails.notification_type == 'sms').update({'updated_at': datetime.min})
 
-    # switch away from mmg. currently both 50/50
-    dao_reduce_sms_provider_priority('mmg', time_threshold=timedelta(minutes=10))
+    # switch away from sns. currently both 50/50
+    dao_reduce_sms_provider_priority('sns', time_threshold=timedelta(minutes=10))
 
-    mock_adjust.assert_any_call(firetext, expected_priorities['firetext'])
-    mock_adjust.assert_any_call(mmg, expected_priorities['mmg'])
+    mock_adjust.assert_any_call(other, expected_priorities['other'])
+    mock_adjust.assert_any_call(sns, expected_priorities['sns'])
 
 
 def test_reduce_sms_provider_priority_does_nothing_if_providers_have_recently_changed(
@@ -256,7 +256,7 @@ def test_reduce_sms_provider_priority_does_nothing_if_there_is_only_one_active_p
 
 
 @pytest.mark.skip(reason="Reenable if/when we add a second SMS provider")
-@pytest.mark.parametrize('existing_mmg, existing_firetext, new_mmg, new_firetext', [
+@pytest.mark.parametrize('existing_sns, existing_other, new_sns, new_other', [
     (50, 50, 60, 40),  # not just 50/50 - 60/40 specifically
     (65, 35, 60, 40),  # doesn't overshoot if there's less than 10 difference
     (0, 100, 10, 90),  # only adjusts by 10
@@ -265,26 +265,26 @@ def test_reduce_sms_provider_priority_does_nothing_if_there_is_only_one_active_p
 def test_adjust_provider_priority_back_to_resting_points_updates_all_providers(
     restore_provider_details,
     mocker,
-    existing_mmg,
-    existing_firetext,
-    new_mmg,
-    new_firetext
+    existing_sns,
+    existing_other,
+    new_sns,
+    new_other
 ):
-    mmg = get_provider_details_by_identifier('mmg')
-    firetext = get_provider_details_by_identifier('firetext')
-    mmg.priority = existing_mmg
-    firetext.priority = existing_firetext
+    sns = get_provider_details_by_identifier('sns')
+    other = get_provider_details_by_identifier('other')
+    sns.priority = existing_sns
+    other.priority = existing_other
 
     mock_adjust = mocker.patch('app.dao.provider_details_dao._adjust_provider_priority')
     mock_get_providers = mocker.patch('app.dao.provider_details_dao._get_sms_providers_for_update', return_value=[
-        mmg, firetext
+        sns, other
     ])
 
     dao_adjust_provider_priority_back_to_resting_points()
 
     mock_get_providers.assert_called_once_with(timedelta(hours=1))
-    mock_adjust.assert_any_call(mmg, new_mmg)
-    mock_adjust.assert_any_call(firetext, new_firetext)
+    mock_adjust.assert_any_call(sns, new_sns)
+    mock_adjust.assert_any_call(other, new_other)
 
 
 def test_adjust_provider_priority_back_to_resting_points_does_nothing_if_theyre_already_at_right_values(

From 77afecf8d2156a05f55b00ad9a93dc76ec9b20e0 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Wed, 21 Dec 2022 15:10:44 -0500
Subject: [PATCH 18/46] Remove mmg & firetext from research_mode_tasks.py

---
 app/celery/research_mode_tasks.py            | 77 ++++++-------------
 tests/app/celery/test_research_mode_tasks.py | 81 ++++++--------------
 2 files changed, 45 insertions(+), 113 deletions(-)

diff --git a/app/celery/research_mode_tasks.py b/app/celery/research_mode_tasks.py
index 4477b3994..b59b8b235 100644
--- a/app/celery/research_mode_tasks.py
+++ b/app/celery/research_mode_tasks.py
@@ -12,9 +12,9 @@ from app.celery.process_ses_receipts_tasks import process_ses_results
 from app.config import QueueNames
 from app.models import SMS_TYPE
 
-temp_fail = "7700900003"
-perm_fail = "7700900002"
-delivered = "7700900001"
+temp_fail = "5558675303"
+perm_fail = "5558675302"
+delivered = "5558675309"
 
 delivered_email = "delivered@simulator.notify"
 perm_fail_email = "perm-fail@simulator.notify"
@@ -22,22 +22,8 @@ temp_fail_email = "temp-fail@simulator.notify"
 
 
 def send_sms_response(provider, reference, to):
-    if provider == "mmg":
-        body = mmg_callback(reference, to)
-        headers = {"Content-type": "application/json"}
-    else:
-        headers = {"Content-type": "application/x-www-form-urlencoded"}
-        body = firetext_callback(reference, to)
-        # to simulate getting a temporary_failure from firetext
-        # we need to send a pending status updated then a permanent-failure
-        if body['status'] == '2':  # pending status
-            make_request(SMS_TYPE, provider, body, headers)
-            # 1 is a declined status for firetext, will result in a temp-failure
-            body = {'mobile': to,
-                    'status': "1",
-                    'time': '2016-03-10 14:17:00',
-                    'reference': reference
-                    }
+    body = sns_callback(reference, to)
+    headers = {"Content-type": "application/json"}
 
     make_request(SMS_TYPE, provider, body, headers)
 
@@ -78,44 +64,25 @@ def make_request(notification_type, provider, data, headers):
     return response.json()
 
 
-def mmg_callback(notification_id, to):
-    """
-        status: 3 - delivered
-        status: 4 - expired (temp failure)
-        status: 5 - rejected (perm failure)
-    """
+def sns_callback(notification_id, to):
+    raise Exception("Need to update for SNS callback format along with test_send_to_providers")
 
-    if to.strip().endswith(temp_fail):
-        status = "4"
-    elif to.strip().endswith(perm_fail):
-        status = "5"
-    else:
-        status = "3"
+    # example from mmg_callback
+    # if to.strip().endswith(temp_fail):
+    #     # status: 4 - expired (temp failure)
+    #     status = "4"
+    # elif to.strip().endswith(perm_fail):
+    #     # status: 5 - rejected (perm failure)
+    #     status = "5"
+    # else:
+    #     # status: 3 - delivered
+    #     status = "3"
 
-    return json.dumps({"reference": "mmg_reference",
-                       "CID": str(notification_id),
-                       "MSISDN": to,
-                       "status": status,
-                       "deliverytime": "2016-04-05 16:01:07"})
-
-
-def firetext_callback(notification_id, to):
-    """
-        status: 0 - delivered
-        status: 1 - perm failure
-    """
-    if to.strip().endswith(perm_fail):
-        status = "1"
-    elif to.strip().endswith(temp_fail):
-        status = "2"
-    else:
-        status = "0"
-    return {
-        'mobile': to,
-        'status': status,
-        'time': '2016-03-10 14:17:00',
-        'reference': notification_id
-    }
+    # return json.dumps({"reference": "mmg_reference",
+    #                    "CID": str(notification_id),
+    #                    "MSISDN": to,
+    #                    "status": status,
+    #                    "deliverytime": "2016-04-05 16:01:07"})
 
 
 @notify_celery.task(bind=True, name="create-fake-letter-response-file", max_retries=5, default_retry_delay=300)
diff --git a/tests/app/celery/test_research_mode_tasks.py b/tests/app/celery/test_research_mode_tasks.py
index 7472917d5..c9e42f62e 100644
--- a/tests/app/celery/test_research_mode_tasks.py
+++ b/tests/app/celery/test_research_mode_tasks.py
@@ -9,11 +9,10 @@ from freezegun import freeze_time
 from app.celery.research_mode_tasks import (
     HTTPError,
     create_fake_letter_response_file,
-    firetext_callback,
-    mmg_callback,
     send_email_response,
     send_sms_response,
     ses_notification_callback,
+    sns_callback,
 )
 from app.config import QueueNames
 from tests.conftest import Matcher, set_config_values
@@ -24,22 +23,24 @@ dvla_response_file_matcher = Matcher(
 )
 
 
-def test_make_mmg_callback(notify_api, rmock):
-    endpoint = "http://localhost:6011/notifications/sms/mmg"
+@pytest.mark.skip(reason="Re-enable when SMS receipts exist")
+def test_make_sns_callback(notify_api, rmock):
+    endpoint = "http://localhost:6011/notifications/sms/sns"
     rmock.request(
         "POST",
         endpoint,
         json={"status": "success"},
         status_code=200)
-    send_sms_response("mmg", "1234", "07700900001")
+    send_sms_response("sns", "1234", "5558675309")
 
     assert rmock.called
     assert rmock.request_history[0].url == endpoint
-    assert json.loads(rmock.request_history[0].text)['MSISDN'] == '07700900001'
+    assert json.loads(rmock.request_history[0].text)['MSISDN'] == '5558675309'
 
 
+@pytest.mark.skip(reason="Re-enable when SMS receipts exist")
 def test_callback_logs_on_api_call_failure(notify_api, rmock, mocker):
-    endpoint = "http://localhost:6011/notifications/sms/mmg"
+    endpoint = "http://localhost:6011/notifications/sms/sns"
     rmock.request(
         "POST",
         endpoint,
@@ -57,23 +58,6 @@ def test_callback_logs_on_api_call_failure(notify_api, rmock, mocker):
     )
 
 
-@pytest.mark.parametrize("phone_number",
-                         ["07700900001", "07700900002", "07700900003",
-                          "07700900236"])
-def test_make_firetext_callback(notify_api, rmock, phone_number):
-    endpoint = "http://localhost:6011/notifications/sms/firetext"
-    rmock.request(
-        "POST",
-        endpoint,
-        json="some data",
-        status_code=200)
-    send_sms_response("firetext", "1234", phone_number)
-
-    assert rmock.called
-    assert rmock.request_history[0].url == endpoint
-    assert 'mobile={}'.format(phone_number) in rmock.request_history[0].text
-
-
 def test_make_ses_callback(notify_api, mocker):
     mock_task = mocker.patch('app.celery.research_mode_tasks.process_ses_results')
     some_ref = str(uuid.uuid4())
@@ -84,55 +68,36 @@ def test_make_ses_callback(notify_api, mocker):
     assert mock_task.apply_async.call_args[0][0][0] == ses_notification_callback(some_ref)
 
 
-@pytest.mark.parametrize("phone_number", ["07700900001", "+447700900001", "7700900001", "+44 7700900001",
-                                          "+447700900236"])
-def test_delivered_mmg_callback(phone_number):
-    data = json.loads(mmg_callback("1234", phone_number))
+@pytest.mark.skip(reason="Re-enable when SNS delivery receipts exist")
+def test_delievered_sns_callback():
+    phone_number = "5558675309"
+    data = json.loads(sns_callback("1234", phone_number))
     assert data['MSISDN'] == phone_number
     assert data['status'] == "3"
-    assert data['reference'] == "mmg_reference"
+    assert data['reference'] == "sns_reference"
     assert data['CID'] == "1234"
 
 
-@pytest.mark.parametrize("phone_number", ["07700900002", "+447700900002", "7700900002", "+44 7700900002"])
-def test_perm_failure_mmg_callback(phone_number):
-    data = json.loads(mmg_callback("1234", phone_number))
+@pytest.mark.skip(reason="Re-enable when SNS delivery receipts exist")
+def test_perm_failure_sns_callback():
+    phone_number = "5558675302"
+    data = json.loads(sns_callback("1234", phone_number))
     assert data['MSISDN'] == phone_number
     assert data['status'] == "5"
-    assert data['reference'] == "mmg_reference"
+    assert data['reference'] == "sns_reference"
     assert data['CID'] == "1234"
 
 
-@pytest.mark.parametrize("phone_number", ["07700900003", "+447700900003", "7700900003", "+44 7700900003"])
-def test_temp_failure_mmg_callback(phone_number):
-    data = json.loads(mmg_callback("1234", phone_number))
+@pytest.mark.skip(reason="Re-enable when SNS delivery receipts exist")
+def test_temp_failure_sns_callback():
+    phone_number = "5558675303"
+    data = json.loads(sns_callback("1234", phone_number))
     assert data['MSISDN'] == phone_number
     assert data['status'] == "4"
-    assert data['reference'] == "mmg_reference"
+    assert data['reference'] == "sns_reference"
     assert data['CID'] == "1234"
 
 
-@pytest.mark.parametrize("phone_number", ["07700900001", "+447700900001", "7700900001", "+44 7700900001",
-                                          "+447700900256"])
-def test_delivered_firetext_callback(phone_number):
-    assert firetext_callback('1234', phone_number) == {
-        'mobile': phone_number,
-        'status': '0',
-        'time': '2016-03-10 14:17:00',
-        'reference': '1234'
-    }
-
-
-@pytest.mark.parametrize("phone_number", ["07700900002", "+447700900002", "7700900002", "+44 7700900002"])
-def test_failure_firetext_callback(phone_number):
-    assert firetext_callback('1234', phone_number) == {
-        'mobile': phone_number,
-        'status': '1',
-        'time': '2016-03-10 14:17:00',
-        'reference': '1234'
-    }
-
-
 @freeze_time("2018-01-25 14:00:30")
 @pytest.mark.skip(reason="Skipping letter-related functionality for now")
 def test_create_fake_letter_response_file_uploads_response_file_s3(

From 44a4b2d1a9999b23b3bd34dd3a0397d88b4afab4 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 22 Dec 2022 08:44:04 -0500
Subject: [PATCH 19/46] Remove firetext from notifications_sms_callback.py

---
 .../process_sms_client_response_tasks.py      | 132 ++++++++----------
 .../notifications_sms_callback.py             |  24 +---
 .../test_notifications_sms_callbacks.py       |  81 -----------
 .../test_receive_notification.py              |   2 +-
 4 files changed, 66 insertions(+), 173 deletions(-)

diff --git a/app/celery/process_sms_client_response_tasks.py b/app/celery/process_sms_client_response_tasks.py
index 2a1cc4dd4..aff0aad15 100644
--- a/app/celery/process_sms_client_response_tasks.py
+++ b/app/celery/process_sms_client_response_tasks.py
@@ -1,88 +1,80 @@
-import uuid
-from datetime import datetime
-
-from flask import current_app
-from notifications_utils.template import SMSMessageTemplate
-
-from app import notify_celery, statsd_client
-from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
-from app.clients import ClientException
-from app.dao import notifications_dao
-from app.dao.templates_dao import dao_get_template_by_id
-from app.models import NOTIFICATION_PENDING
+from app import notify_celery
 
 sms_response_mapper = {
     # 'SNS': get_sns_responses,
 }
 
 
-# gUpdate with new providers")
+# this is used by notifications_sms_callback and needs to be heavily changed for SNS
+# leaving for now as an example of what MMG did and what we may want to replicate in the eventual
+# SNS method
 @notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
 def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
-    # validate reference
-    try:
-        uuid.UUID(provider_reference, version=4)
-    except ValueError as e:
-        current_app.logger.exception(f'{client_name} callback with invalid reference {provider_reference}')
-        raise e
+    raise Exception("process_sms_client_response not implemented")
+#     # validate reference
+#     try:
+#         uuid.UUID(provider_reference, version=4)
+#     except ValueError as e:
+#         current_app.logger.exception(f'{client_name} callback with invalid reference {provider_reference}')
+#         raise e
 
-    response_parser = sms_response_mapper[client_name]
+#     response_parser = sms_response_mapper[client_name]
 
-    # validate status
-    try:
-        notification_status, detailed_status = response_parser(status, detailed_status_code)
-        current_app.logger.info(
-            f'{client_name} callback returned status of {notification_status}'
-            f'({status}): {detailed_status}({detailed_status_code}) for reference: {provider_reference}'
-        )
-    except KeyError:
-        _process_for_status(
-            notification_status='technical-failure',
-            client_name=client_name,
-            provider_reference=provider_reference
-        )
-        raise ClientException(f'{client_name} callback failed: status {status} not found.')
+#     # validate status
+#     try:
+#         notification_status, detailed_status = response_parser(status, detailed_status_code)
+#         current_app.logger.info(
+#             f'{client_name} callback returned status of {notification_status}'
+#             f'({status}): {detailed_status}({detailed_status_code}) for reference: {provider_reference}'
+#         )
+#     except KeyError:
+#         _process_for_status(
+#             notification_status='technical-failure',
+#             client_name=client_name,
+#             provider_reference=provider_reference
+#         )
+#         raise ClientException(f'{client_name} callback failed: status {status} not found.')
 
-    _process_for_status(
-        notification_status=notification_status,
-        client_name=client_name,
-        provider_reference=provider_reference,
-        detailed_status_code=detailed_status_code
-    )
+#     _process_for_status(
+#         notification_status=notification_status,
+#         client_name=client_name,
+#         provider_reference=provider_reference,
+#         detailed_status_code=detailed_status_code
+#     )
 
 
-def _process_for_status(notification_status, client_name, provider_reference, detailed_status_code=None):
-    # record stats
-    notification = notifications_dao.update_notification_status_by_id(
-        notification_id=provider_reference,
-        status=notification_status,
-        sent_by=client_name.lower(),
-        detailed_status_code=detailed_status_code
-    )
-    if not notification:
-        return
+# def _process_for_status(notification_status, client_name, provider_reference, detailed_status_code=None):
+#     # record stats
+#     notification = notifications_dao.update_notification_status_by_id(
+#         notification_id=provider_reference,
+#         status=notification_status,
+#         sent_by=client_name.lower(),
+#         detailed_status_code=detailed_status_code
+#     )
+#     if not notification:
+#         return
 
-    statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status))
+#     statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status))
 
-    if notification.sent_at:
-        statsd_client.timing_with_dates(
-            f'callback.{client_name.lower()}.{notification_status}.elapsed-time',
-            datetime.utcnow(),
-            notification.sent_at
-        )
+#     if notification.sent_at:
+#         statsd_client.timing_with_dates(
+#             f'callback.{client_name.lower()}.{notification_status}.elapsed-time',
+#             datetime.utcnow(),
+#             notification.sent_at
+#         )
 
-    if notification.billable_units == 0:
-        service = notification.service
-        template_model = dao_get_template_by_id(notification.template_id, notification.template_version)
+#     if notification.billable_units == 0:
+#         service = notification.service
+#         template_model = dao_get_template_by_id(notification.template_id, notification.template_version)
 
-        template = SMSMessageTemplate(
-            template_model.__dict__,
-            values=notification.personalisation,
-            prefix=service.name,
-            show_prefix=service.prefix_sms,
-        )
-        notification.billable_units = template.fragment_count
-        notifications_dao.dao_update_notification(notification)
+#         template = SMSMessageTemplate(
+#             template_model.__dict__,
+#             values=notification.personalisation,
+#             prefix=service.name,
+#             show_prefix=service.prefix_sms,
+#         )
+#         notification.billable_units = template.fragment_count
+#         notifications_dao.dao_update_notification(notification)
 
-    if notification_status != NOTIFICATION_PENDING:
-        check_and_queue_callback_task(notification)
+#     if notification_status != NOTIFICATION_PENDING:
+#         check_and_queue_callback_task(notification)
diff --git a/app/notifications/notifications_sms_callback.py b/app/notifications/notifications_sms_callback.py
index 4448191d2..87fb99c3d 100644
--- a/app/notifications/notifications_sms_callback.py
+++ b/app/notifications/notifications_sms_callback.py
@@ -6,6 +6,9 @@ sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifi
 register_errors(sms_callback_blueprint)
 
 # TODO SNS SMS delivery receipts delivered here
+# This file should likely be deleted, since SNS does not use callback https calls
+# Leaving for now to have an example of what jobs MMG did that we may want to replicate in the
+# eventual SNS method.
 
 # @sms_callback_blueprint.route('/mmg', methods=['POST'])
 # def process_mmg_response():
@@ -30,27 +33,6 @@ register_errors(sms_callback_blueprint)
 #     return jsonify(result='success'), 200
 
 
-# @sms_callback_blueprint.route('/firetext', methods=['POST'])
-# def process_firetext_response():
-#     client_name = 'Firetext'
-#     errors = validate_callback_data(data=request.form,
-#                                     fields=['status', 'reference'],
-#                                     client_name=client_name)
-#     if errors:
-#         raise InvalidRequest(errors, status_code=400)
-
-#     status = request.form.get('status')
-#     detailed_status_code = request.form.get('code')
-#     provider_reference = request.form.get('reference')
-
-#     process_sms_client_response.apply_async(
-#         [status, provider_reference, client_name, detailed_status_code],
-#         queue=QueueNames.SMS_CALLBACKS,
-#     )
-
-#     return jsonify(result='success'), 200
-
-
 def validate_callback_data(data, fields, client_name):
     errors = []
     for f in fields:
diff --git a/tests/app/notifications/test_notifications_sms_callbacks.py b/tests/app/notifications/test_notifications_sms_callbacks.py
index 523e7e584..773aba892 100644
--- a/tests/app/notifications/test_notifications_sms_callbacks.py
+++ b/tests/app/notifications/test_notifications_sms_callbacks.py
@@ -4,13 +4,6 @@ from flask import json
 from app.notifications.notifications_sms_callback import validate_callback_data
 
 
-def firetext_post(client, data):
-    return client.post(
-        path='/notifications/sms/firetext',
-        data=data,
-        headers=[('Content-Type', 'application/x-www-form-urlencoded')])
-
-
 def mmg_post(client, data):
     return client.post(
         path='/notifications/sms/mmg',
@@ -18,80 +11,6 @@ def mmg_post(client, data):
         headers=[('Content-Type', 'application/json')])
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
-def test_firetext_callback_should_not_need_auth(client, mocker):
-    mocker.patch('app.notifications.notifications_sms_callback.process_sms_client_response')
-    data = 'mobile=441234123123&status=0&reference=notification_id&time=2016-03-10 14:17:00'
-
-    response = firetext_post(client, data)
-    assert response.status_code == 200
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
-def test_firetext_callback_should_return_400_if_empty_reference(client, mocker):
-    data = 'mobile=441234123123&status=0&reference=&time=2016-03-10 14:17:00'
-    response = firetext_post(client, data)
-
-    json_resp = json.loads(response.get_data(as_text=True))
-    assert response.status_code == 400
-    assert json_resp['result'] == 'error'
-    assert json_resp['message'] == ['Firetext callback failed: reference missing']
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
-def test_firetext_callback_should_return_400_if_no_reference(client, mocker):
-    data = 'mobile=441234123123&status=0&time=2016-03-10 14:17:00'
-    response = firetext_post(client, data)
-    json_resp = json.loads(response.get_data(as_text=True))
-    assert response.status_code == 400
-    assert json_resp['result'] == 'error'
-    assert json_resp['message'] == ['Firetext callback failed: reference missing']
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
-def test_firetext_callback_should_return_400_if_no_status(client, mocker):
-    data = 'mobile=441234123123&time=2016-03-10 14:17:00&reference=notification_id'
-    response = firetext_post(client, data)
-    json_resp = json.loads(response.get_data(as_text=True))
-    assert response.status_code == 400
-    assert json_resp['result'] == 'error'
-    assert json_resp['message'] == ['Firetext callback failed: status missing']
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
-def test_firetext_callback_should_return_200_and_call_task_with_valid_data(client, mocker):
-    mock_celery = mocker.patch(
-        'app.notifications.notifications_sms_callback.process_sms_client_response.apply_async')
-
-    data = 'mobile=441234123123&status=0&time=2016-03-10 14:17:00&reference=notification_id'
-    response = firetext_post(client, data)
-    json_resp = json.loads(response.get_data(as_text=True))
-    assert response.status_code == 200
-    assert json_resp['result'] == 'success'
-
-    mock_celery.assert_called_once_with(
-        ['0', 'notification_id', 'Firetext', None],
-        queue='sms-callbacks',
-    )
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Firetext removal")
-def test_firetext_callback_including_a_code_should_return_200_and_call_task_with_valid_data(client, mocker):
-    mock_celery = mocker.patch(
-        'app.notifications.notifications_sms_callback.process_sms_client_response.apply_async')
-
-    data = 'mobile=441234123123&status=1&code=101&time=2016-03-10 14:17:00&reference=notification_id'
-    response = firetext_post(client, data)
-    json_resp = json.loads(response.get_data(as_text=True))
-    assert response.status_code == 200
-    assert json_resp['result'] == 'success'
-
-    mock_celery.assert_called_once_with(
-        ['1', 'notification_id', 'Firetext', '101'],
-        queue='sms-callbacks',
-    )
-
-
 @pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
 def test_mmg_callback_should_not_need_auth(client, mocker, sample_notification):
     mocker.patch('app.notifications.notifications_sms_callback.process_sms_client_response')
diff --git a/tests/app/notifications/test_receive_notification.py b/tests/app/notifications/test_receive_notification.py
index 61a967d58..90ddbd6a5 100644
--- a/tests/app/notifications/test_receive_notification.py
+++ b/tests/app/notifications/test_receive_notification.py
@@ -262,7 +262,7 @@ def test_strip_leading_country_code(number, expected):
     ["", [], 401],
     ["testkey", [], 403],
 ])
-def test_mmg_inbound_sms_auth(notify_db_session, notify_api, client, mocker, auth, keys, status_code):
+def test_sns_inbound_sms_auth(notify_db_session, notify_api, client, mocker, auth, keys, status_code):
     mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
 
     create_service_with_inbound_number(

From d70e1b125a22d6f234c5411cdf520b06d9543983 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 22 Dec 2022 10:52:08 -0500
Subject: [PATCH 20/46] Remove NOTIFY_EMAIL_DOMAIN override in CI test runs

---
 .github/workflows/checks.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 9143504a4..fe5e1e57f 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -11,7 +11,6 @@ env:
   FLASK_APP: application.py
   FLASK_ENV: development
   WERKZEUG_DEBUG_PIN: off
-  NOTIFY_EMAIL_DOMAIN: dispostable.com
   REDIS_ENABLED: 0
   AWS_REGION: us-west-2
   AWS_PINPOINT_REGION: us-west-2

From 036a511e12aae69cc2f9c7d3baf7db532bf39706 Mon Sep 17 00:00:00 2001
From: Steven Reilly <stvnrlly@users.noreply.github.com>
Date: Fri, 23 Dec 2022 10:13:28 -0500
Subject: [PATCH 21/46] update sandbox teardown steps

---
 docs/deploying.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/deploying.md b/docs/deploying.md
index 15c41afbc..079960d26 100644
--- a/docs/deploying.md
+++ b/docs/deploying.md
@@ -42,7 +42,12 @@ for experimenting with infrastructure changes without going through the full CI/
 Rules for use:
 
 1. Ensure that no other developer is using the environment, as there is nothing stopping changes from overwriting each other.
-1. Clean up services you create when you are done. `terraform destroy` from within the `terraform/sandbox` directory should do it.
+1. Clean up when you are done: 
+    - `terraform destroy` from within the `terraform/sandbox` directory will take care of the provisioned services
+    - Delete the routes shown in `cf routes`
+    - Delete the apps shown in `cf apps`
+    - Delete the service keys for any remaining space deployers, likely `cf dsk <space-deployer> service-account-key`
+    - Delete the space deployers still shown in `cf services`
 
 ### Deploying to the sandbox
 

From 276546fc99348237407623baab8b03b518d2560f Mon Sep 17 00:00:00 2001
From: Steven Reilly <stvnrlly@users.noreply.github.com>
Date: Fri, 23 Dec 2022 11:25:44 -0500
Subject: [PATCH 22/46] simplify cleanup steps

Co-authored-by: Ryan Ahearn <ryan.ahearn@gsa.gov>
---
 docs/deploying.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/deploying.md b/docs/deploying.md
index 079960d26..264c87955 100644
--- a/docs/deploying.md
+++ b/docs/deploying.md
@@ -44,8 +44,7 @@ Rules for use:
 1. Ensure that no other developer is using the environment, as there is nothing stopping changes from overwriting each other.
 1. Clean up when you are done: 
     - `terraform destroy` from within the `terraform/sandbox` directory will take care of the provisioned services
-    - Delete the routes shown in `cf routes`
-    - Delete the apps shown in `cf apps`
+    - Delete the apps and routes shown in `cf apps` by running `cf delete APP_NAME -r`
     - Delete the service keys for any remaining space deployers, likely `cf dsk <space-deployer> service-account-key`
     - Delete the space deployers still shown in `cf services`
 

From d7b889767e0afb636e725bc3f6497e0963227532 Mon Sep 17 00:00:00 2001
From: Steven Reilly <stvnrlly@users.noreply.github.com>
Date: Fri, 23 Dec 2022 11:26:11 -0500
Subject: [PATCH 23/46] simplify to use the script for this exact purpose

Co-authored-by: Ryan Ahearn <ryan.ahearn@gsa.gov>
---
 docs/deploying.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/deploying.md b/docs/deploying.md
index 264c87955..94d85af36 100644
--- a/docs/deploying.md
+++ b/docs/deploying.md
@@ -45,8 +45,7 @@ Rules for use:
 1. Clean up when you are done: 
     - `terraform destroy` from within the `terraform/sandbox` directory will take care of the provisioned services
     - Delete the apps and routes shown in `cf apps` by running `cf delete APP_NAME -r`
-    - Delete the service keys for any remaining space deployers, likely `cf dsk <space-deployer> service-account-key`
-    - Delete the space deployers still shown in `cf services`
+    - Delete the space deployers still shown in `cf services` by running `terraform/destroy_service_account.sh -s notify-sandbox -u <space-deployer>`
 
 ### Deploying to the sandbox
 

From bc21cd3d3f3052e87fea95f2c8dbed9e447fb391 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 3 Jan 2023 09:43:32 -0500
Subject: [PATCH 24/46] Remove restart: always from devcontainer

---
 docker-compose.devcontainer.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker-compose.devcontainer.yml b/docker-compose.devcontainer.yml
index 62bdfad51..d08ca2972 100644
--- a/docker-compose.devcontainer.yml
+++ b/docker-compose.devcontainer.yml
@@ -48,7 +48,6 @@ services:
       - redis
     links:
       - db
-    restart: always
   worker:
     container_name: worker
     image: dev-notification-api

From 7e02e6b33d5629cc097a6e299d6da825bcd4d958 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 3 Jan 2023 09:44:53 -0500
Subject: [PATCH 25/46] Update to most recent pip-audit action

---
 .github/workflows/checks.yml       | 2 +-
 .github/workflows/daily_checks.yml | 2 +-
 Makefile                           | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 9143504a4..0d3ca945a 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -60,7 +60,7 @@ jobs:
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
         run: pipenv requirements > requirements.txt
-      - uses: trailofbits/gh-action-pip-audit@v1.0.0
+      - uses: pypa/gh-action-pip-audit@v1.0.4
         with:
           inputs: requirements.txt
           ignore-vulns: PYSEC-2022-237
diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml
index c0b981b7c..34cef2173 100644
--- a/.github/workflows/daily_checks.yml
+++ b/.github/workflows/daily_checks.yml
@@ -29,7 +29,7 @@ jobs:
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
         run: pipenv requirements > requirements.txt
-      - uses: trailofbits/gh-action-pip-audit@v1.0.0
+      - uses: pypa/gh-action-pip-audit@v1.0.4
         with:
           inputs: requirements.txt
           ignore-vulns: PYSEC-2022-237
diff --git a/Makefile b/Makefile
index 9ced0aaa5..9b22547bc 100644
--- a/Makefile
+++ b/Makefile
@@ -66,8 +66,8 @@ freeze-requirements: ## Pin all requirements including sub dependencies into req
 audit:
 	pipenv requirements > requirements.txt
 	pipenv requirements --dev > requirements_for_test.txt
-	pipenv run pip-audit -r requirements.txt -l --ignore-vuln PYSEC-2022-237
-	-pipenv run pip-audit -r requirements_for_test.txt -l
+	pipenv run pip-audit -r requirements.txt --ignore-vuln PYSEC-2022-237
+	-pipenv run pip-audit -r requirements_for_test.txt
 
 .PHONY: static-scan
 static-scan:

From 7878316790d6597d2f23ef09df697874490f5fd2 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 3 Jan 2023 10:23:14 -0500
Subject: [PATCH 26/46] Remove placeholder sms callback comments

---
 app/__init__.py                               |   8 -
 .../process_sms_client_response_tasks.py      |  80 --------
 .../notifications_sms_callback.py             |  42 ----
 .../test_process_sms_client_response_tasks.py | 179 ------------------
 .../test_notifications_sms_callbacks.py       | 103 ----------
 5 files changed, 412 deletions(-)
 delete mode 100644 app/celery/process_sms_client_response_tasks.py
 delete mode 100644 app/notifications/notifications_sms_callback.py
 delete mode 100644 tests/app/celery/test_process_sms_client_response_tasks.py
 delete mode 100644 tests/app/notifications/test_notifications_sms_callbacks.py

diff --git a/app/__init__.py b/app/__init__.py
index bfd519fe5..456194694 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -150,9 +150,6 @@ def register_blueprint(application):
     from app.notifications.notifications_ses_callback import (
         ses_callback_blueprint,
     )
-    from app.notifications.notifications_sms_callback import (
-        sms_callback_blueprint,
-    )
     from app.notifications.receive_notifications import (
         receive_notifications_blueprint,
     )
@@ -201,11 +198,6 @@ def register_blueprint(application):
     ses_callback_blueprint.before_request(requires_no_auth)
     application.register_blueprint(ses_callback_blueprint)
 
-    # delivery receipts
-    # TODO: make sure research mode can still trigger sms callbacks, then re-enable this
-    sms_callback_blueprint.before_request(requires_no_auth)
-    application.register_blueprint(sms_callback_blueprint)
-
     # inbound sms
     receive_notifications_blueprint.before_request(requires_no_auth)
     application.register_blueprint(receive_notifications_blueprint)
diff --git a/app/celery/process_sms_client_response_tasks.py b/app/celery/process_sms_client_response_tasks.py
deleted file mode 100644
index aff0aad15..000000000
--- a/app/celery/process_sms_client_response_tasks.py
+++ /dev/null
@@ -1,80 +0,0 @@
-from app import notify_celery
-
-sms_response_mapper = {
-    # 'SNS': get_sns_responses,
-}
-
-
-# this is used by notifications_sms_callback and needs to be heavily changed for SNS
-# leaving for now as an example of what MMG did and what we may want to replicate in the eventual
-# SNS method
-@notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
-def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
-    raise Exception("process_sms_client_response not implemented")
-#     # validate reference
-#     try:
-#         uuid.UUID(provider_reference, version=4)
-#     except ValueError as e:
-#         current_app.logger.exception(f'{client_name} callback with invalid reference {provider_reference}')
-#         raise e
-
-#     response_parser = sms_response_mapper[client_name]
-
-#     # validate status
-#     try:
-#         notification_status, detailed_status = response_parser(status, detailed_status_code)
-#         current_app.logger.info(
-#             f'{client_name} callback returned status of {notification_status}'
-#             f'({status}): {detailed_status}({detailed_status_code}) for reference: {provider_reference}'
-#         )
-#     except KeyError:
-#         _process_for_status(
-#             notification_status='technical-failure',
-#             client_name=client_name,
-#             provider_reference=provider_reference
-#         )
-#         raise ClientException(f'{client_name} callback failed: status {status} not found.')
-
-#     _process_for_status(
-#         notification_status=notification_status,
-#         client_name=client_name,
-#         provider_reference=provider_reference,
-#         detailed_status_code=detailed_status_code
-#     )
-
-
-# def _process_for_status(notification_status, client_name, provider_reference, detailed_status_code=None):
-#     # record stats
-#     notification = notifications_dao.update_notification_status_by_id(
-#         notification_id=provider_reference,
-#         status=notification_status,
-#         sent_by=client_name.lower(),
-#         detailed_status_code=detailed_status_code
-#     )
-#     if not notification:
-#         return
-
-#     statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status))
-
-#     if notification.sent_at:
-#         statsd_client.timing_with_dates(
-#             f'callback.{client_name.lower()}.{notification_status}.elapsed-time',
-#             datetime.utcnow(),
-#             notification.sent_at
-#         )
-
-#     if notification.billable_units == 0:
-#         service = notification.service
-#         template_model = dao_get_template_by_id(notification.template_id, notification.template_version)
-
-#         template = SMSMessageTemplate(
-#             template_model.__dict__,
-#             values=notification.personalisation,
-#             prefix=service.name,
-#             show_prefix=service.prefix_sms,
-#         )
-#         notification.billable_units = template.fragment_count
-#         notifications_dao.dao_update_notification(notification)
-
-#     if notification_status != NOTIFICATION_PENDING:
-#         check_and_queue_callback_task(notification)
diff --git a/app/notifications/notifications_sms_callback.py b/app/notifications/notifications_sms_callback.py
deleted file mode 100644
index 87fb99c3d..000000000
--- a/app/notifications/notifications_sms_callback.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from flask import Blueprint
-
-from app.errors import register_errors
-
-sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms")
-register_errors(sms_callback_blueprint)
-
-# TODO SNS SMS delivery receipts delivered here
-# This file should likely be deleted, since SNS does not use callback https calls
-# Leaving for now to have an example of what jobs MMG did that we may want to replicate in the
-# eventual SNS method.
-
-# @sms_callback_blueprint.route('/mmg', methods=['POST'])
-# def process_mmg_response():
-#     client_name = 'MMG'
-#     data = json.loads(request.data)
-#     errors = validate_callback_data(data=data,
-#                                     fields=['status', 'CID'],
-#                                     client_name=client_name)
-#     if errors:
-#         raise InvalidRequest(errors, status_code=400)
-
-#     status = str(data.get('status'))
-#     detailed_status_code = str(data.get('substatus'))
-
-#     provider_reference = data.get('CID')
-
-#     process_sms_client_response.apply_async(
-#         [status, provider_reference, client_name, detailed_status_code],
-#         queue=QueueNames.SMS_CALLBACKS,
-#     )
-
-#     return jsonify(result='success'), 200
-
-
-def validate_callback_data(data, fields, client_name):
-    errors = []
-    for f in fields:
-        if not str(data.get(f, '')):
-            error = "{} callback failed: {} missing".format(client_name, f)
-            errors.append(error)
-    return errors if len(errors) > 0 else None
diff --git a/tests/app/celery/test_process_sms_client_response_tasks.py b/tests/app/celery/test_process_sms_client_response_tasks.py
deleted file mode 100644
index 34bd6807c..000000000
--- a/tests/app/celery/test_process_sms_client_response_tasks.py
+++ /dev/null
@@ -1,179 +0,0 @@
-import uuid
-from datetime import datetime
-
-import pytest
-from freezegun import freeze_time
-
-from app import statsd_client
-from app.celery.process_sms_client_response_tasks import (
-    process_sms_client_response,
-)
-from app.clients import ClientException
-from app.models import NOTIFICATION_TECHNICAL_FAILURE
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-def test_process_sms_client_response_raises_error_if_reference_is_not_a_valid_uuid(client):
-    with pytest.raises(ValueError):
-        process_sms_client_response(
-            status='000', provider_reference='something-bad', client_name='sms-client')
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-@pytest.mark.parametrize('client_name', ('Firetext', 'MMG'))
-def test_process_sms_response_raises_client_exception_for_unknown_status(
-    sample_notification,
-    mocker,
-    client_name,
-):
-    with pytest.raises(ClientException) as e:
-        process_sms_client_response(
-            status='000',
-            provider_reference=str(sample_notification.id),
-            client_name=client_name,
-        )
-
-    assert f"{client_name} callback failed: status {'000'} not found." in str(e.value)
-    assert sample_notification.status == NOTIFICATION_TECHNICAL_FAILURE
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-@pytest.mark.parametrize('status, detailed_status_code, sms_provider, expected_notification_status, reason', [
-    ('0', None, 'Firetext', 'delivered', None),
-    ('1', '101', 'Firetext', 'permanent-failure', 'Unknown Subscriber'),
-    ('2', '102', 'Firetext', 'pending', 'Absent Subscriber'),
-    ('2', '1', 'MMG', 'permanent-failure', "Number does not exist"),
-    ('3', '2', 'MMG', 'delivered', "Delivered to operator"),
-    ('4', '27', 'MMG', 'temporary-failure', "Absent Subscriber"),
-    ('5', '13', 'MMG', 'permanent-failure', "Sender id blacklisted"),
-])
-def test_process_sms_client_response_updates_notification_status(
-    sample_notification,
-    mocker,
-    status,
-    detailed_status_code,
-    sms_provider,
-    expected_notification_status,
-    reason
-):
-    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.info')
-    sample_notification.status = 'sending'
-
-    process_sms_client_response(status, str(sample_notification.id), sms_provider, detailed_status_code)
-
-    message = f"{sms_provider} callback returned status of {expected_notification_status}({status}): {reason}({detailed_status_code}) for reference: {sample_notification.id}"  # noqa
-    mock_logger.assert_any_call(message)
-    assert sample_notification.status == expected_notification_status
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-@pytest.mark.parametrize('detailed_status_code, expected_notification_status, reason', [
-    ('101', 'permanent-failure', 'Unknown Subscriber'),
-    ('102', 'temporary-failure', 'Absent Subscriber'),
-    (None, 'temporary-failure', None),
-    ('000', 'temporary-failure', 'No error reported')
-])
-def test_process_sms_client_response_updates_notification_status_when_called_second_time(
-    sample_notification,
-    mocker,
-    detailed_status_code,
-    expected_notification_status,
-    reason
-):
-    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.info')
-    sample_notification.status = 'sending'
-    process_sms_client_response('2', str(sample_notification.id), 'Firetext')
-
-    process_sms_client_response('1', str(sample_notification.id), 'Firetext', detailed_status_code)
-
-    if detailed_status_code:
-        message = f'Updating notification id {sample_notification.id} to status {expected_notification_status}, reason: {reason}'  # noqa
-        mock_logger.assert_called_with(message)
-
-    assert sample_notification.status == expected_notification_status
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-@pytest.mark.parametrize('detailed_status_code', ['102', None, '000'])
-def test_process_sms_client_response_updates_notification_status_to_pending_with_and_without_failure_code_present(
-    sample_notification,
-    mocker,
-    detailed_status_code
-):
-    sample_notification.status = 'sending'
-
-    process_sms_client_response('2', str(sample_notification.id), 'Firetext', detailed_status_code)
-
-    assert sample_notification.status == 'pending'
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-def test_process_sms_client_response_updates_notification_status_when_detailed_status_code_not_recognised(
-    sample_notification,
-    mocker,
-):
-    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning')
-    sample_notification.status = 'sending'
-    process_sms_client_response('2', str(sample_notification.id), 'Firetext')
-
-    process_sms_client_response('1', str(sample_notification.id), 'Firetext', '789')
-
-    mock_logger.assert_called_once_with('Failure code 789 from Firetext not recognised')
-    assert sample_notification.status == 'temporary-failure'
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-def test_sms_response_does_not_send_callback_if_notification_is_not_in_the_db(sample_service, mocker):
-    send_mock = mocker.patch('app.celery.process_sms_client_response_tasks.check_and_queue_callback_task')
-    reference = str(uuid.uuid4())
-    process_sms_client_response(status='3', provider_reference=reference, client_name='MMG')
-    send_mock.assert_not_called()
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-@freeze_time('2001-01-01T12:00:00')
-def test_process_sms_client_response_records_statsd_metrics(sample_notification, client, mocker):
-    mocker.patch('app.statsd_client.incr')
-    mocker.patch('app.statsd_client.timing_with_dates')
-
-    sample_notification.status = 'sending'
-    sample_notification.sent_at = datetime.utcnow()
-
-    process_sms_client_response('0', str(sample_notification.id), 'Firetext')
-
-    statsd_client.incr.assert_any_call("callback.firetext.delivered")
-    statsd_client.timing_with_dates.assert_any_call(
-        "callback.firetext.delivered.elapsed-time", datetime.utcnow(), sample_notification.sent_at
-    )
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-def test_process_sms_updates_billable_units_if_zero(sample_notification):
-    sample_notification.billable_units = 0
-    process_sms_client_response('3', str(sample_notification.id), 'MMG')
-
-    assert sample_notification.billable_units == 1
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-def test_process_sms_response_does_not_send_service_callback_for_pending_notifications(sample_notification, mocker):
-    send_mock = mocker.patch('app.celery.process_sms_client_response_tasks.check_and_queue_callback_task')
-    process_sms_client_response('2', str(sample_notification.id), 'Firetext')
-    send_mock.assert_not_called()
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-def test_outcome_statistics_called_for_successful_callback(sample_notification, mocker):
-    send_mock = mocker.patch('app.celery.process_sms_client_response_tasks.check_and_queue_callback_task')
-    reference = str(sample_notification.id)
-
-    process_sms_client_response('3', reference, 'MMG')
-    send_mock.assert_called_once_with(sample_notification)
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers")
-def test_process_sms_updates_sent_by_with_client_name_if_not_in_noti(sample_notification):
-    sample_notification.sent_by = None
-    process_sms_client_response('3', str(sample_notification.id), 'MMG')
-
-    assert sample_notification.sent_by == 'mmg'
diff --git a/tests/app/notifications/test_notifications_sms_callbacks.py b/tests/app/notifications/test_notifications_sms_callbacks.py
deleted file mode 100644
index 773aba892..000000000
--- a/tests/app/notifications/test_notifications_sms_callbacks.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import pytest
-from flask import json
-
-from app.notifications.notifications_sms_callback import validate_callback_data
-
-
-def mmg_post(client, data):
-    return client.post(
-        path='/notifications/sms/mmg',
-        data=data,
-        headers=[('Content-Type', 'application/json')])
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
-def test_mmg_callback_should_not_need_auth(client, mocker, sample_notification):
-    mocker.patch('app.notifications.notifications_sms_callback.process_sms_client_response')
-    data = json.dumps({"reference": "mmg_reference",
-                       "CID": str(sample_notification.id),
-                       "MSISDN": "447777349060",
-                       "status": "3",
-                       "deliverytime": "2016-04-05 16:01:07"})
-
-    response = mmg_post(client, data)
-    assert response.status_code == 200
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
-def test_process_mmg_response_returns_400_for_malformed_data(client):
-    data = json.dumps({"reference": "mmg_reference",
-                       "monkey": 'random thing',
-                       "MSISDN": "447777349060",
-                       "no_status": 00,
-                       "deliverytime": "2016-04-05 16:01:07"})
-
-    response = mmg_post(client, data)
-    assert response.status_code == 400
-    json_data = json.loads(response.data)
-    assert json_data['result'] == 'error'
-    assert len(json_data['message']) == 2
-    assert "{} callback failed: {} missing".format('MMG', 'status') in json_data['message']
-    assert "{} callback failed: {} missing".format('MMG', 'CID') in json_data['message']
-
-
-@pytest.mark.skip(reason="Needs updating for TTS: MMG removal")
-def test_mmg_callback_should_return_200_and_call_task_with_valid_data(client, mocker):
-    mock_celery = mocker.patch(
-        'app.notifications.notifications_sms_callback.process_sms_client_response.apply_async')
-    data = json.dumps({"reference": "mmg_reference",
-                       "CID": "notification_id",
-                       "MSISDN": "447777349060",
-                       "status": "3",
-                       "substatus": "5",
-                       "deliverytime": "2016-04-05 16:01:07"})
-
-    response = mmg_post(client, data)
-
-    assert response.status_code == 200
-    json_data = json.loads(response.data)
-    assert json_data['result'] == 'success'
-
-    mock_celery.assert_called_once_with(
-        ['3', 'notification_id', 'MMG', '5'],
-        queue='sms-callbacks',
-    )
-
-
-def test_validate_callback_data_returns_none_when_valid():
-    form = {'status': 'good',
-            'reference': 'send-sms-code'}
-    fields = ['status', 'reference']
-    client_name = 'sms client'
-
-    assert validate_callback_data(form, fields, client_name) is None
-
-
-def test_validate_callback_data_return_errors_when_fields_are_empty():
-    form = {'monkey': 'good'}
-    fields = ['status', 'cid']
-    client_name = 'sms client'
-
-    errors = validate_callback_data(form, fields, client_name)
-    assert len(errors) == 2
-    assert "{} callback failed: {} missing".format(client_name, 'status') in errors
-    assert "{} callback failed: {} missing".format(client_name, 'cid') in errors
-
-
-def test_validate_callback_data_can_handle_integers():
-    form = {'status': 00, 'cid': 'fsdfadfsdfas'}
-    fields = ['status', 'cid']
-    client_name = 'sms client'
-
-    result = validate_callback_data(form, fields, client_name)
-    assert result is None
-
-
-def test_validate_callback_data_returns_error_for_empty_string():
-    form = {'status': '', 'cid': 'fsdfadfsdfas'}
-    fields = ['status', 'cid']
-    client_name = 'sms client'
-
-    result = validate_callback_data(form, fields, client_name)
-    assert result is not None
-    assert "{} callback failed: {} missing".format(client_name, 'status') in result

From abc7b09d0d9b083ca4cf533c145ed7e73b0bd9d0 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Wed, 4 Jan 2023 16:35:18 -0500
Subject: [PATCH 27/46] Update app to utilize updated phone validation methods

---
 Pipfile                           |   2 +-
 Pipfile.lock                      | 388 +++++++++++++++---------------
 app/celery/research_mode_tasks.py |   6 +-
 app/config.py                     |   2 +-
 app/notifications/validators.py   |   5 +-
 app/user/rest.py                  |   4 +-
 docs/data-storage.md              |  11 +-
 docs/openapi.yml                  |   2 +-
 8 files changed, 211 insertions(+), 209 deletions(-)

diff --git a/Pipfile b/Pipfile
index 1b9ef1a27..75a534f21 100644
--- a/Pipfile
+++ b/Pipfile
@@ -59,7 +59,7 @@ werkzeug = "~=2.1.1"
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
 packaging = "==21.3"
-notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils.git"}
+notifications-utils = {editable = true, ref = "validate-us-numbers", git = "https://github.com/GSA/notifications-utils.git"}
 
 [dev-packages]
 flake8 = "==4.0.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index d08abf43d..948641d49 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "d0a90dd082bf51ae47e04412c0848f00bc5e4b538804d1333e9264ca0ad2d4e9"
+            "sha256": "2f43f1aefdea61dea472c0f41aa81166c852769a8156d3e9644e5535c30a1c2a"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -277,35 +277,32 @@
         },
         "cryptography": {
             "hashes": [
-                "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd",
-                "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db",
-                "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290",
-                "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744",
-                "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb",
-                "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d",
-                "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70",
-                "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b",
-                "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876",
-                "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083",
-                "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6",
-                "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1",
-                "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00",
-                "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b",
-                "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b",
-                "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285",
-                "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9",
-                "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0",
-                "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d",
-                "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2",
-                "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8",
-                "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee",
-                "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b",
-                "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7",
-                "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353",
-                "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"
+                "sha256:1a6915075c6d3a5e1215eab5d99bcec0da26036ff2102a1038401d6ef5bef25b",
+                "sha256:1ee1fd0de9851ff32dbbb9362a4d833b579b4a6cc96883e8e6d2ff2a6bc7104f",
+                "sha256:407cec680e811b4fc829de966f88a7c62a596faa250fc1a4b520a0355b9bc190",
+                "sha256:50386acb40fbabbceeb2986332f0287f50f29ccf1497bae31cf5c3e7b4f4b34f",
+                "sha256:6f97109336df5c178ee7c9c711b264c502b905c2d2a29ace99ed761533a3460f",
+                "sha256:754978da4d0457e7ca176f58c57b1f9de6556591c19b25b8bcce3c77d314f5eb",
+                "sha256:76c24dd4fd196a80f9f2f5405a778a8ca132f16b10af113474005635fe7e066c",
+                "sha256:7dacfdeee048814563eaaec7c4743c8aea529fe3dd53127313a792f0dadc1773",
+                "sha256:80ee674c08aaef194bc4627b7f2956e5ba7ef29c3cc3ca488cf15854838a8f72",
+                "sha256:844ad4d7c3850081dffba91cdd91950038ee4ac525c575509a42d3fc806b83c8",
+                "sha256:875aea1039d78557c7c6b4db2fe0e9d2413439f4676310a5f269dd342ca7a717",
+                "sha256:887cbc1ea60786e534b00ba8b04d1095f4272d380ebd5f7a7eb4cc274710fad9",
+                "sha256:ad04f413436b0781f20c52a661660f1e23bcd89a0e9bb1d6d20822d048cf2856",
+                "sha256:bae6c7f4a36a25291b619ad064a30a07110a805d08dc89984f4f441f6c1f3f96",
+                "sha256:c52a1a6f81e738d07f43dab57831c29e57d21c81a942f4602fac7ee21b27f288",
+                "sha256:e0a05aee6a82d944f9b4edd6a001178787d1546ec7c6223ee9a848a7ade92e39",
+                "sha256:e324de6972b151f99dc078defe8fb1b0a82c6498e37bff335f5bc6b1e3ab5a1e",
+                "sha256:e5d71c5d5bd5b5c3eebcf7c5c2bb332d62ec68921a8c593bea8c394911a005ce",
+                "sha256:f3ed2d864a2fa1666e749fe52fb8e23d8e06b8012e8bd8147c73797c506e86f1",
+                "sha256:f671c1bb0d6088e94d61d80c606d65baacc0d374e67bf895148883461cd848de",
+                "sha256:f6c0db08d81ead9576c4d94bbb27aed8d7a430fa27890f39084c2d0e2ec6b0df",
+                "sha256:f964c7dcf7802d133e8dbd1565914fa0194f9d683d82411989889ecd701e8adf",
+                "sha256:fec8b932f51ae245121c4671b4bbc030880f363354b2f0e0bd1366017d891458"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==38.0.4"
+            "version": "==39.0.0"
         },
         "defusedxml": {
             "hashes": [
@@ -420,10 +417,10 @@
         },
         "govuk-bank-holidays": {
             "hashes": [
-                "sha256:6c993f59bff512740066ae5891a9ce3155e94291560d1d5169fea6e634111a46",
-                "sha256:9e214828802bbed303f01dacee3facc2296a8bcac9c814654ac7300b37381f78"
+                "sha256:32694697e1d9795d133cf8e39a1dd69baa23bda4a72b76f476dd4ee3df7d9b13",
+                "sha256:4cf28203b96c8ac07065538a0e49f44e4f0c6553ad418600b275f5ab60d23c68"
             ],
-            "version": "==0.11"
+            "version": "==0.12"
         },
         "greenlet": {
             "hashes": [
@@ -508,11 +505,11 @@
         },
         "importlib-metadata": {
             "hashes": [
-                "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b",
-                "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"
+                "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad",
+                "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==5.1.0"
+            "version": "==6.0.0"
         },
         "iso8601": {
             "hashes": [
@@ -742,41 +739,41 @@
         "notifications-utils": {
             "editable": true,
             "git": "https://github.com/GSA/notifications-utils.git",
-            "ref": "553f4affbef4d0a9e94e9bcf2ceb5c81d92bf032"
+            "ref": "d88dff885f5db43f6573ccb5878c376dea92ec2e"
         },
         "numpy": {
             "hashes": [
-                "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d",
-                "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07",
-                "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df",
-                "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9",
-                "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d",
-                "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a",
-                "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719",
-                "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2",
-                "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280",
-                "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa",
-                "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387",
-                "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1",
-                "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43",
-                "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f",
-                "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398",
-                "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63",
-                "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de",
-                "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8",
-                "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481",
-                "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0",
-                "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d",
-                "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e",
-                "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96",
-                "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb",
-                "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6",
-                "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d",
-                "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a",
-                "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"
+                "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9",
+                "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398",
+                "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7",
+                "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2",
+                "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954",
+                "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6",
+                "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032",
+                "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36",
+                "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c",
+                "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8",
+                "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7",
+                "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f",
+                "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700",
+                "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086",
+                "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7",
+                "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2",
+                "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9",
+                "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407",
+                "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2",
+                "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36",
+                "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566",
+                "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1",
+                "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e",
+                "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51",
+                "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1",
+                "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d",
+                "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf",
+                "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.23.5"
+            "version": "==1.24.1"
         },
         "orderedset": {
             "hashes": [
@@ -802,10 +799,10 @@
         },
         "phonenumbers": {
             "hashes": [
-                "sha256:0179f688d48c0e7e161eb7b9d86d587940af1f5174f97c1fdfd893c599c0d94a",
-                "sha256:884b26f775205261f4dc861371dce217c1661a4942fb3ec3624e290fb51869bf"
+                "sha256:14f5a639eb47548595bb2baeff7bb63b32c95a4f79eed4dc7529afa2a65002d1",
+                "sha256:7d0c45c154013970b78b86f4188e0e00a8e77d6bebef7c81efdc038db876eeba"
             ],
-            "version": "==8.13.2"
+            "version": "==8.13.3"
         },
         "prometheus-client": {
             "hashes": [
@@ -931,11 +928,11 @@
         },
         "pypdf2": {
             "hashes": [
-                "sha256:41ff16ee122bad9790d57a4235281a838002d7f1cc8d631d91b6f65d709bd825",
-                "sha256:e03ef18abcc75da741a0acc1a7749253496887be38cd9887bcce1cee393da45e"
+                "sha256:a74408f69ba6271f71b9352ef4ed03dc53a31aa404d29b5d31f53bfecfee1440",
+                "sha256:d16e4205cfee272fbdc0568b68d82be796540b1537508cef59388f839c191928"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.12.1"
+            "version": "==3.0.1"
         },
         "pyproj": {
             "hashes": [
@@ -980,31 +977,36 @@
         },
         "pyrsistent": {
             "hashes": [
-                "sha256:055ab45d5911d7cae397dc418808d8802fb95262751872c841c170b0dbf51eed",
-                "sha256:111156137b2e71f3a9936baf27cb322e8024dac3dc54ec7fb9f0bcf3249e68bb",
-                "sha256:187d5730b0507d9285a96fca9716310d572e5464cadd19f22b63a6976254d77a",
-                "sha256:21455e2b16000440e896ab99e8304617151981ed40c29e9507ef1c2e4314ee95",
-                "sha256:2aede922a488861de0ad00c7630a6e2d57e8023e4be72d9d7147a9fcd2d30712",
-                "sha256:3ba4134a3ff0fc7ad225b6b457d1309f4698108fb6b35532d015dca8f5abed73",
-                "sha256:456cb30ca8bff00596519f2c53e42c245c09e1a4543945703acd4312949bfd41",
-                "sha256:71d332b0320642b3261e9fee47ab9e65872c2bd90260e5d225dabeed93cbd42b",
-                "sha256:879b4c2f4d41585c42df4d7654ddffff1239dc4065bc88b745f0341828b83e78",
-                "sha256:9cd3e9978d12b5d99cbdc727a3022da0430ad007dacf33d0bf554b96427f33ab",
-                "sha256:a178209e2df710e3f142cbd05313ba0c5ebed0a55d78d9945ac7a4e09d923308",
-                "sha256:b39725209e06759217d1ac5fcdb510e98670af9e37223985f330b611f62e7425",
-                "sha256:bfa0351be89c9fcbcb8c9879b826f4353be10f58f8a677efab0c017bf7137ec2",
-                "sha256:bfd880614c6237243ff53a0539f1cb26987a6dc8ac6e66e0c5a40617296a045e",
-                "sha256:c43bec251bbd10e3cb58ced80609c5c1eb238da9ca78b964aea410fb820d00d6",
-                "sha256:d690b18ac4b3e3cab73b0b7aa7dbe65978a172ff94970ff98d82f2031f8971c2",
-                "sha256:d6982b5a0237e1b7d876b60265564648a69b14017f3b5f908c5be2de3f9abb7a",
-                "sha256:dec3eac7549869365fe263831f576c8457f6c833937c68542d08fde73457d291",
-                "sha256:e371b844cec09d8dc424d940e54bba8f67a03ebea20ff7b7b0d56f526c71d584",
-                "sha256:e5d8f84d81e3729c3b506657dddfe46e8ba9c330bf1858ee33108f8bb2adb38a",
-                "sha256:ea6b79a02a28550c98b6ca9c35b9f492beaa54d7c5c9e9949555893c8a9234d0",
-                "sha256:f1258f4e6c42ad0b20f9cfcc3ada5bd6b83374516cd01c0960e3cb75fdca6770"
+                "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8",
+                "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440",
+                "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a",
+                "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c",
+                "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3",
+                "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393",
+                "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9",
+                "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da",
+                "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf",
+                "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64",
+                "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a",
+                "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3",
+                "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98",
+                "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2",
+                "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8",
+                "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf",
+                "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc",
+                "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7",
+                "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28",
+                "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2",
+                "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b",
+                "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a",
+                "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64",
+                "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19",
+                "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1",
+                "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9",
+                "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==0.19.2"
+            "version": "==0.19.3"
         },
         "python-dateutil": {
             "hashes": [
@@ -1032,10 +1034,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427",
-                "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"
+                "sha256:7ccfae7b4b2c067464a6733c6261673fdb8fd1be905460396b97a073e9fa683a",
+                "sha256:93007def75ae22f7cd991c84e02d434876818661f8df9ad5df9e950ff4e52cfd"
             ],
-            "version": "==2022.6"
+            "version": "==2022.7"
         },
         "pyyaml": {
             "hashes": [
@@ -1507,91 +1509,89 @@
                 "toml"
             ],
             "hashes": [
-                "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79",
-                "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a",
-                "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f",
-                "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a",
-                "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa",
-                "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398",
-                "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba",
-                "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d",
-                "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf",
-                "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b",
-                "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518",
-                "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d",
-                "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795",
-                "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2",
-                "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e",
-                "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32",
-                "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745",
-                "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b",
-                "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e",
-                "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d",
-                "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f",
-                "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660",
-                "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62",
-                "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6",
-                "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04",
-                "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c",
-                "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5",
-                "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef",
-                "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc",
-                "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae",
-                "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578",
-                "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466",
-                "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4",
-                "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91",
-                "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0",
-                "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4",
-                "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b",
-                "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe",
-                "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b",
-                "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75",
-                "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b",
-                "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c",
-                "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72",
-                "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b",
-                "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f",
-                "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e",
-                "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53",
-                "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3",
-                "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84",
-                "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"
+                "sha256:037b51ee86bc600f99b3b957c20a172431c35c2ef9c1ca34bc813ab5b51fd9f5",
+                "sha256:0bce4ad5bdd0b02e177a085d28d2cea5fc57bb4ba2cead395e763e34cf934eb1",
+                "sha256:112cfead1bd22eada8a8db9ed387bd3e8be5528debc42b5d3c1f7da4ffaf9fb5",
+                "sha256:13121fa22dcd2c7b19c5161e3fd725692448f05377b788da4502a383573227b3",
+                "sha256:18b09811f849cc958d23f733a350a66b54a8de3fed1e6128ba55a5c97ffb6f65",
+                "sha256:25fde928306034e8deecd5fc91a07432dcc282c8acb76749581a28963c9f4f3f",
+                "sha256:2b865aa679bee7fbd1c55960940dbd3252621dd81468268786c67122bbd15343",
+                "sha256:2f7c51b6074a8a3063c341953dffe48fd6674f8e4b1d3c8aa8a91f58d6e716a8",
+                "sha256:349d0b545520e8516f7b4f12373afc705d17d901e1de6a37a20e4ec9332b61f7",
+                "sha256:44d6a556de4418f1f3bfd57094b8c49f0408df5a433cf0d253eeb3075261c762",
+                "sha256:4959dc506be74e4963bd2c42f7b87d8e4b289891201e19ec551e64c6aa5441f8",
+                "sha256:49da0ff241827ebb52d5d6d5a36d33b455fa5e721d44689c95df99fd8db82437",
+                "sha256:55e46fa4168ccb7497c9be78627fcb147e06f474f846a10d55feeb5108a24ef0",
+                "sha256:5722269ed05fbdb94eef431787c66b66260ff3125d1a9afcc00facff8c45adf9",
+                "sha256:628f47eaf66727fc986d3b190d6fa32f5e6b7754a243919d28bc0fd7974c449f",
+                "sha256:62ef3800c4058844e2e3fa35faa9dd0ccde8a8aba6c763aae50342e00d4479d4",
+                "sha256:75e43c6f4ea4d122dac389aabdf9d4f0e160770a75e63372f88005d90f5bcc80",
+                "sha256:7e8b0642c38b3d3b3c01417643ccc645345b03c32a2e84ef93cdd6844d6fe530",
+                "sha256:88834e5d56d01c141c29deedacba5773fe0bed900b1edc957595a8a6c0da1c3c",
+                "sha256:985ad2af5ec3dbb4fd75d5b0735752c527ad183455520055a08cf8d6794cabfc",
+                "sha256:a530663a361eb27375cec28aea5cd282089b5e4b022ae451c4c3493b026a68a5",
+                "sha256:acef7f3a3825a2d218a03dd02f5f3cc7f27aa31d882dd780191d1ad101120d74",
+                "sha256:ae871d09901911eedda1981ea6fd0f62a999107293cdc4c4fd612321c5b34745",
+                "sha256:af6cef3796b8068713a48dd67d258dc9a6e2ebc3bd4645bfac03a09672fa5d20",
+                "sha256:af87e906355fa42447be5c08c5d44e6e1c005bf142f303f726ddf5ed6e0c8a4d",
+                "sha256:b07651e3b9af8f1a092861d88b4c74d913634a7f1f2280fca0ad041ad84e9e96",
+                "sha256:b1ffc8f58b81baed3f8962e28c30d99442079b82ce1ec836a1f67c0accad91c1",
+                "sha256:b5b38813eee5b4739f505d94247604c72eae626d5088a16dd77b08b8b1724ab3",
+                "sha256:b791beb17b32ac019a78cfbe6184f992b6273fdca31145b928ad2099435e2fcb",
+                "sha256:b82343a5bc51627b9d606f0b6b6b9551db7b6311a5dd920fa52a94beae2e8959",
+                "sha256:ba9af1218fa01b1f11c72271bc7290b701d11ad4dbc2ae97c445ecacf6858dba",
+                "sha256:bdbda870e0fda7dd0fe7db7135ca226ec4c1ade8aa76e96614829b56ca491012",
+                "sha256:bf76d79dfaea802f0f28f50153ffbc1a74ae1ee73e480baeda410b4f3e7ab25f",
+                "sha256:c1cee10662c25c94415bbb987f2ec0e6ba9e8fce786334b10be7e6a7ab958f69",
+                "sha256:c5648c7eec5cf1ba5db1cf2d6c10036a582d7f09e172990474a122e30c841361",
+                "sha256:c58cd6bb46dcb922e0d5792850aab5964433d511b3a020867650f8d930dde4f4",
+                "sha256:c5d9b480ebae60fc2cbc8d6865194136bc690538fa542ba58726433bed6e04cc",
+                "sha256:ca15308ef722f120967af7474ba6a453e0f5b6f331251e20b8145497cf1bc14a",
+                "sha256:d0df04495b76a885bfef009f45eebe8fe2fbf815ad7a83dabcf5aced62f33162",
+                "sha256:d5be4e93acce64f516bf4fd239c0e6118fc913c93fa1a3f52d15bdcc60d97b2d",
+                "sha256:d8249666c23683f74f8f93aeaa8794ac87cc61c40ff70374a825f3352a4371dc",
+                "sha256:e3f1cd1cd65695b1540b3cf7828d05b3515974a9d7c7530f762ac40f58a18161",
+                "sha256:e56fae4292e216b8deeee38ace84557b9fa85b52db005368a275427cdabb8192",
+                "sha256:e6dcc70a25cb95df0ae33dfc701de9b09c37f7dd9f00394d684a5b57257f8246",
+                "sha256:e89d5abf86c104de808108a25d171ad646c07eda96ca76c8b237b94b9c71e518",
+                "sha256:ed7c9debf7bfc63c9b9f8b595409237774ff4b061bf29fba6f53b287a2fdeab9",
+                "sha256:ef001a60e888f8741e42e5aa79ae55c91be73761e4df5e806efca1ddd62fd400",
+                "sha256:f30090e22a301952c5abd0e493a1c8358b4f0b368b49fa3e4568ed3ed68b8d1f",
+                "sha256:f79691335257d60951638dd43576b9bcd6f52baa5c1c2cd07a509bb003238372",
+                "sha256:f918e9ef4c98f477a5458238dde2a1643aed956c7213873ab6b6b82e32b8ef61",
+                "sha256:fd0a8aa431f9b7ad9eb8264f55ef83cbb254962af3775092fb6e93890dea9ca2"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.5.0"
+            "version": "==7.0.3"
         },
         "cryptography": {
             "hashes": [
-                "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd",
-                "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db",
-                "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290",
-                "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744",
-                "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb",
-                "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d",
-                "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70",
-                "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b",
-                "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876",
-                "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083",
-                "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6",
-                "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1",
-                "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00",
-                "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b",
-                "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b",
-                "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285",
-                "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9",
-                "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0",
-                "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d",
-                "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2",
-                "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8",
-                "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee",
-                "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b",
-                "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7",
-                "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353",
-                "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"
+                "sha256:1a6915075c6d3a5e1215eab5d99bcec0da26036ff2102a1038401d6ef5bef25b",
+                "sha256:1ee1fd0de9851ff32dbbb9362a4d833b579b4a6cc96883e8e6d2ff2a6bc7104f",
+                "sha256:407cec680e811b4fc829de966f88a7c62a596faa250fc1a4b520a0355b9bc190",
+                "sha256:50386acb40fbabbceeb2986332f0287f50f29ccf1497bae31cf5c3e7b4f4b34f",
+                "sha256:6f97109336df5c178ee7c9c711b264c502b905c2d2a29ace99ed761533a3460f",
+                "sha256:754978da4d0457e7ca176f58c57b1f9de6556591c19b25b8bcce3c77d314f5eb",
+                "sha256:76c24dd4fd196a80f9f2f5405a778a8ca132f16b10af113474005635fe7e066c",
+                "sha256:7dacfdeee048814563eaaec7c4743c8aea529fe3dd53127313a792f0dadc1773",
+                "sha256:80ee674c08aaef194bc4627b7f2956e5ba7ef29c3cc3ca488cf15854838a8f72",
+                "sha256:844ad4d7c3850081dffba91cdd91950038ee4ac525c575509a42d3fc806b83c8",
+                "sha256:875aea1039d78557c7c6b4db2fe0e9d2413439f4676310a5f269dd342ca7a717",
+                "sha256:887cbc1ea60786e534b00ba8b04d1095f4272d380ebd5f7a7eb4cc274710fad9",
+                "sha256:ad04f413436b0781f20c52a661660f1e23bcd89a0e9bb1d6d20822d048cf2856",
+                "sha256:bae6c7f4a36a25291b619ad064a30a07110a805d08dc89984f4f441f6c1f3f96",
+                "sha256:c52a1a6f81e738d07f43dab57831c29e57d21c81a942f4602fac7ee21b27f288",
+                "sha256:e0a05aee6a82d944f9b4edd6a001178787d1546ec7c6223ee9a848a7ade92e39",
+                "sha256:e324de6972b151f99dc078defe8fb1b0a82c6498e37bff335f5bc6b1e3ab5a1e",
+                "sha256:e5d71c5d5bd5b5c3eebcf7c5c2bb332d62ec68921a8c593bea8c394911a005ce",
+                "sha256:f3ed2d864a2fa1666e749fe52fb8e23d8e06b8012e8bd8147c73797c506e86f1",
+                "sha256:f671c1bb0d6088e94d61d80c606d65baacc0d374e67bf895148883461cd848de",
+                "sha256:f6c0db08d81ead9576c4d94bbb27aed8d7a430fa27890f39084c2d0e2ec6b0df",
+                "sha256:f964c7dcf7802d133e8dbd1565914fa0194f9d683d82411989889ecd701e8adf",
+                "sha256:fec8b932f51ae245121c4671b4bbc030880f363354b2f0e0bd1366017d891458"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==38.0.4"
+            "version": "==39.0.0"
         },
         "cyclonedx-python-lib": {
             "hashes": [
@@ -1643,11 +1643,11 @@
         },
         "gitpython": {
             "hashes": [
-                "sha256:41eea0deec2deea139b459ac03656f0dd28fc4a3387240ec1d3c259a2c47850f",
-                "sha256:cc36bfc4a3f913e66805a28e84703e419d9c264c1077e537b54f0e1af85dbefd"
+                "sha256:769c2d83e13f5d938b7688479da374c4e3d49f71549aaf462b646db9602ea6f8",
+                "sha256:cd455b0000615c60e286208ba540271af9fe531fa6a87cc590a7298785ab2882"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.1.29"
+            "version": "==3.1.30"
         },
         "honcho": {
             "hashes": [
@@ -1881,19 +1881,19 @@
         },
         "pip-audit": {
             "hashes": [
-                "sha256:40876d6ad6adf5ac64fba1ecef034db9804c35c2c18f7c22cb7b11b382c10df9",
-                "sha256:a45540ab0c5a9311315ca42c78fa8f72cf3598d5968a67d883d2d6194eda598c"
+                "sha256:e65dfbccf29f290606911d3a49cf62497d3c140e4aac0a17775438d7630588da",
+                "sha256:eab6aab5a649e93f17309bf8d57bb369a63fa9ef60e96b585bba22acdbb54b97"
             ],
             "index": "pypi",
-            "version": "==2.4.8"
+            "version": "==2.4.12"
         },
         "pip-requirements-parser": {
             "hashes": [
-                "sha256:5159b8a9485a5a0d0254a29c9bd8b8ce66db2eb9a1a0244c64dfce43f7f2ac90",
-                "sha256:aa664781e69f82ecefda542d5210aa5b0f6533be95aea39a5453943db09359ed"
+                "sha256:4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526",
+                "sha256:b4fa3a7a0be38243123cf9d1f3518da10c51bdb165a2b2985566247f9155a7d3"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==32.0.0"
+            "markers": "python_full_version >= '3.6.0'",
+            "version": "==32.0.1"
         },
         "pluggy": {
             "hashes": [
@@ -1936,11 +1936,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1",
-                "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"
+                "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297",
+                "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.13.0"
+            "version": "==2.14.0"
         },
         "pyparsing": {
             "hashes": [
@@ -2007,10 +2007,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427",
-                "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"
+                "sha256:7ccfae7b4b2c067464a6733c6261673fdb8fd1be905460396b97a073e9fa683a",
+                "sha256:93007def75ae22f7cd991c84e02d434876818661f8df9ad5df9e950ff4e52cfd"
             ],
-            "version": "==2022.6"
+            "version": "==2022.7"
         },
         "pyyaml": {
             "hashes": [
@@ -2080,11 +2080,11 @@
         },
         "rich": {
             "hashes": [
-                "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e",
-                "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"
+                "sha256:12b1d77ee7edf251b741531323f0d990f5f570a4e7c054d0bfb59fb7981ad977",
+                "sha256:3aa9eba7219b8c575c6494446a59f702552efe1aa261e7eeb95548fa586e1950"
             ],
-            "markers": "python_full_version >= '3.6.3' and python_full_version < '4.0.0'",
-            "version": "==12.6.0"
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==13.0.0"
         },
         "s3transfer": {
             "hashes": [
diff --git a/app/celery/research_mode_tasks.py b/app/celery/research_mode_tasks.py
index b59b8b235..b19de9c2a 100644
--- a/app/celery/research_mode_tasks.py
+++ b/app/celery/research_mode_tasks.py
@@ -12,9 +12,9 @@ from app.celery.process_ses_receipts_tasks import process_ses_results
 from app.config import QueueNames
 from app.models import SMS_TYPE
 
-temp_fail = "5558675303"
-perm_fail = "5558675302"
-delivered = "5558675309"
+temp_fail = "2028675303"
+perm_fail = "2028675302"
+delivered = "2028675309"
 
 delivered_email = "delivered@simulator.notify"
 perm_fail_email = "perm-fail@simulator.notify"
diff --git a/app/config.py b/app/config.py
index 8d9d58d8f..10d4220c8 100644
--- a/app/config.py
+++ b/app/config.py
@@ -316,7 +316,7 @@ class Config(object):
         'simulate-delivered-2@notifications.service.gov.uk',
         'simulate-delivered-3@notifications.service.gov.uk',
     )
-    SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222')
+    SIMULATED_SMS_NUMBERS = ('+12028675000', '+12028675111', '+12028675222')
 
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 
diff --git a/app/notifications/validators.py b/app/notifications/validators.py
index 03a98b812..7a079df66 100644
--- a/app/notifications/validators.py
+++ b/app/notifications/validators.py
@@ -151,10 +151,7 @@ def check_if_service_can_send_to_number(service, number):
     else:
         permissions = service.permissions
 
-    if (
-        # if number is international and not a crown dependency
-        international_phone_info.international and not international_phone_info.crown_dependency
-    ) and INTERNATIONAL_SMS_TYPE not in permissions:
+    if international_phone_info.international and INTERNATIONAL_SMS_TYPE not in permissions:
         raise BadRequestError(message="Cannot send to international mobile numbers")
     else:
         return international_phone_info
diff --git a/app/user/rest.py b/app/user/rest.py
index 3ec2adb9b..6d85ddd98 100644
--- a/app/user/rest.py
+++ b/app/user/rest.py
@@ -5,7 +5,7 @@ from urllib.parse import urlencode
 
 from flask import Blueprint, abort, current_app, jsonify, request
 from notifications_utils.recipients import (
-    is_uk_phone_number,
+    is_us_phone_number,
     use_numeric_sender,
 )
 from sqlalchemy.exc import IntegrityError
@@ -140,7 +140,7 @@ def update_user_attribute(user_id):
 
 
 def get_sms_reply_to_for_notify_service(recipient, template):
-    if not is_uk_phone_number(recipient) and use_numeric_sender(recipient):
+    if not is_us_phone_number(recipient) and use_numeric_sender(recipient):
         reply_to = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER']
     else:
         reply_to = template.service.get_default_sms_sender()
diff --git a/docs/data-storage.md b/docs/data-storage.md
index 54045c742..2a872c9a6 100644
--- a/docs/data-storage.md
+++ b/docs/data-storage.md
@@ -29,11 +29,11 @@ No db data is PII, but each job has a csv file in s3 containing phone numbers an
 
 * to
 * _personalization<sup>2</sup>
-* phone_prefix (?)
+* phone_prefix<sup>3</sup>
 
 #### notification_history
 
-* phone_prefix (?)
+* phone_prefix<sup>3</sup>
 
 #### inbound_sms
 
@@ -42,7 +42,7 @@ No db data is PII, but each job has a csv file in s3 containing phone numbers an
 
 #### events
 
-* data (contains user IP addresses)
+* data (contains user IP addresses)<sup>1</sup>
 
 ### Notes
 
@@ -56,6 +56,11 @@ Field-level encryption is used on these fields.
 
 Details on encryption schemes and algorithms can be found in [SC-28(1)](https://github.com/GSA/us-notify-compliance/blob/main/dist/system-security-plans/lato/sc-28.1.md)
 
+#### Note 3.
+
+Probably not PII, this is the country code of the phone.
+
+
 Data Retention Policy
 ---------------------
 
diff --git a/docs/openapi.yml b/docs/openapi.yml
index abbf8ff67..5cb4717e5 100644
--- a/docs/openapi.yml
+++ b/docs/openapi.yml
@@ -570,7 +570,7 @@ paths:
                 reference:
                   type: string
             example:
-              phone_number: "5558675309"
+              phone_number: "2028675309"
               template_id: "85b58733-7ebf-494e-bee2-a21a4ce17d58"
               personalisation:
                 variable: "value"

From 82df01653f5f0bfcc7cae9e978b66c62afcf5447 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Wed, 4 Jan 2023 16:35:25 -0500
Subject: [PATCH 28/46] Update tests to focus on US numbers

---
 tests/app/celery/test_research_mode_tasks.py  | 10 ++--
 tests/app/celery/test_tasks.py                | 22 +++----
 .../notification_dao/test_notification_dao.py | 34 +++++------
 tests/app/dao/test_services_dao.py            | 10 ++--
 tests/app/dao/test_users_dao.py               |  2 +-
 tests/app/db.py                               |  2 +-
 tests/app/delivery/test_send_to_providers.py  | 16 ++---
 .../test_process_notification.py              | 43 ++++++-------
 .../test_receive_notification.py              |  8 +--
 tests/app/notifications/test_validators.py    | 22 +++----
 .../test_POST_notification.py                 |  2 +-
 .../test_send_notification.py                 | 60 ++++++-------------
 .../test_send_one_off_notification.py         | 36 +++++------
 tests/app/service/test_rest.py                |  6 +-
 tests/app/service/test_service_guest_list.py  | 12 ++--
 tests/app/test_commands.py                    |  2 +-
 tests/app/test_model.py                       |  6 +-
 tests/app/user/test_rest.py                   | 20 +++----
 tests/app/user/test_rest_verify.py            |  2 +-
 .../test_notification_schemas.py              | 26 ++++----
 .../notifications/test_post_notifications.py  | 32 +++++-----
 21 files changed, 172 insertions(+), 201 deletions(-)

diff --git a/tests/app/celery/test_research_mode_tasks.py b/tests/app/celery/test_research_mode_tasks.py
index c9e42f62e..996b779e2 100644
--- a/tests/app/celery/test_research_mode_tasks.py
+++ b/tests/app/celery/test_research_mode_tasks.py
@@ -31,11 +31,11 @@ def test_make_sns_callback(notify_api, rmock):
         endpoint,
         json={"status": "success"},
         status_code=200)
-    send_sms_response("sns", "1234", "5558675309")
+    send_sms_response("sns", "1234", "2028675309")
 
     assert rmock.called
     assert rmock.request_history[0].url == endpoint
-    assert json.loads(rmock.request_history[0].text)['MSISDN'] == '5558675309'
+    assert json.loads(rmock.request_history[0].text)['MSISDN'] == '2028675309'
 
 
 @pytest.mark.skip(reason="Re-enable when SMS receipts exist")
@@ -70,7 +70,7 @@ def test_make_ses_callback(notify_api, mocker):
 
 @pytest.mark.skip(reason="Re-enable when SNS delivery receipts exist")
 def test_delievered_sns_callback():
-    phone_number = "5558675309"
+    phone_number = "2028675309"
     data = json.loads(sns_callback("1234", phone_number))
     assert data['MSISDN'] == phone_number
     assert data['status'] == "3"
@@ -80,7 +80,7 @@ def test_delievered_sns_callback():
 
 @pytest.mark.skip(reason="Re-enable when SNS delivery receipts exist")
 def test_perm_failure_sns_callback():
-    phone_number = "5558675302"
+    phone_number = "2028675302"
     data = json.loads(sns_callback("1234", phone_number))
     assert data['MSISDN'] == phone_number
     assert data['status'] == "5"
@@ -90,7 +90,7 @@ def test_perm_failure_sns_callback():
 
 @pytest.mark.skip(reason="Re-enable when SNS delivery receipts exist")
 def test_temp_failure_sns_callback():
-    phone_number = "5558675303"
+    phone_number = "2028675303"
     data = json.loads(sns_callback("1234", phone_number))
     assert data['MSISDN'] == phone_number
     assert data['status'] == "4"
diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py
index 5c55a439d..3780f3ad5 100644
--- a/tests/app/celery/test_tasks.py
+++ b/tests/app/celery/test_tasks.py
@@ -502,10 +502,10 @@ def test_should_put_save_sms_task_in_research_mode_queue_if_research_mode_servic
 
 
 def test_should_save_sms_if_restricted_service_and_valid_number(notify_db_session, mocker):
-    user = create_user(mobile_number="07700 900890")
+    user = create_user(mobile_number="202-867-5309")
     service = create_service(user=user, restricted=True)
     template = create_template(service=service)
-    notification = _notification_json(template, "+447700900890")  # The user’s own number, but in a different format
+    notification = _notification_json(template, "+12028675309")  # The user’s own number, but in a different format
 
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
@@ -518,7 +518,7 @@ def test_should_save_sms_if_restricted_service_and_valid_number(notify_db_sessio
     )
 
     persisted_notification = Notification.query.one()
-    assert persisted_notification.to == '+447700900890'
+    assert persisted_notification.to == '+12028675309'
     assert persisted_notification.template_id == template.id
     assert persisted_notification.template_version == template.version
     assert persisted_notification.status == 'created'
@@ -553,11 +553,11 @@ def test_save_email_should_save_default_email_reply_to_text_on_notification(noti
     assert persisted_notification.reply_to_text == 'reply_to@digital.gov.uk'
 
 
-def test_save_sms_should_save_default_smm_sender_notification_reply_to_text_on(notify_db_session, mocker):
+def test_save_sms_should_save_default_sms_sender_notification_reply_to_text_on(notify_db_session, mocker):
     service = create_service_with_defined_sms_sender(sms_sender_value='12345')
     template = create_template(service=service)
 
-    notification = _notification_json(template, to="07700 900205")
+    notification = _notification_json(template, to="2028675309")
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     notification_id = uuid.uuid4()
@@ -1204,10 +1204,10 @@ def test_save_letter_uses_template_reply_to_text(mocker, notify_db_session):
 
 
 def test_save_sms_uses_sms_sender_reply_to_text(mocker, notify_db_session):
-    service = create_service_with_defined_sms_sender(sms_sender_value='07123123123')
+    service = create_service_with_defined_sms_sender(sms_sender_value='2028675309')
     template = create_template(service=service)
 
-    notification = _notification_json(template, to="07700 900205")
+    notification = _notification_json(template, to="2028675301")
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     notification_id = uuid.uuid4()
@@ -1218,15 +1218,15 @@ def test_save_sms_uses_sms_sender_reply_to_text(mocker, notify_db_session):
     )
 
     persisted_notification = Notification.query.one()
-    assert persisted_notification.reply_to_text == '447123123123'
+    assert persisted_notification.reply_to_text == '+12028675309'
 
 
 def test_save_sms_uses_non_default_sms_sender_reply_to_text_if_provided(mocker, notify_db_session):
-    service = create_service_with_defined_sms_sender(sms_sender_value='07123123123')
+    service = create_service_with_defined_sms_sender(sms_sender_value='2028675309')
     template = create_template(service=service)
     new_sender = service_sms_sender_dao.dao_add_sms_sender_for_service(service.id, 'new-sender', False)
 
-    notification = _notification_json(template, to="07700 900205")
+    notification = _notification_json(template, to="202-867-5301")
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     notification_id = uuid.uuid4()
@@ -1918,7 +1918,7 @@ def test_save_api_email_dont_retry_if_notification_already_exists(sample_service
     ), (
         save_sms,
         'app.celery.provider_tasks.deliver_sms.apply_async',
-        '07700 900890',
+        '202-867-5309',
         {'template_type': 'sms'}
     ),
 ))
diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py
index 49e9b92c0..8ae304ebf 100644
--- a/tests/app/dao/notification_dao/test_notification_dao.py
+++ b/tests/app/dao/notification_dao/test_notification_dao.py
@@ -1080,16 +1080,14 @@ def test_dao_get_notifications_by_reference_escapes_special_character(
 
 
 @pytest.mark.parametrize('search_term', [
-    '001',
-    '100',
-    '09001',
-    '077009001',
-    '07700 9001',
-    '(0)7700 9001',
-    '4477009001',
-    '+4477009001',
-    pytest.param('+44077009001', marks=pytest.mark.skip(reason='No easy way to normalise this')),
-    pytest.param('+44(0)77009001', marks=pytest.mark.skip(reason='No easy way to normalise this')),
+    '309',
+    '530',
+    '8675309',
+    '202867',
+    '202 867',
+    '202-867-5309',
+    '2028675309',
+    '+12028675309',
 ])
 def test_dao_get_notifications_by_recipient_matches_partial_phone_numbers(
     sample_template,
@@ -1098,13 +1096,13 @@ def test_dao_get_notifications_by_recipient_matches_partial_phone_numbers(
 
     notification_1 = create_notification(
         template=sample_template,
-        to_field='+447700900100',
-        normalised_to='447700900100',
+        to_field='202-867-5309',
+        normalised_to='+12028675309',
     )
     notification_2 = create_notification(
         template=sample_template,
-        to_field='+447700900200',
-        normalised_to='447700900200',
+        to_field='202-678-5000',
+        normalised_to='+12026785000',
     )
     results = dao_get_notifications_by_recipient_or_reference(
         notification_1.service_id, search_term, notification_type='sms'
@@ -1156,7 +1154,7 @@ def test_dao_get_notifications_by_recipient_ignores_spaces(sample_template):
 
 
 @pytest.mark.parametrize('phone_search', (
-    '077', '7-7', '+44(0)7711 111111'
+    '202', '7-5', '+1 (202) 867-5309'
 ))
 @pytest.mark.parametrize('email_search', (
     'example', 'eXaMpLe',
@@ -1169,9 +1167,9 @@ def test_dao_get_notifications_by_recipient_searches_across_notification_types(
     service = create_service()
     sms_template = create_template(service=service)
     email_template = create_template(service=service, template_type='email')
-    sms = create_notification(template=sms_template, to_field='07711111111', normalised_to='447711111111')
+    sms = create_notification(template=sms_template, to_field='202-867-5309', normalised_to='+12028675309')
     email = create_notification(
-        template=email_template, to_field='077@example.com', normalised_to='077@example.com'
+        template=email_template, to_field='202@example.com', normalised_to='202@example.com'
     )
 
     results = dao_get_notifications_by_recipient_or_reference(
@@ -1186,7 +1184,7 @@ def test_dao_get_notifications_by_recipient_searches_across_notification_types(
     assert len(results.items) == 1
     assert results.items[0].id == email.id
 
-    results = dao_get_notifications_by_recipient_or_reference(service.id, '77')
+    results = dao_get_notifications_by_recipient_or_reference(service.id, '202')
     assert len(results.items) == 2
     assert results.items[0].id == email.id
     assert results.items[1].id == sms.id
diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py
index 899f003c4..404f751e2 100644
--- a/tests/app/dao/test_services_dao.py
+++ b/tests/app/dao/test_services_dao.py
@@ -258,7 +258,7 @@ def test_should_add_user_to_service(notify_db_session):
         name='Test User',
         email_address='new_user@digital.cabinet-office.gov.uk',
         password='password',
-        mobile_number='+447700900986'
+        mobile_number='+12028675309'
     )
     save_model_user(new_user, validated_email_access=True)
     dao_add_user_to_service(service, new_user)
@@ -327,7 +327,7 @@ def test_should_remove_user_from_service(notify_db_session):
         name='Test User',
         email_address='new_user@digital.cabinet-office.gov.uk',
         password='password',
-        mobile_number='+447700900986'
+        mobile_number='+12028675309'
     )
     save_model_user(new_user, validated_email_access=True)
     dao_add_user_to_service(service, new_user)
@@ -428,7 +428,7 @@ def test_get_all_user_services_only_returns_services_user_has_access_to(notify_d
         name='Test User',
         email_address='new_user@digital.cabinet-office.gov.uk',
         password='password',
-        mobile_number='+447700900986'
+        mobile_number='+12028675309'
     )
     save_model_user(new_user, validated_email_access=True)
     dao_add_user_to_service(service_3, new_user)
@@ -486,7 +486,7 @@ def test_dao_fetch_live_services_data(sample_user):
     assert results == [
         {'service_id': mock.ANY, 'service_name': 'Sample service', 'organisation_name': 'test_org_1',
             'organisation_type': 'federal', 'consent_to_research': None, 'contact_name': 'Test User',
-            'contact_email': 'notify@digital.cabinet-office.gov.uk', 'contact_mobile': '+447700900986',
+            'contact_email': 'notify@digital.cabinet-office.gov.uk', 'contact_mobile': '+12028675309',
             'live_date': datetime(2014, 4, 20, 10, 0), 'sms_volume_intent': None, 'email_volume_intent': None,
             'letter_volume_intent': None, 'sms_totals': 2, 'email_totals': 1, 'letter_totals': 1,
             'free_sms_fragment_limit': 100},
@@ -498,7 +498,7 @@ def test_dao_fetch_live_services_data(sample_user):
             'free_sms_fragment_limit': 200},
         {'service_id': mock.ANY, 'service_name': 'second', 'organisation_name': None, 'consent_to_research': None,
             'contact_name': 'Test User', 'contact_email': 'notify@digital.cabinet-office.gov.uk',
-            'contact_mobile': '+447700900986', 'live_date': datetime(2017, 4, 20, 10, 0), 'sms_volume_intent': None,
+            'contact_mobile': '+12028675309', 'live_date': datetime(2017, 4, 20, 10, 0), 'sms_volume_intent': None,
             'organisation_type': None, 'email_volume_intent': None, 'letter_volume_intent': None,
             'sms_totals': 0, 'email_totals': 0, 'letter_totals': 1,
             'free_sms_fragment_limit': 300}
diff --git a/tests/app/dao/test_users_dao.py b/tests/app/dao/test_users_dao.py
index c128d594d..46720fd72 100644
--- a/tests/app/dao/test_users_dao.py
+++ b/tests/app/dao/test_users_dao.py
@@ -38,7 +38,7 @@ from tests.app.db import (
 
 @freeze_time('2020-01-28T12:00:00')
 @pytest.mark.parametrize('phone_number', [
-    '+447700900986',
+    '+12028675309',
     '+1-800-555-5555',
 ])
 def test_create_user(notify_db_session, phone_number):
diff --git a/tests/app/db.py b/tests/app/db.py
index 729ed8b29..edcfd6cd7 100644
--- a/tests/app/db.py
+++ b/tests/app/db.py
@@ -72,7 +72,7 @@ from app.models import (
 
 def create_user(
     *,
-    mobile_number="+447700900986",
+    mobile_number="+12028675309",
     email=None,
     state='active',
     id_=None,
diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py
index 107c841d5..48ff35368 100644
--- a/tests/app/delivery/test_send_to_providers.py
+++ b/tests/app/delivery/test_send_to_providers.py
@@ -117,10 +117,10 @@ def test_should_send_personalised_template_to_correct_sms_provider_and_persist(
     mocker
 ):
     db_notification = create_notification(template=sample_sms_template_with_html,
-                                          to_field="5558675309", personalisation={"name": "Jo"},
+                                          to_field="2028675309", personalisation={"name": "Jo"},
                                           status='created',
                                           reply_to_text=sample_sms_template_with_html.service.get_default_sms_sender(),
-                                          normalised_to="5558675309"
+                                          normalised_to="2028675309"
                                           )
 
     mocker.patch('app.aws_sns_client.send_sms')
@@ -130,7 +130,7 @@ def test_should_send_personalised_template_to_correct_sms_provider_and_persist(
     )
 
     aws_sns_client.send_sms.assert_called_once_with(
-        to="5558675309",
+        to="2028675309",
         content="Sample service: Hello Jo\nHere is <em>some HTML</em> & entities",
         reference=str(db_notification.id),
         sender=current_app.config['FROM_NUMBER'],
@@ -210,9 +210,9 @@ def test_should_not_send_sms_message_when_service_is_inactive_notification_is_in
 def test_send_sms_should_use_template_version_from_notification_not_latest(
         sample_template,
         mocker):
-    db_notification = create_notification(template=sample_template, to_field='5558675309', status='created',
+    db_notification = create_notification(template=sample_template, to_field='2028675309', status='created',
                                           reply_to_text=sample_template.service.get_default_sms_sender(),
-                                          normalised_to='5558675309')
+                                          normalised_to='2028675309')
 
     mocker.patch('app.aws_sns_client.send_sms')
 
@@ -234,7 +234,7 @@ def test_send_sms_should_use_template_version_from_notification_not_latest(
     )
 
     aws_sns_client.send_sms.assert_called_once_with(
-        to="5558675309",
+        to="2028675309",
         content="Sample service: This is a template:\nwith a newline",
         reference=str(db_notification.id),
         sender=current_app.config['FROM_NUMBER'],
@@ -737,8 +737,8 @@ def test_send_sms_to_provider_should_use_normalised_to(
 ):
     send_mock = mocker.patch('app.aws_sns_client.send_sms')
     notification = create_notification(template=sample_template,
-                                       to_field='+15558675309',
-                                       normalised_to='5558675309')
+                                       to_field='+12028675309',
+                                       normalised_to='2028675309')
     send_to_providers.send_sms_to_provider(notification)
     send_mock.assert_called_once_with(to=notification.normalised_to,
                                       content=ANY,
diff --git a/tests/app/notifications/test_process_notification.py b/tests/app/notifications/test_process_notification.py
index df7e9a5e1..98e698888 100644
--- a/tests/app/notifications/test_process_notification.py
+++ b/tests/app/notifications/test_process_notification.py
@@ -122,7 +122,7 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key):
     persist_notification(
         template_id=sample_job.template.id,
         template_version=sample_job.template.version,
-        recipient='+447111111111',
+        recipient='+12028675309',
         service=sample_job.service,
         personalisation=None,
         notification_type='sms',
@@ -146,7 +146,7 @@ def test_persist_notification_with_optionals(sample_job, sample_api_key):
     assert persisted_notification.client_reference == "ref from client"
     assert persisted_notification.reference is None
     assert persisted_notification.international is False
-    assert persisted_notification.phone_prefix == '44'
+    assert persisted_notification.phone_prefix == '1'
     assert persisted_notification.rate_multiplier == 1
     assert persisted_notification.created_by_id == sample_job.created_by_id
     assert not persisted_notification.reply_to_text
@@ -297,15 +297,15 @@ def test_send_notification_to_queue_throws_exception_deletes_notification(sample
 
 
 @pytest.mark.parametrize("to_address, notification_type, expected", [
-    ("+447700900000", "sms", True),
-    ("+447700900111", "sms", True),
-    ("+447700900222", "sms", True),
-    ("07700900000", "sms", True),
-    ("7700900111", "sms", True),
+    ("+12028675000", "sms", True),
+    ("+12028675111", "sms", True),
+    ("+12028675222", "sms", True),
+    ("2028675000", "sms", True),
+    ("2028675111", "sms", True),
     ("simulate-delivered@notifications.service.gov.uk", "email", True),
     ("simulate-delivered-2@notifications.service.gov.uk", "email", True),
     ("simulate-delivered-3@notifications.service.gov.uk", "email", True),
-    ("07515896969", "sms", False),
+    ("2028675309", "sms", False),
     ("valid_email@test.com", "email", False)
 ])
 def test_simulated_recipient(notify_api, to_address, notification_type, expected):
@@ -317,7 +317,7 @@ def test_simulated_recipient(notify_api, to_address, notification_type, expected
         'simulate-delivered-2@notifications.service.gov.uk',
         'simulate-delivered-2@notifications.service.gov.uk'
     )
-    SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222')
+    SIMULATED_SMS_NUMBERS = ('+12028675000', '+12028675111', '+12028675222')
     """
     formatted_address = None
 
@@ -332,12 +332,10 @@ def test_simulated_recipient(notify_api, to_address, notification_type, expected
 
 
 @pytest.mark.parametrize('recipient, expected_international, expected_prefix, expected_units', [
-    ('7900900123', False, '44', 1),  # UK
-    ('+447900900123', False, '44', 1),  # UK
-    ('07700910222', True, '44', 1),  # UK (Jersey)
-    ('07700900222', False, '44', 1),  # TV number
-    ('73122345678', True, '7', 1),  # Russia
-    ('360623400400', True, '36', 3)]  # Hungary
+    ('+447900900123', True, '44', 1),  # UK
+    ('+73122345678', True, '7', 1),  # Russia
+    ('+360623400400', True, '36', 1),  # Hungary
+    ('2028675309', False, '1', 1)]  # USA
 )
 def test_persist_notification_with_international_info_stores_correct_info(
     sample_job,
@@ -394,15 +392,12 @@ def test_persist_notification_with_international_info_does_not_store_for_email(
 
 
 @pytest.mark.parametrize('recipient, expected_recipient_normalised', [
-    ('7900900123', '447900900123'),
-    ('+447900   900 123', '447900900123'),
-    ('  07700900222', '447700900222'),
-    ('07700900222', '447700900222'),
-    (' 73122345678', '73122345678'),
-    ('360623400400', '360623400400'),
-    ('-077-00900222-', '447700900222'),
-    ('(360623(400400)', '360623400400')
-
+    ('+4407900900123', '+447900900123'),
+    ('202-867-5309', '+12028675309'),
+    ('1 202-867-5309', '+12028675309'),
+    ('+1 (202) 867-5309', '+12028675309'),
+    ('(202) 867-5309', '+12028675309'),
+    ('2028675309', '+12028675309')
 ])
 def test_persist_sms_notification_stores_normalised_number(
     sample_job,
diff --git a/tests/app/notifications/test_receive_notification.py b/tests/app/notifications/test_receive_notification.py
index 90ddbd6a5..e90a29896 100644
--- a/tests/app/notifications/test_receive_notification.py
+++ b/tests/app/notifications/test_receive_notification.py
@@ -40,7 +40,7 @@ def test_receive_notification_returns_received_to_sns(client, mocker, sample_ser
     mocked = mocker.patch("app.notifications.receive_notifications.tasks.send_inbound_sms_to_service.apply_async")
     prom_counter_labels_mock = mocker.patch('app.notifications.receive_notifications.INBOUND_SMS_COUNTER.labels')
     data = {
-        "originationNumber": "+15558675309",
+        "originationNumber": "+12028675309",
         "destinationNumber": sample_service_full_permissions.get_inbound_number(),
         "messageKeyword": "JOIN",
         "messageBody": "EXAMPLE",
@@ -241,9 +241,9 @@ def test_receive_notification_error_if_not_single_matching_service(client, notif
 @pytest.mark.parametrize(
     'number, expected',
     [
-        ('15558675309', '5558675309'),
-        ('+15558675309', '5558675309'),
-        ('5558675309', '5558675309'),
+        ('12028675309', '2028675309'),
+        ('+12028675309', '2028675309'),
+        ('2028675309', '2028675309'),
         ('15111111111', '5111111111')
     ]
 )
diff --git a/tests/app/notifications/test_validators.py b/tests/app/notifications/test_validators.py
index babada8fc..f354d184e 100644
--- a/tests/app/notifications/test_validators.py
+++ b/tests/app/notifications/test_validators.py
@@ -167,10 +167,10 @@ def test_service_can_send_to_recipient_passes(key_type, notify_db_session):
 
 
 @pytest.mark.parametrize('user_number, recipient_number', [
-    ['0048601234567', '+486 012 34567'],
-    ['07513332413', '(07513) 332413'],
+    ['+12028675309', '202-867-5309'],
+    ['+447513332413', '+44 (07513) 332413'],
 ])
-def test_service_can_send_to_recipient_passes_with_non_normalised_number(sample_service, user_number, recipient_number):
+def test_service_can_send_to_recipient_passes_with_non_normalized_number(sample_service, user_number, recipient_number):
     sample_service.users[0].mobile_number = user_number
 
     serialised_service = SerialisedService.from_id(sample_service.id)
@@ -181,7 +181,7 @@ def test_service_can_send_to_recipient_passes_with_non_normalised_number(sample_
 @pytest.mark.parametrize('user_email, recipient_email', [
     ['test@example.com', 'TeSt@EXAMPLE.com'],
 ])
-def test_service_can_send_to_recipient_passes_with_non_normalised_email(sample_service, user_email, recipient_email):
+def test_service_can_send_to_recipient_passes_with_non_normalized_email(sample_service, user_email, recipient_email):
     sample_service.users[0].email_address = user_email
 
     serialised_service = SerialisedService.from_id(sample_service.id)
@@ -206,15 +206,15 @@ def test_service_can_send_to_recipient_passes_for_guest_list_recipient_passes(sa
     assert service_can_send_to_recipient("some_other_email@test.com",
                                          'team',
                                          sample_service) is None
-    create_service_guest_list(sample_service, mobile_number='07513332413')
-    assert service_can_send_to_recipient('07513332413',
+    create_service_guest_list(sample_service, mobile_number='2028675309')
+    assert service_can_send_to_recipient('2028675309',
                                          'team',
                                          sample_service) is None
 
 
 @pytest.mark.parametrize('recipient', [
     {"email_address": "some_other_email@test.com"},
-    {"mobile_number": "07513332413"},
+    {"mobile_number": "2028675300"},
 ])
 def test_service_can_send_to_recipient_fails_when_ignoring_guest_list(
     notify_db_session,
@@ -234,7 +234,7 @@ def test_service_can_send_to_recipient_fails_when_ignoring_guest_list(
     assert exec_info.value.fields == []
 
 
-@pytest.mark.parametrize('recipient', ['07513332413', 'some_other_email@test.com'])
+@pytest.mark.parametrize('recipient', ['2028675300', 'some_other_email@test.com'])
 @pytest.mark.parametrize('key_type, error_message',
                          [('team', 'Can’t send to this recipient using a team-only API key'),
                           ('normal',
@@ -482,7 +482,7 @@ def test_validate_and_format_recipient_fails_when_international_number_and_servi
     service = create_service(service_permissions=[SMS_TYPE])
     service_model = SerialisedService.from_id(service.id)
     with pytest.raises(BadRequestError) as e:
-        validate_and_format_recipient('20-12-1234-1234', key_type, service_model, SMS_TYPE)
+        validate_and_format_recipient('+20-12-1234-1234', key_type, service_model, SMS_TYPE)
     assert e.value.status_code == 400
     assert e.value.message == 'Cannot send to international mobile numbers'
     assert e.value.fields == []
@@ -492,8 +492,8 @@ def test_validate_and_format_recipient_fails_when_international_number_and_servi
 def test_validate_and_format_recipient_succeeds_with_international_numbers_if_service_does_allow_int_sms(
         key_type, sample_service_full_permissions):
     service_model = SerialisedService.from_id(sample_service_full_permissions.id)
-    result = validate_and_format_recipient('20-12-1234-1234', key_type, service_model, SMS_TYPE)
-    assert result == '201212341234'
+    result = validate_and_format_recipient('+4407513332413', key_type, service_model, SMS_TYPE)
+    assert result == '+447513332413'
 
 
 def test_validate_and_format_recipient_fails_when_no_recipient():
diff --git a/tests/app/public_contracts/test_POST_notification.py b/tests/app/public_contracts/test_POST_notification.py
index 2396359a5..3a4d2dc10 100644
--- a/tests/app/public_contracts/test_POST_notification.py
+++ b/tests/app/public_contracts/test_POST_notification.py
@@ -24,7 +24,7 @@ def test_post_sms_contract(client, mocker, sample_template):
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     response_json = return_json_from_response(_post_notification(
-        client, sample_template, url='/notifications/sms', to='07700 900 855'
+        client, sample_template, url='/notifications/sms', to='202-867-5309'
     ))
     validate_v0(response_json, 'POST_notification_return_sms.json')
 
diff --git a/tests/app/service/send_notification/test_send_notification.py b/tests/app/service/send_notification/test_send_notification.py
index 2dc6f7ae5..ffeb10fa8 100644
--- a/tests/app/service/send_notification/test_send_notification.py
+++ b/tests/app/service/send_notification/test_send_notification.py
@@ -83,7 +83,8 @@ def test_should_reject_bad_phone_numbers(notify_api, sample_template, mocker):
             mocked.assert_not_called()
             assert json_resp['result'] == 'error'
             assert len(json_resp['message'].keys()) == 1
-            assert 'Invalid phone number: Must not contain letters or symbols' in json_resp['message']['to']
+            assert 'Invalid phone number: The string supplied did not seem to be a phone number.' \
+                in json_resp['message']['to']
             assert response.status_code == 400
 
 
@@ -338,7 +339,7 @@ def test_should_allow_valid_sms_notification(notify_api, sample_template, mocker
             mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
             data = {
-                'to': '07700 900 855',
+                'to': '202 867 5309',
                 'template': str(sample_template.id)
             }
 
@@ -497,7 +498,7 @@ def test_should_not_send_sms_if_team_api_key_and_not_a_service_user(client, samp
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     data = {
-        'to': '07123123123',
+        'to': '2028675300',
         'template': str(sample_template.id),
     }
 
@@ -550,7 +551,7 @@ def test_should_send_sms_to_anyone_with_test_key(
     mocker.patch('app.notifications.process_notifications.uuid.uuid4', return_value=fake_uuid)
 
     data = {
-        'to': '07811111111',
+        'to': '2028675300',
         'template': sample_template.id
     }
     sample_template.service.restricted = restricted
@@ -756,9 +757,9 @@ def test_should_not_persist_notification_or_send_email_if_simulated_email(
 
 
 @pytest.mark.parametrize('to_sms', [
-    '07700 900000',
-    '07700 900111',
-    '07700 900222'
+    '2028675000',
+    '2028675111',
+    '+12028675222'
 ])
 def test_should_not_persist_notification_or_send_sms_if_simulated_number(
         client,
@@ -788,7 +789,7 @@ def test_should_not_persist_notification_or_send_sms_if_simulated_number(
     KEY_TYPE_NORMAL, KEY_TYPE_TEAM
 ])
 @pytest.mark.parametrize('notification_type, to', [
-    (SMS_TYPE, '07827992635'),
+    (SMS_TYPE, '2028675300'),
     (EMAIL_TYPE, 'non_guest_list_recipient@mail.com')]
 )
 def test_should_not_send_notification_to_non_guest_list_recipient_in_trial_mode(
@@ -842,7 +843,7 @@ def test_should_not_send_notification_to_non_guest_list_recipient_in_trial_mode(
     KEY_TYPE_NORMAL, KEY_TYPE_TEAM
 ])
 @pytest.mark.parametrize('notification_type, to', [
-    (SMS_TYPE, '07123123123'),
+    (SMS_TYPE, '2028675300'),
     (EMAIL_TYPE, 'guest_list_recipient@mail.com')]
 )
 def test_should_send_notification_to_guest_list_recipient(
@@ -893,7 +894,7 @@ def test_should_send_notification_to_guest_list_recipient(
 @pytest.mark.parametrize(
     'notification_type, template_type, to', [
         (EMAIL_TYPE, SMS_TYPE, 'notify@digital.cabinet-office.gov.uk'),
-        (SMS_TYPE, EMAIL_TYPE, '+447700900986')
+        (SMS_TYPE, EMAIL_TYPE, '+12028675309')
     ])
 def test_should_error_if_notification_type_does_not_match_template_type(
         client,
@@ -966,7 +967,7 @@ def test_create_template_raises_invalid_request_when_content_too_large(
 
 
 @pytest.mark.parametrize("notification_type, send_to",
-                         [("sms", "07700 900 855"),
+                         [("sms", "2028675309"),
                           ("email", "sample@email.com")])
 def test_send_notification_uses_priority_queue_when_template_is_marked_as_priority(
     client,
@@ -999,7 +1000,7 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori
 
 @pytest.mark.parametrize(
     "notification_type, send_to",
-    [("sms", "07700 900 855"), ("email", "sample@email.com")]
+    [("sms", "2028675309"), ("email", "sample@email.com")]
 )
 def test_returns_a_429_limit_exceeded_if_rate_limit_exceeded(
     client,
@@ -1042,7 +1043,7 @@ def test_should_allow_store_original_number_on_sms_notification(client, sample_t
     mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     data = {
-        'to': '+(44) 7700-900 855',
+        'to': '(202) 867-5309',
         'template': str(sample_template.id)
     }
 
@@ -1061,7 +1062,7 @@ def test_should_allow_store_original_number_on_sms_notification(client, sample_t
     assert notification_id
     notifications = Notification.query.all()
     assert len(notifications) == 1
-    assert '+(44) 7700-900 855' == notifications[0].to
+    assert '(202) 867-5309' == notifications[0].to
 
 
 def test_should_not_allow_sending_to_international_number_without_international_permission(
@@ -1070,7 +1071,7 @@ def test_should_not_allow_sending_to_international_number_without_international_
     mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     data = {
-        'to': '20-12-1234-1234',
+        'to': '+(44) 7700-900 855',
         'template': str(sample_template.id)
     }
 
@@ -1088,29 +1089,6 @@ def test_should_not_allow_sending_to_international_number_without_international_
     assert error_json['message'] == 'Cannot send to international mobile numbers'
 
 
-def test_should_allow_sending_to_crown_dependency_number_without_international_permission(
-    client, mocker, notify_db_session
-):
-    service = create_service()
-
-    mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
-    template = create_template(service)
-
-    data = {
-        'to': '07700-900-123',
-        'template': str(template.id)
-    }
-
-    auth_header = create_service_authorization_header(service_id=service.id)
-
-    response = client.post(
-        path='/notifications/sms',
-        data=json.dumps(data),
-        headers=[('Content-Type', 'application/json'), auth_header])
-
-    assert response.status_code == 201
-
-
 def test_should_allow_sending_to_international_number_with_international_permission(
     client, sample_service_full_permissions, mocker
 ):
@@ -1118,7 +1096,7 @@ def test_should_allow_sending_to_international_number_with_international_permiss
     template = create_template(sample_service_full_permissions)
 
     data = {
-        'to': '20-12-1234-1234',
+        'to': '+(44) 7700-900 855',
         'template': str(template.id)
     }
 
@@ -1140,7 +1118,7 @@ def test_should_not_allow_sms_notifications_if_service_permission_not_set(
     mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     data = {
-        'to': '+447700900986',
+        'to': '+12028675309',
         'template': str(sample_template_without_sms_permission.id)
     }
 
@@ -1201,7 +1179,7 @@ def test_should_throw_exception_if_notification_type_is_invalid(client, sample_s
 
 
 @pytest.mark.parametrize("notification_type, recipient",
-                         [("sms", '07700 900 855'),
+                         [("sms", '2028675309'),
                           ("email", "test@gov.uk")
                           ]
                          )
diff --git a/tests/app/service/send_notification/test_send_one_off_notification.py b/tests/app/service/send_notification/test_send_one_off_notification.py
index 5443f42fb..d0065d0be 100644
--- a/tests/app/service/send_notification/test_send_one_off_notification.py
+++ b/tests/app/service/send_notification/test_send_one_off_notification.py
@@ -50,7 +50,7 @@ def test_send_one_off_notification_calls_celery_correctly(persist_mock, celery_m
 
     post_data = {
         'template_id': str(template.id),
-        'to': '07700 900 001',
+        'to': '202-867-5309',
         'created_by': str(service.created_by_id)
     }
 
@@ -81,7 +81,7 @@ def test_send_one_off_notification_calls_persist_correctly_for_sms(
 
     post_data = {
         'template_id': str(template.id),
-        'to': '07700 900 001',
+        'to': '202-867-5309',
         'personalisation': {'name': 'foo'},
         'created_by': str(service.created_by_id)
     }
@@ -118,14 +118,14 @@ def test_send_one_off_notification_calls_persist_correctly_for_international_sms
 
     post_data = {
         'template_id': str(template.id),
-        'to': '+1 555 0100',
+        'to': '+(44) 7700-900 855',
         'personalisation': {'name': 'foo'},
         'created_by': str(service.created_by_id)
     }
 
     send_one_off_notification(service.id, post_data)
 
-    assert persist_mock.call_args[1]['recipient'] == '+1 555 0100'
+    assert persist_mock.call_args[1]['recipient'] == '+(44) 7700-900 855'
 
 
 def test_send_one_off_notification_calls_persist_correctly_for_email(
@@ -223,7 +223,7 @@ def test_send_one_off_notification_honors_research_mode(notify_db_session, persi
 
     post_data = {
         'template_id': str(template.id),
-        'to': '07700 900 001',
+        'to': '202-867-5309',
         'created_by': str(service.created_by_id)
     }
 
@@ -239,7 +239,7 @@ def test_send_one_off_notification_honors_priority(notify_db_session, persist_mo
 
     post_data = {
         'template_id': str(template.id),
-        'to': '07700 900 001',
+        'to': '202-867-5309',
         'created_by': str(service.created_by_id)
     }
 
@@ -263,9 +263,9 @@ def test_send_one_off_notification_raises_if_invalid_recipient(notify_db_session
 
 
 @pytest.mark.parametrize('recipient', [
-    '07700 900 001',  # not in team or guest_list
-    '07700900123',  # in guest_list
-    '+447700-900-123',  # in guest_list in different format
+    '2028675300',  # not in team or guest_list
+    '2028765309',  # in guest_list
+    '+1-202-876-5309',  # in guest_list in different format
 ])
 def test_send_one_off_notification_raises_if_cant_send_to_recipient(
     notify_db_session,
@@ -274,7 +274,7 @@ def test_send_one_off_notification_raises_if_cant_send_to_recipient(
     service = create_service(restricted=True)
     template = create_template(service=service)
     dao_add_and_commit_guest_list_contacts([
-        ServiceGuestList.from_string(service.id, MOBILE_TYPE, '07700900123'),
+        ServiceGuestList.from_string(service.id, MOBILE_TYPE, '2028765309'),
     ])
 
     post_data = {
@@ -331,7 +331,7 @@ def test_send_one_off_notification_fails_if_created_by_other_service(sample_temp
 
     post_data = {
         'template_id': str(sample_template.id),
-        'to': '07700 900 001',
+        'to': '202-867-5309',
         'created_by': str(user_not_in_service.id)
     }
 
@@ -414,12 +414,12 @@ def test_send_one_off_sms_notification_should_use_sms_sender_reply_to_text(sampl
     template = create_template(service=sample_service, template_type=SMS_TYPE)
     sms_sender = create_service_sms_sender(
         service=sample_service,
-        sms_sender='07123123123',
+        sms_sender='2028675309',
         is_default=False
     )
 
     data = {
-        'to': '07111111111',
+        'to': '2028675000',
         'template_id': str(template.id),
         'created_by': str(sample_service.created_by_id),
         'sender_id': str(sms_sender.id),
@@ -433,7 +433,7 @@ def test_send_one_off_sms_notification_should_use_sms_sender_reply_to_text(sampl
         queue=None
     )
 
-    assert notification.reply_to_text == "447123123123"
+    assert notification.reply_to_text == "+12028675309"
 
 
 def test_send_one_off_sms_notification_should_use_default_service_reply_to_text(sample_service, celery_mock):
@@ -441,12 +441,12 @@ def test_send_one_off_sms_notification_should_use_default_service_reply_to_text(
     sample_service.service_sms_senders[0].is_default = False
     create_service_sms_sender(
         service=sample_service,
-        sms_sender='07123123456',
+        sms_sender='2028675309',
         is_default=True
     )
 
     data = {
-        'to': '07111111111',
+        'to': '2028675000',
         'template_id': str(template.id),
         'created_by': str(sample_service.created_by_id),
     }
@@ -459,7 +459,7 @@ def test_send_one_off_sms_notification_should_use_default_service_reply_to_text(
         queue=None
     )
 
-    assert notification.reply_to_text == "447123123456"
+    assert notification.reply_to_text == "+12028675309"
 
 
 def test_send_one_off_notification_should_throw_exception_if_reply_to_id_doesnot_exist(
@@ -481,7 +481,7 @@ def test_send_one_off_notification_should_throw_exception_if_sms_sender_id_doesn
         sample_template
 ):
     data = {
-        'to': '07700 900 001',
+        'to': '2028675000',
         'template_id': str(sample_template.id),
         'sender_id': str(uuid.uuid4()),
         'created_by': str(sample_template.service.created_by_id)
diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py
index 278446a7b..cbaa26c47 100644
--- a/tests/app/service/test_rest.py
+++ b/tests/app/service/test_rest.py
@@ -204,7 +204,7 @@ def test_get_live_services_data(sample_user, admin_request):
         {
             'consent_to_research': None,
             'contact_email': 'notify@digital.cabinet-office.gov.uk',
-            'contact_mobile': '+447700900986',
+            'contact_mobile': '+12028675309',
             'contact_name': 'Test User',
             'email_totals': 1,
             'email_volume_intent': None,
@@ -222,7 +222,7 @@ def test_get_live_services_data(sample_user, admin_request):
         {
             'consent_to_research': None,
             'contact_email': 'notify@digital.cabinet-office.gov.uk',
-            'contact_mobile': '+447700900986',
+            'contact_mobile': '+12028675309',
             'contact_name': 'Test User',
             'email_totals': 0,
             'email_volume_intent': None,
@@ -2474,7 +2474,7 @@ def test_send_one_off_notification(sample_service, admin_request, mocker):
         service_id=sample_service.id,
         _data={
             'template_id': str(template.id),
-            'to': '07700900001',
+            'to': '2028675309',
             'created_by': str(sample_service.created_by_id)
         },
         _expected_status=201
diff --git a/tests/app/service/test_service_guest_list.py b/tests/app/service/test_service_guest_list.py
index 344777d26..d3e82307f 100644
--- a/tests/app/service/test_service_guest_list.py
+++ b/tests/app/service/test_service_guest_list.py
@@ -22,8 +22,8 @@ def test_get_guest_list_returns_data(client, sample_service_guest_list):
 def test_get_guest_list_separates_emails_and_phones(client, sample_service):
     dao_add_and_commit_guest_list_contacts([
         ServiceGuestList.from_string(sample_service.id, EMAIL_TYPE, 'service@example.com'),
-        ServiceGuestList.from_string(sample_service.id, MOBILE_TYPE, '07123456789'),
-        ServiceGuestList.from_string(sample_service.id, MOBILE_TYPE, '+1800-555-555'),
+        ServiceGuestList.from_string(sample_service.id, MOBILE_TYPE, '2028675309'),
+        ServiceGuestList.from_string(sample_service.id, MOBILE_TYPE, '+1800-555-5555'),
     ])
 
     response = client.get(
@@ -32,7 +32,7 @@ def test_get_guest_list_separates_emails_and_phones(client, sample_service):
     assert response.status_code == 200
     json_resp = json.loads(response.get_data(as_text=True))
     assert json_resp['email_addresses'] == ['service@example.com']
-    assert sorted(json_resp['phone_numbers']) == sorted(['+1800-555-555', '07123456789'])
+    assert sorted(json_resp['phone_numbers']) == sorted(['+1800-555-5555', '2028675309'])
 
 
 def test_get_guest_list_404s_with_unknown_service_id(client):
@@ -57,7 +57,7 @@ def test_get_guest_list_returns_no_data(client, sample_service):
 def test_update_guest_list_replaces_old_guest_list(client, sample_service_guest_list):
     data = {
         'email_addresses': ['foo@bar.com'],
-        'phone_numbers': ['07123456789']
+        'phone_numbers': ['2028765309']
     }
 
     response = client.put(
@@ -69,7 +69,7 @@ def test_update_guest_list_replaces_old_guest_list(client, sample_service_guest_
     assert response.status_code == 204
     guest_list = ServiceGuestList.query.order_by(ServiceGuestList.recipient).all()
     assert len(guest_list) == 2
-    assert guest_list[0].recipient == '07123456789'
+    assert guest_list[0].recipient == '2028765309'
     assert guest_list[1].recipient == 'foo@bar.com'
 
 
@@ -77,7 +77,7 @@ def test_update_guest_list_doesnt_remove_old_guest_list_if_error(client, sample_
 
     data = {
         'email_addresses': [''],
-        'phone_numbers': ['07123456789']
+        'phone_numbers': ['2028675309']
     }
 
     response = client.put(
diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py
index b2b6abd14..6f50f30f7 100644
--- a/tests/app/test_commands.py
+++ b/tests/app/test_commands.py
@@ -19,7 +19,7 @@ def test_create_test_user_command(notify_db_session, notify_api):
     notify_api.test_cli_runner().invoke(
         create_test_user, [
             '--email', 'somebody@fake.gov',
-            '--mobile_number', '555-555-5555',
+            '--mobile_number', '202-555-5555',
             '--password', 'correct horse battery staple',
             '--name', 'Fake Personson',
             # '--auth_type', 'sms_auth',  # this is the default
diff --git a/tests/app/test_model.py b/tests/app/test_model.py
index ae38ff48a..8e575504e 100644
--- a/tests/app/test_model.py
+++ b/tests/app/test_model.py
@@ -32,8 +32,8 @@ from tests.app.db import (
 
 
 @pytest.mark.parametrize('mobile_number', [
-    '5558675309',
-    '+15558675309'
+    '2348675309',
+    '+12348675309'
 ])
 def test_should_build_service_guest_list_from_mobile_number(mobile_number):
     service_guest_list = ServiceGuestList.from_string('service_id', MOBILE_TYPE, mobile_number)
@@ -93,7 +93,7 @@ def test_status_conversion(initial_statuses, expected_statuses):
 
 @freeze_time("2016-01-01 11:09:00.000000")
 @pytest.mark.parametrize('template_type, recipient', [
-    ('sms', '+15558675309'),
+    ('sms', '+12028675309'),
     ('email', 'foo@bar.com'),
 ])
 def test_notification_for_csv_returns_correct_type(sample_service, template_type, recipient):
diff --git a/tests/app/user/test_rest.py b/tests/app/user/test_rest.py
index 1031685cf..f06a26275 100644
--- a/tests/app/user/test_rest.py
+++ b/tests/app/user/test_rest.py
@@ -107,7 +107,7 @@ def test_post_user(admin_request, notify_db_session):
         "name": "Test User",
         "email_address": "user@digital.cabinet-office.gov.uk",
         "password": "password",
-        "mobile_number": "+447700900986",
+        "mobile_number": "+12028675309",
         "logged_in_at": None,
         "state": "active",
         "failed_login_count": 0,
@@ -130,7 +130,7 @@ def test_post_user_without_auth_type(admin_request, notify_db_session):
         "name": "Test User",
         "email_address": "user@digital.cabinet-office.gov.uk",
         "password": "password",
-        "mobile_number": "+447700900986",
+        "mobile_number": "+12028675309",
         "permissions": {},
     }
 
@@ -149,7 +149,7 @@ def test_post_user_missing_attribute_email(admin_request, notify_db_session):
     data = {
         "name": "Test User",
         "password": "password",
-        "mobile_number": "+447700900986",
+        "mobile_number": "+12028675309",
         "logged_in_at": None,
         "state": "active",
         "failed_login_count": 0,
@@ -169,7 +169,7 @@ def test_create_user_missing_attribute_password(admin_request, notify_db_session
     data = {
         "name": "Test User",
         "email_address": "user@digital.cabinet-office.gov.uk",
-        "mobile_number": "+447700900986",
+        "mobile_number": "+12028675309",
         "logged_in_at": None,
         "state": "active",
         "failed_login_count": 0,
@@ -224,7 +224,7 @@ def test_cannot_create_user_with_empty_strings(admin_request, notify_db_session)
     )
     assert resp['message'] == {
         'email_address': ['Not a valid email address'],
-        'mobile_number': ['Invalid phone number: Not enough digits'],
+        'mobile_number': ['Invalid phone number: The string supplied did not seem to be a phone number.'],
         'name': ['Invalid name']
     }
 
@@ -883,7 +883,8 @@ def test_cannot_update_user_with_mobile_number_as_empty_string(admin_request, sa
         _data={'mobile_number': ''},
         _expected_status=400
     )
-    assert resp['message']['mobile_number'] == ['Invalid phone number: Not enough digits']
+    assert resp['message']['mobile_number'] == [
+        'Invalid phone number: The string supplied did not seem to be a phone number.']
 
 
 def test_cannot_update_user_password_using_attributes_method(admin_request, sample_user):
@@ -1100,10 +1101,9 @@ def test_search_for_users_by_email_handles_incorrect_data_format(notify_db_sessi
 
 @pytest.mark.parametrize('number, expected_reply_to',
                          [
-                          ("1-403-123-4567", "notify_international_sender"),
-                          ("27 123 4569 2312", "notify_international_sender"),
-                          ("30 123 4567 7890", "Notify"),
-                          ("+20 123 4567 7890", "Notify"),
+                          ("403-123-4567", "Notify"),
+                          ("+30 123 4567 7890", "Notify"),
+                          ("+27 123 4569 2312", "notify_international_sender"),
                           ])
 def test_get_sms_reply_to_for_notify_service(team_member_mobile_edit_template, number, expected_reply_to):
     # need to import locally to avoid db session errors,
diff --git a/tests/app/user/test_rest_verify.py b/tests/app/user/test_rest_verify.py
index 8bf0cd9e4..65217f0b2 100644
--- a/tests/app/user/test_rest_verify.py
+++ b/tests/app/user/test_rest_verify.py
@@ -522,7 +522,7 @@ def test_user_verify_email_code_fails_if_code_already_used(admin_request, sample
 def test_send_user_2fa_code_sends_from_number_for_international_numbers(
         client, sample_user, mocker, sms_code_template
 ):
-    sample_user.mobile_number = "601117224412"
+    sample_user.mobile_number = "+601117224412"
     auth_header = create_admin_authorization_header()
     mocker.patch('app.user.rest.create_secret_code', return_value='11111')
     mocker.patch('app.user.rest.send_notification_to_queue')
diff --git a/tests/app/v2/notifications/test_notification_schemas.py b/tests/app/v2/notifications/test_notification_schemas.py
index c41e9c12b..33a5b2030 100644
--- a/tests/app/v2/notifications/test_notification_schemas.py
+++ b/tests/app/v2/notifications/test_notification_schemas.py
@@ -102,11 +102,11 @@ def test_get_notifications_request_invalid_statuses_and_template_types():
             .format(invalid_template_type) in error_messages
 
 
-valid_json = {"phone_number": "07515111111",
+valid_json = {"phone_number": "2028675309",
               "template_id": str(uuid.uuid4())
               }
 valid_json_with_optionals = {
-    "phone_number": "07515111111",
+    "phone_number": "2028675309",
     "template_id": str(uuid.uuid4()),
     "reference": "reference from caller",
     "personalisation": {"key": "value"}
@@ -130,7 +130,7 @@ def test_post_sms_schema_is_valid(input):
 def test_post_sms_json_schema_bad_uuid(template_id):
     j = {
         "template_id": template_id,
-        "phone_number": "07515111111"
+        "phone_number": "2028675309"
     }
     with pytest.raises(ValidationError) as e:
         validate(j, post_sms_request_schema)
@@ -158,7 +158,7 @@ def test_post_sms_json_schema_bad_uuid_and_missing_phone_number():
 
 def test_post_sms_schema_with_personalisation_that_is_not_a_dict():
     j = {
-        "phone_number": "07515111111",
+        "phone_number": "2028675309",
         "template_id": str(uuid.uuid4()),
         "reference": "reference from caller",
         "personalisation": "not_a_dict"
@@ -174,9 +174,9 @@ def test_post_sms_schema_with_personalisation_that_is_not_a_dict():
 
 
 @pytest.mark.parametrize('invalid_phone_number, err_msg', [
-    ('08515111111', 'phone_number Not a UK mobile number'),
-    ('07515111*11', 'phone_number Must not contain letters or symbols'),
-    ('notaphoneumber', 'phone_number Must not contain letters or symbols'),
+    ('08515111111', 'phone_number Phone number is not possible'),
+    ('07515111*11', 'phone_number Not enough digits'),
+    ('notaphoneumber', 'phone_number The string supplied did not seem to be a phone number.'),
     (7700900001, 'phone_number 7700900001 is not of type string'),
     (None, 'phone_number None is not of type string'),
     ([], 'phone_number [] is not of type string'),
@@ -194,13 +194,13 @@ def test_post_sms_request_schema_invalid_phone_number(invalid_phone_number, err_
 
 
 def test_post_sms_request_schema_invalid_phone_number_and_missing_template():
-    j = {"phone_number": '08515111111',
+    j = {"phone_number": '5558675309',
          }
     with pytest.raises(ValidationError) as e:
         validate(j, post_sms_request_schema)
     errors = json.loads(str(e.value)).get('errors')
     assert len(errors) == 2
-    assert {"error": "ValidationError", "message": "phone_number Not a UK mobile number"} in errors
+    assert {"error": "ValidationError", "message": "phone_number Phone number range is not in use"} in errors
     assert {"error": "ValidationError", "message": "template_id is a required property"} in errors
 
 
@@ -269,7 +269,7 @@ def test_post_schema_valid_scheduled_for(schema):
     if schema == post_email_request_schema:
         j.update({"email_address": "joe@gmail.com"})
     else:
-        j.update({"phone_number": "07515111111"})
+        j.update({"phone_number": "2028675309"})
     assert validate(j, schema) == j
 
 
@@ -286,7 +286,7 @@ def test_post_email_schema_invalid_scheduled_for(invalid_datetime, schema):
     if schema == post_email_request_schema:
         j.update({"email_address": "joe@gmail.com"})
     else:
-        j.update({"phone_number": "07515111111"})
+        j.update({"phone_number": "2028675309"})
     with pytest.raises(ValidationError) as e:
         validate(j, schema)
     error = json.loads(str(e.value))
@@ -299,7 +299,7 @@ def test_post_email_schema_invalid_scheduled_for(invalid_datetime, schema):
 
 @freeze_time("2017-05-12 13:00:00")
 def test_scheduled_for_raises_validation_error_when_in_the_past():
-    j = {"phone_number": "07515111111",
+    j = {"phone_number": "2028675309",
          "template_id": str(uuid.uuid4()),
          "scheduled_for": "2017-05-12 10:00"}
     with pytest.raises(ValidationError) as e:
@@ -312,7 +312,7 @@ def test_scheduled_for_raises_validation_error_when_in_the_past():
 
 @freeze_time("2017-05-12 13:00:00")
 def test_scheduled_for_raises_validation_error_when_more_than_24_hours_in_the_future():
-    j = {"phone_number": "07515111111",
+    j = {"phone_number": "2028675309",
          "template_id": str(uuid.uuid4()),
          "scheduled_for": "2017-05-13 14:00"}
     with pytest.raises(ValidationError) as e:
diff --git a/tests/app/v2/notifications/test_post_notifications.py b/tests/app/v2/notifications/test_post_notifications.py
index e5b2ab073..50e098b89 100644
--- a/tests/app/v2/notifications/test_post_notifications.py
+++ b/tests/app/v2/notifications/test_post_notifications.py
@@ -102,7 +102,7 @@ def test_post_sms_notification_uses_inbound_number_as_sender(client, notify_db_s
 
 
 def test_post_sms_notification_uses_inbound_number_reply_to_as_sender(client, notify_db_session, mocker):
-    service = create_service_with_inbound_number(inbound_number='07123123123')
+    service = create_service_with_inbound_number(inbound_number='2028675309')
 
     template = create_template(service=service, content="Hello (( Name))\nYour thing is due soon")
     mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
@@ -124,8 +124,8 @@ def test_post_sms_notification_uses_inbound_number_reply_to_as_sender(client, no
     assert len(notifications) == 1
     notification_id = notifications[0].id
     assert resp_json['id'] == str(notification_id)
-    assert resp_json['content']['from_number'] == '447123123123'
-    assert notifications[0].reply_to_text == '447123123123'
+    assert resp_json['content']['from_number'] == '+12028675309'
+    assert notifications[0].reply_to_text == '+12028675309'
     mocked.assert_called_once_with([str(notification_id)], queue='send-sms-tasks')
 
 
@@ -159,7 +159,7 @@ def test_post_sms_notification_returns_201_with_sms_sender_id(
 def test_post_sms_notification_uses_sms_sender_id_reply_to(
         client, sample_template_with_placeholders, mocker
 ):
-    sms_sender = create_service_sms_sender(service=sample_template_with_placeholders.service, sms_sender='07123123123')
+    sms_sender = create_service_sms_sender(service=sample_template_with_placeholders.service, sms_sender='2028675309')
     mocked = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
     data = {
         'phone_number': '+447700900855',
@@ -176,10 +176,10 @@ def test_post_sms_notification_uses_sms_sender_id_reply_to(
     assert response.status_code == 201
     resp_json = json.loads(response.get_data(as_text=True))
     assert validate(resp_json, post_sms_response) == resp_json
-    assert resp_json['content']['from_number'] == '447123123123'
+    assert resp_json['content']['from_number'] == '+12028675309'
     notifications = Notification.query.all()
     assert len(notifications) == 1
-    assert notifications[0].reply_to_text == '447123123123'
+    assert notifications[0].reply_to_text == '+12028675309'
     mocked.assert_called_once_with([resp_json['id']], queue='send-sms-tasks')
 
 
@@ -189,7 +189,7 @@ def test_notification_reply_to_text_is_original_value_if_sender_is_changed_after
     sms_sender = create_service_sms_sender(service=sample_template.service, sms_sender='123456', is_default=False)
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
     data = {
-        'phone_number': '+447700900855',
+        'phone_number': '+12028675309',
         'template_id': str(sample_template.id),
         'sms_sender_id': str(sms_sender.id)
     }
@@ -220,7 +220,7 @@ def test_should_cache_template_lookups_in_memory(mocker, client, sample_template
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     data = {
-        'phone_number': '+447700900855',
+        'phone_number': '2028675309',
         'template_id': str(sample_template.id),
     }
 
@@ -455,9 +455,9 @@ def test_post_email_notification_returns_201(client, sample_email_template_with_
     ('simulate-delivered@notifications.service.gov.uk', EMAIL_TYPE),
     ('simulate-delivered-2@notifications.service.gov.uk', EMAIL_TYPE),
     ('simulate-delivered-3@notifications.service.gov.uk', EMAIL_TYPE),
-    ('07700 900000', 'sms'),
-    ('07700 900111', 'sms'),
-    ('07700 900222', 'sms')
+    ('2028675000', 'sms'),
+    ('2028675111', 'sms'),
+    ('2028675222', 'sms')
 ])
 def test_should_not_persist_or_send_notification_if_simulated_recipient(
         client,
@@ -493,7 +493,7 @@ def test_should_not_persist_or_send_notification_if_simulated_recipient(
 
 
 @pytest.mark.parametrize("notification_type, key_send_to, send_to",
-                         [("sms", "phone_number", "07700 900 855"),
+                         [("sms", "phone_number", "2028675309"),
                           ("email", "email_address", "sample@email.com")])
 def test_send_notification_uses_priority_queue_when_template_is_marked_as_priority(
     client,
@@ -532,7 +532,7 @@ def test_send_notification_uses_priority_queue_when_template_is_marked_as_priori
 
 @pytest.mark.parametrize(
     "notification_type, key_send_to, send_to",
-    [("sms", "phone_number", "07700 900 855"), ("email", "email_address", "sample@email.com")]
+    [("sms", "phone_number", "2028675309"), ("email", "email_address", "sample@email.com")]
 )
 def test_returns_a_429_limit_exceeded_if_rate_limit_exceeded(
         client,
@@ -581,7 +581,7 @@ def test_post_sms_notification_returns_400_if_not_allowed_to_send_int_sms(
     template = create_template(service=service)
 
     data = {
-        'phone_number': '20-12-1234-1234',
+        'phone_number': '+20-12-1234-1234',
         'template_id': template.id
     }
     auth_header = create_service_authorization_header(service_id=service.id)
@@ -627,7 +627,7 @@ def test_post_sms_notification_with_archived_reply_to_id_returns_400(client, sam
 
 
 @pytest.mark.parametrize('recipient,label,permission_type, notification_type,expected_error', [
-    ('07700 900000', 'phone_number', 'email', 'sms', 'text messages'),
+    ('2028675309', 'phone_number', 'email', 'sms', 'text messages'),
     ('someone@test.com', 'email_address', 'sms', 'email', 'emails')])
 def test_post_sms_notification_returns_400_if_not_allowed_to_send_notification(
         notify_db_session, client, recipient, label, permission_type, notification_type, expected_error
@@ -691,7 +691,7 @@ def test_post_sms_notification_returns_201_if_allowed_to_send_int_sms(
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     data = {
-        'phone_number': '20-12-1234-1234',
+        'phone_number': '+20-12-1234-1234',
         'template_id': sample_template.id
     }
     auth_header = create_service_authorization_header(service_id=sample_service.id)

From 8a0535fa033a83eaeb93d100a784c696e21485e9 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Fri, 6 Jan 2023 10:02:23 -0500
Subject: [PATCH 29/46] Save normalized phone format to DB

---
 Pipfile                                           |  2 +-
 Pipfile.lock                                      | 10 +++++-----
 app/models.py                                     | 15 +++++++++++----
 docs/data-storage.md                              |  1 +
 tests/app/celery/test_tasks.py                    |  8 ++++----
 tests/app/dao/test_services_dao.py                |  2 +-
 tests/app/dao/test_users_dao.py                   | 10 +++++-----
 .../send_notification/test_send_notification.py   |  9 +++++----
 tests/app/service/test_service_guest_list.py      |  4 ++--
 tests/app/test_model.py                           |  2 +-
 10 files changed, 36 insertions(+), 27 deletions(-)

diff --git a/Pipfile b/Pipfile
index 75a534f21..1b9ef1a27 100644
--- a/Pipfile
+++ b/Pipfile
@@ -59,7 +59,7 @@ werkzeug = "~=2.1.1"
 prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
 packaging = "==21.3"
-notifications-utils = {editable = true, ref = "validate-us-numbers", git = "https://github.com/GSA/notifications-utils.git"}
+notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils.git"}
 
 [dev-packages]
 flake8 = "==4.0.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index 948641d49..e392dd9ea 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "2f43f1aefdea61dea472c0f41aa81166c852769a8156d3e9644e5535c30a1c2a"
+            "sha256": "d0a90dd082bf51ae47e04412c0848f00bc5e4b538804d1333e9264ca0ad2d4e9"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -739,7 +739,7 @@
         "notifications-utils": {
             "editable": true,
             "git": "https://github.com/GSA/notifications-utils.git",
-            "ref": "d88dff885f5db43f6573ccb5878c376dea92ec2e"
+            "ref": "d88d48026171991de4d8398e5f9f8c2063297885"
         },
         "numpy": {
             "hashes": [
@@ -2080,11 +2080,11 @@
         },
         "rich": {
             "hashes": [
-                "sha256:12b1d77ee7edf251b741531323f0d990f5f570a4e7c054d0bfb59fb7981ad977",
-                "sha256:3aa9eba7219b8c575c6494446a59f702552efe1aa261e7eeb95548fa586e1950"
+                "sha256:25f83363f636995627a99f6e4abc52ed0970ebbd544960cc63cbb43aaac3d6f0",
+                "sha256:41fe1d05f433b0f4724cda8345219213d2bfa472ef56b2f64f415b5b94d51b04"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==13.0.0"
+            "version": "==13.0.1"
         },
         "s3transfer": {
             "hashes": [
diff --git a/app/models.py b/app/models.py
index c0399430a..d104c7f8b 100644
--- a/app/models.py
+++ b/app/models.py
@@ -36,6 +36,7 @@ from sqlalchemy.dialects.postgresql import JSON, JSONB, UUID
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy.orm import validates
 from sqlalchemy.orm.collections import attribute_mapped_collection
 
 from app import db, encryption
@@ -140,6 +141,14 @@ class User(db.Model):
         secondary='user_to_organisation',
         backref='users')
 
+    @validates("mobile_number")
+    def validate_mobile_number(self, key, number):
+        try:
+            if number is not None:
+                return validate_phone_number(number, international=True)
+        except InvalidPhoneError as err:
+            raise ValueError(str(err)) from err
+
     @property
     def password(self):
         raise AttributeError("Password not readable")
@@ -708,11 +717,9 @@ class ServiceGuestList(db.Model):
 
         try:
             if recipient_type == MOBILE_TYPE:
-                validate_phone_number(recipient, international=True)
-                instance.recipient = recipient
+                instance.recipient = validate_phone_number(recipient, international=True)
             elif recipient_type == EMAIL_TYPE:
-                validate_email_address(recipient)
-                instance.recipient = recipient
+                instance.recipient = validate_email_address(recipient)
             else:
                 raise ValueError('Invalid recipient type')
         except InvalidPhoneError:
diff --git a/docs/data-storage.md b/docs/data-storage.md
index 2a872c9a6..1ab7cf3ed 100644
--- a/docs/data-storage.md
+++ b/docs/data-storage.md
@@ -28,6 +28,7 @@ No db data is PII, but each job has a csv file in s3 containing phone numbers an
 #### notifications
 
 * to
+* normalized_to
 * _personalization<sup>2</sup>
 * phone_prefix<sup>3</sup>
 
diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py
index 3780f3ad5..ee8d7ccf7 100644
--- a/tests/app/celery/test_tasks.py
+++ b/tests/app/celery/test_tasks.py
@@ -572,11 +572,11 @@ def test_save_sms_should_save_default_sms_sender_notification_reply_to_text_on(n
 
 
 def test_should_not_save_sms_if_restricted_service_and_invalid_number(notify_db_session, mocker):
-    user = create_user(mobile_number="07700 900205")
+    user = create_user(mobile_number="2028675309")
     service = create_service(user=user, restricted=True)
     template = create_template(service=service)
 
-    notification = _notification_json(template, "07700 900849")
+    notification = _notification_json(template, "2028675400")
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     notification_id = uuid.uuid4()
@@ -667,14 +667,14 @@ def test_should_save_sms_template_to_and_persist_with_job_id(sample_job, mocker)
 
 def test_should_not_save_sms_if_team_key_and_recipient_not_in_team(notify_db_session, mocker):
     assert Notification.query.count() == 0
-    user = create_user(mobile_number="07700 900205")
+    user = create_user(mobile_number="2028675309")
     service = create_service(user=user, restricted=True)
     template = create_template(service=service)
 
     team_members = [user.mobile_number for user in service.users]
     assert "07890 300000" not in team_members
 
-    notification = _notification_json(template, "07700 900849")
+    notification = _notification_json(template, "2028675400")
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     notification_id = uuid.uuid4()
diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py
index 404f751e2..afc5dfc38 100644
--- a/tests/app/dao/test_services_dao.py
+++ b/tests/app/dao/test_services_dao.py
@@ -748,7 +748,7 @@ def test_add_existing_user_to_another_service_doesnot_change_old_permissions(not
         name='Other Test User',
         email_address='other_user@digital.cabinet-office.gov.uk',
         password='password',
-        mobile_number='+447700900987'
+        mobile_number='+12028672000'
     )
     save_model_user(other_user, validated_email_access=True)
     service_two = Service(name="service_two",
diff --git a/tests/app/dao/test_users_dao.py b/tests/app/dao/test_users_dao.py
index 46720fd72..2e953a4e7 100644
--- a/tests/app/dao/test_users_dao.py
+++ b/tests/app/dao/test_users_dao.py
@@ -37,11 +37,11 @@ from tests.app.db import (
 
 
 @freeze_time('2020-01-28T12:00:00')
-@pytest.mark.parametrize('phone_number', [
-    '+12028675309',
-    '+1-800-555-5555',
+@pytest.mark.parametrize('phone_number, expected_phone_number', [
+    ('2028675309', '+12028675309'),
+    ('+1-800-555-5555', '+18005555555'),
 ])
-def test_create_user(notify_db_session, phone_number):
+def test_create_user(notify_db_session, phone_number, expected_phone_number):
     email = 'notify@digital.cabinet-office.gov.uk'
     data = {
         'name': 'Test User',
@@ -55,7 +55,7 @@ def test_create_user(notify_db_session, phone_number):
     user_query = User.query.first()
     assert user_query.email_address == email
     assert user_query.id == user.id
-    assert user_query.mobile_number == phone_number
+    assert user_query.mobile_number == expected_phone_number
     assert user_query.email_access_validated_at == datetime.utcnow()
     assert not user_query.platform_admin
 
diff --git a/tests/app/service/send_notification/test_send_notification.py b/tests/app/service/send_notification/test_send_notification.py
index ffeb10fa8..86e4c6038 100644
--- a/tests/app/service/send_notification/test_send_notification.py
+++ b/tests/app/service/send_notification/test_send_notification.py
@@ -842,15 +842,16 @@ def test_should_not_send_notification_to_non_guest_list_recipient_in_trial_mode(
 @pytest.mark.parametrize('key_type', [
     KEY_TYPE_NORMAL, KEY_TYPE_TEAM
 ])
-@pytest.mark.parametrize('notification_type, to', [
-    (SMS_TYPE, '2028675300'),
-    (EMAIL_TYPE, 'guest_list_recipient@mail.com')]
+@pytest.mark.parametrize('notification_type, to, normalized_to', [
+    (SMS_TYPE, '2028675300', '+12028675300'),
+    (EMAIL_TYPE, 'guest_list_recipient@mail.com', None)]
 )
 def test_should_send_notification_to_guest_list_recipient(
     client,
     sample_service,
     notification_type,
     to,
+    normalized_to,
     key_type,
     service_restricted,
     mocker
@@ -866,7 +867,7 @@ def test_should_send_notification_to_guest_list_recipient(
         service_guest_list = create_service_guest_list(sample_service, email_address=to)
 
     assert service_guest_list.service_id == sample_service.id
-    assert to in [member.recipient for member in sample_service.guest_list]
+    assert (normalized_to or to) in [member.recipient for member in sample_service.guest_list]
 
     create_notification(template=template)
 
diff --git a/tests/app/service/test_service_guest_list.py b/tests/app/service/test_service_guest_list.py
index d3e82307f..a7946b6ba 100644
--- a/tests/app/service/test_service_guest_list.py
+++ b/tests/app/service/test_service_guest_list.py
@@ -32,7 +32,7 @@ def test_get_guest_list_separates_emails_and_phones(client, sample_service):
     assert response.status_code == 200
     json_resp = json.loads(response.get_data(as_text=True))
     assert json_resp['email_addresses'] == ['service@example.com']
-    assert sorted(json_resp['phone_numbers']) == sorted(['+1800-555-5555', '2028675309'])
+    assert sorted(json_resp['phone_numbers']) == sorted(['+18005555555', '+12028675309'])
 
 
 def test_get_guest_list_404s_with_unknown_service_id(client):
@@ -69,7 +69,7 @@ def test_update_guest_list_replaces_old_guest_list(client, sample_service_guest_
     assert response.status_code == 204
     guest_list = ServiceGuestList.query.order_by(ServiceGuestList.recipient).all()
     assert len(guest_list) == 2
-    assert guest_list[0].recipient == '2028765309'
+    assert guest_list[0].recipient == '+12028765309'
     assert guest_list[1].recipient == 'foo@bar.com'
 
 
diff --git a/tests/app/test_model.py b/tests/app/test_model.py
index 8e575504e..8e42900d0 100644
--- a/tests/app/test_model.py
+++ b/tests/app/test_model.py
@@ -32,7 +32,7 @@ from tests.app.db import (
 
 
 @pytest.mark.parametrize('mobile_number', [
-    '2348675309',
+    '+447700900855',
     '+12348675309'
 ])
 def test_should_build_service_guest_list_from_mobile_number(mobile_number):

From 828671ecfc4e15fb4408c0bf0d68348abcad1a2b Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Fri, 6 Jan 2023 11:14:45 -0500
Subject: [PATCH 30/46] Remove obsolete strip_leading_plus_one method

---
 app/notifications/receive_notifications.py         |  8 --------
 .../app/notifications/test_receive_notification.py | 14 --------------
 2 files changed, 22 deletions(-)

diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py
index 9343aa1d5..ae0997217 100644
--- a/app/notifications/receive_notifications.py
+++ b/app/notifications/receive_notifications.py
@@ -134,11 +134,3 @@ def fetch_potential_service(inbound_number, provider_name):
 def has_inbound_sms_permissions(permissions):
     str_permissions = [p.permission for p in permissions]
     return set([INBOUND_SMS_TYPE, SMS_TYPE]).issubset(set(str_permissions))
-
-
-def strip_leading_plus_one(number):
-    if number.startswith('+'):
-        number = number.replace('+', '', 1)
-    if number.startswith('1'):
-        number = number.replace('1', '', 1)
-    return number
diff --git a/tests/app/notifications/test_receive_notification.py b/tests/app/notifications/test_receive_notification.py
index e90a29896..89577b3ad 100644
--- a/tests/app/notifications/test_receive_notification.py
+++ b/tests/app/notifications/test_receive_notification.py
@@ -8,7 +8,6 @@ from app.models import EMAIL_TYPE, INBOUND_SMS_TYPE, SMS_TYPE, InboundSms
 from app.notifications.receive_notifications import (
     create_inbound_sms_object,
     has_inbound_sms_permissions,
-    strip_leading_plus_one,
     unescape_string,
 )
 from tests.app.db import (
@@ -238,19 +237,6 @@ def test_receive_notification_error_if_not_single_matching_service(client, notif
     assert InboundSms.query.count() == 0
 
 
-@pytest.mark.parametrize(
-    'number, expected',
-    [
-        ('12028675309', '2028675309'),
-        ('+12028675309', '2028675309'),
-        ('2028675309', '2028675309'),
-        ('15111111111', '5111111111')
-    ]
-)
-def test_strip_leading_country_code(number, expected):
-    assert strip_leading_plus_one(number) == expected
-
-
 @pytest.mark.skip(reason="Need to implement inbound SNS tests. Body here from MMG")
 @pytest.mark.parametrize("auth, keys, status_code", [
     ["testkey", ["testkey"], 200],

From b88de4210149190407f49150f82b601a6cba6d70 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Fri, 6 Jan 2023 16:20:03 -0500
Subject: [PATCH 31/46] Add a system description entry to docs

---
 docs/system-description.md | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 docs/system-description.md

diff --git a/docs/system-description.md b/docs/system-description.md
new file mode 100644
index 000000000..e913f7ca6
--- /dev/null
+++ b/docs/system-description.md
@@ -0,0 +1,38 @@
+US Notify
+=========
+
+System Description
+------------------
+
+US Notify is a service being developed by the TTS Benefits Studio to increase the availability of
+SMS and email notifications to Federal, State, and Local Benefits agencies.
+
+Agencies that sign up will be able to create and use personalized message templates for sending
+notifications to members of the public regarding their benefits. These could include reminders
+about upcoming enrollment deadlines and tasks, or information about upcoming appointments, events,
+or services.
+
+The templates are sent by the agency using one of two methods:
+
+* using the US Notify API to send a message to a given recipient with given personalization values
+* using the US Notify website to upload a CSV file of recipients and their personalization values, one row per message
+
+### Environment
+
+US Notify is comprised of two applications both running on cloud.gov:
+
+* Admin, a Flask website running on the python_buildpack which hosts agency user-facing UI
+* API, a Flask application running on the python_buildpack hosting the US Notify API
+
+US Notify utilizes several cloud.gov-provided services:
+
+* S3 buckets for temporary file storage
+* Elasticache (redis) for cacheing data and enqueueing background tasks
+* RDS (PostgreSQL) for system data storage
+
+US Notify also provides access to two AWS services via a supplemental service broker:
+
+* SNS for sending SMS messages
+* SES for sending email messages
+
+For further details of the system and how it connects to supporting services, see the [application boundary diagram](https://github.com/GSA/us-notify-compliance/blob/main/diagrams/rendered/apps/application.boundary.png)

From 7c2e17bf1f3f6852709e36f905ec27eb3411a196 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Mon, 9 Jan 2023 09:34:47 -0500
Subject: [PATCH 32/46] Clarify that we are not providing direct access to
 SES/SNS

---
 docs/system-description.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/system-description.md b/docs/system-description.md
index e913f7ca6..95eca3898 100644
--- a/docs/system-description.md
+++ b/docs/system-description.md
@@ -30,9 +30,9 @@ US Notify utilizes several cloud.gov-provided services:
 * Elasticache (redis) for cacheing data and enqueueing background tasks
 * RDS (PostgreSQL) for system data storage
 
-US Notify also provides access to two AWS services via a supplemental service broker:
+US Notify also provisions and uses two AWS services via a [supplemental service broker](https://github.com/GSA/datagov-ssb):
 
-* SNS for sending SMS messages
-* SES for sending email messages
+* [SNS](https://aws.amazon.com/sns/) for sending SMS messages
+* [SES](https://aws.amazon.com/ses/) for sending email messages
 
 For further details of the system and how it connects to supporting services, see the [application boundary diagram](https://github.com/GSA/us-notify-compliance/blob/main/diagrams/rendered/apps/application.boundary.png)

From a218b58151788874ab7cedcf042e54032da96afb Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Mon, 9 Jan 2023 12:15:07 -0500
Subject: [PATCH 33/46] Update docs/system-description.md

Co-authored-by: Steven Reilly <stvnrlly@users.noreply.github.com>
---
 docs/system-description.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/system-description.md b/docs/system-description.md
index 95eca3898..b4e8f910c 100644
--- a/docs/system-description.md
+++ b/docs/system-description.md
@@ -4,7 +4,7 @@ US Notify
 System Description
 ------------------
 
-US Notify is a service being developed by the TTS Benefits Studio to increase the availability of
+US Notify is a service being developed by the TTS Public Benefits Studio to increase the availability of
 SMS and email notifications to Federal, State, and Local Benefits agencies.
 
 Agencies that sign up will be able to create and use personalized message templates for sending

From bfeb65c9149387787c23c76e55c1847dfbbf4544 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 17 Jan 2023 14:08:17 -0500
Subject: [PATCH 34/46] Install newrelic and enable in development

---
 Makefile                                   |   7 +-
 Pipfile                                    |   1 +
 Pipfile.lock                               | 225 ++++++++++++---------
 app/notifications/process_notifications.py |  10 +-
 newrelic.ini                               | 219 ++++++++++++++++++++
 sample.env                                 |   5 +
 6 files changed, 358 insertions(+), 109 deletions(-)
 create mode 100644 newrelic.ini

diff --git a/Makefile b/Makefile
index 9b22547bc..d041013e0 100644
--- a/Makefile
+++ b/Makefile
@@ -26,12 +26,12 @@ run-procfile:
 
 .PHONY: run-flask
 run-flask: ## Run flask
-	pipenv run flask run -p 6011 --host=0.0.0.0
+	pipenv run newrelic-admin run-program flask run -p 6011 --host=0.0.0.0
 
 .PHONY: run-celery
 run-celery: ## Run celery, TODO remove purge for staging/prod
 	pipenv run celery -A run_celery.notify_celery purge -f
-	pipenv run celery \
+	pipenv run newrelic-admin run-program celery \
 		-A run_celery.notify_celery worker \
 		--pidfile="/tmp/celery.pid" \
 		--loglevel=INFO \
@@ -39,7 +39,7 @@ run-celery: ## Run celery, TODO remove purge for staging/prod
 
 .PHONY: run-celery-beat
 run-celery-beat: ## Run celery beat
-	pipenv run celery \
+	pipenv run newrelic-admin run-program celery \
 	-A run_celery.notify_celery beat \
 	--loglevel=INFO
 
@@ -52,6 +52,7 @@ generate-version-file: ## Generates the app version file
 	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
 
 .PHONY: test
+test: export NEW_RELIC_ENVIRONMENT=test
 test: ## Run tests
 	pipenv run flake8 .
 	pipenv run isort --check-only ./app ./tests
diff --git a/Pipfile b/Pipfile
index 1b9ef1a27..eee14c122 100644
--- a/Pipfile
+++ b/Pipfile
@@ -60,6 +60,7 @@ prometheus-client = "==0.14.1"
 gds-metrics = {version = "==0.2.4", ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"}
 packaging = "==21.3"
 notifications-utils = {editable = true, ref = "main", git = "https://github.com/GSA/notifications-utils.git"}
+newrelic = "*"
 
 [dev-packages]
 flake8 = "==4.0.1"
diff --git a/Pipfile.lock b/Pipfile.lock
index e392dd9ea..004a17eb8 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "d0a90dd082bf51ae47e04412c0848f00bc5e4b538804d1333e9264ca0ad2d4e9"
+            "sha256": "ad2dc506d24393f641531ca9d6de586ee5593ec895ad674578e48d718897eff0"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -417,10 +417,11 @@
         },
         "govuk-bank-holidays": {
             "hashes": [
-                "sha256:32694697e1d9795d133cf8e39a1dd69baa23bda4a72b76f476dd4ee3df7d9b13",
-                "sha256:4cf28203b96c8ac07065538a0e49f44e4f0c6553ad418600b275f5ab60d23c68"
+                "sha256:2b91901be492235c4160a64d56d4d1945f6ca7f11c1ea4277395981866c35bef",
+                "sha256:ffb6ac050701cb850fd2f08eb9c7c91753090f00525e69cebec4f869f6f0de41"
             ],
-            "version": "==0.12"
+            "markers": "python_version >= '3.6'",
+            "version": "==0.13"
         },
         "greenlet": {
             "hashes": [
@@ -729,6 +730,27 @@
             ],
             "version": "==0.8.4"
         },
+        "newrelic": {
+            "hashes": [
+                "sha256:05ceedb760a2f76fc8477f969520426656f6949d5b5bb5634a430a9d9d5c18fb",
+                "sha256:2d12c4c96187c7a51958d3145dcbfebcbb6fe48d9d3219804e558da3769992a3",
+                "sha256:37821d6d47288605ed9de138b8ec5498bd597dd91bc296af7733e5b864c40cc1",
+                "sha256:6a13e6956042eeb8cf013a219f936c88293b8d6460234c5d772fd53640264d3a",
+                "sha256:720929793f4e17838b354fa997caa0d6474710149ac68c3aea23c29ba9123d1c",
+                "sha256:8810ea27ae22c7621caf61971f3b3ab1356d237317c2cffe80f605b73f5bcb37",
+                "sha256:9571efebcc330c157cab9a8d0c6d55f6d7ec146370c59423716534e9989d880d",
+                "sha256:a5650024071525256df7d74a9bc887840889c35d1e7ade51eda8eedb25d85e95",
+                "sha256:a8aeae34a518ac961e6c972f9d9479a008925faa585c9a89a9307c79927d92e4",
+                "sha256:b425ec7653bc7e6e73f2a7f55e707cd9341ac90696c0f3dcafc3ac6776aec959",
+                "sha256:c006ff70fbc87fe1f201cec0d22b5d209a58bbe31a2530fefe457f051ce44b86",
+                "sha256:d529e272cb433cee50bab89e2f01ec57ac1b8511845f9b777dfadd3eab3842dd",
+                "sha256:d79df5fb486ca2a0997f4f9cda9a7e8de39ae07df8161b5ab2742f22bcf50974",
+                "sha256:f3f66de81c4cce20474017397f372ef11cb52a629c97c2e1fe5c445e31c2bf07",
+                "sha256:fd33ec7e227701ceafb8e3894dcc3dafae5dbffc4a43814ad9e7976fdb418696"
+            ],
+            "index": "pypi",
+            "version": "==8.5.0"
+        },
         "notifications-python-client": {
             "hashes": [
                 "sha256:47c803fcc8b4098d069b92547bb52607b558cec25c19e2697a74faab2e5ef4c0"
@@ -799,10 +821,10 @@
         },
         "phonenumbers": {
             "hashes": [
-                "sha256:14f5a639eb47548595bb2baeff7bb63b32c95a4f79eed4dc7529afa2a65002d1",
-                "sha256:7d0c45c154013970b78b86f4188e0e00a8e77d6bebef7c81efdc038db876eeba"
+                "sha256:6d63455012fc9431105ffc7739befca61c3efc551b287dca58d2be2e745475a9",
+                "sha256:a577a46c069ad889c7b7cf4dd978751d059edeab28b97acead4775d2ea1fc70a"
             ],
-            "version": "==8.13.3"
+            "version": "==8.13.4"
         },
         "prometheus-client": {
             "hashes": [
@@ -1034,10 +1056,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:7ccfae7b4b2c067464a6733c6261673fdb8fd1be905460396b97a073e9fa683a",
-                "sha256:93007def75ae22f7cd991c84e02d434876818661f8df9ad5df9e950ff4e52cfd"
+                "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0",
+                "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"
             ],
-            "version": "==2022.7"
+            "version": "==2022.7.1"
         },
         "pyyaml": {
             "hashes": [
@@ -1076,18 +1098,18 @@
         },
         "redis": {
             "hashes": [
-                "sha256:7b8c87d19c45d3f1271b124858d2a5c13160c4e74d4835e28273400fa34d5228",
-                "sha256:cae3ee5d1f57d8caf534cd8764edf3163c77e073bdd74b6f54a87ffafdc5e7d9"
+                "sha256:a010f6cb7378065040a02839c3f75c7e0fb37a87116fb4a95be82a95552776c7",
+                "sha256:e6206448e2f8a432871d07d432c13ed6c2abcf6b74edb436c99752b1371be387"
             ],
-            "version": "==4.4.0"
+            "version": "==4.4.2"
         },
         "requests": {
             "hashes": [
-                "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
-                "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
+                "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa",
+                "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"
             ],
             "markers": "python_version >= '3.7' and python_version < '4'",
-            "version": "==2.28.1"
+            "version": "==2.28.2"
         },
         "rfc3339-validator": {
             "hashes": [
@@ -1121,11 +1143,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54",
-                "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"
+                "sha256:a78d01d1e2c175c474884671dde039962c9d74c7223db7369771fcf6e29ceeab",
+                "sha256:bd6eb2d6722568de6d14b87c44a96fac54b2a45ff5e940e639979a3d1792adb6"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==65.6.3"
+            "version": "==66.0.0"
         },
         "shapely": {
             "hashes": [
@@ -1259,11 +1281,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc",
-                "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"
+                "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72",
+                "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.13"
+            "version": "==1.26.14"
         },
         "vine": {
             "hashes": [
@@ -1275,10 +1297,10 @@
         },
         "wcwidth": {
             "hashes": [
-                "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
-                "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"
+                "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e",
+                "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"
             ],
-            "version": "==0.2.5"
+            "version": "==0.2.6"
         },
         "webcolors": {
             "hashes": [
@@ -1509,60 +1531,60 @@
                 "toml"
             ],
             "hashes": [
-                "sha256:037b51ee86bc600f99b3b957c20a172431c35c2ef9c1ca34bc813ab5b51fd9f5",
-                "sha256:0bce4ad5bdd0b02e177a085d28d2cea5fc57bb4ba2cead395e763e34cf934eb1",
-                "sha256:112cfead1bd22eada8a8db9ed387bd3e8be5528debc42b5d3c1f7da4ffaf9fb5",
-                "sha256:13121fa22dcd2c7b19c5161e3fd725692448f05377b788da4502a383573227b3",
-                "sha256:18b09811f849cc958d23f733a350a66b54a8de3fed1e6128ba55a5c97ffb6f65",
-                "sha256:25fde928306034e8deecd5fc91a07432dcc282c8acb76749581a28963c9f4f3f",
-                "sha256:2b865aa679bee7fbd1c55960940dbd3252621dd81468268786c67122bbd15343",
-                "sha256:2f7c51b6074a8a3063c341953dffe48fd6674f8e4b1d3c8aa8a91f58d6e716a8",
-                "sha256:349d0b545520e8516f7b4f12373afc705d17d901e1de6a37a20e4ec9332b61f7",
-                "sha256:44d6a556de4418f1f3bfd57094b8c49f0408df5a433cf0d253eeb3075261c762",
-                "sha256:4959dc506be74e4963bd2c42f7b87d8e4b289891201e19ec551e64c6aa5441f8",
-                "sha256:49da0ff241827ebb52d5d6d5a36d33b455fa5e721d44689c95df99fd8db82437",
-                "sha256:55e46fa4168ccb7497c9be78627fcb147e06f474f846a10d55feeb5108a24ef0",
-                "sha256:5722269ed05fbdb94eef431787c66b66260ff3125d1a9afcc00facff8c45adf9",
-                "sha256:628f47eaf66727fc986d3b190d6fa32f5e6b7754a243919d28bc0fd7974c449f",
-                "sha256:62ef3800c4058844e2e3fa35faa9dd0ccde8a8aba6c763aae50342e00d4479d4",
-                "sha256:75e43c6f4ea4d122dac389aabdf9d4f0e160770a75e63372f88005d90f5bcc80",
-                "sha256:7e8b0642c38b3d3b3c01417643ccc645345b03c32a2e84ef93cdd6844d6fe530",
-                "sha256:88834e5d56d01c141c29deedacba5773fe0bed900b1edc957595a8a6c0da1c3c",
-                "sha256:985ad2af5ec3dbb4fd75d5b0735752c527ad183455520055a08cf8d6794cabfc",
-                "sha256:a530663a361eb27375cec28aea5cd282089b5e4b022ae451c4c3493b026a68a5",
-                "sha256:acef7f3a3825a2d218a03dd02f5f3cc7f27aa31d882dd780191d1ad101120d74",
-                "sha256:ae871d09901911eedda1981ea6fd0f62a999107293cdc4c4fd612321c5b34745",
-                "sha256:af6cef3796b8068713a48dd67d258dc9a6e2ebc3bd4645bfac03a09672fa5d20",
-                "sha256:af87e906355fa42447be5c08c5d44e6e1c005bf142f303f726ddf5ed6e0c8a4d",
-                "sha256:b07651e3b9af8f1a092861d88b4c74d913634a7f1f2280fca0ad041ad84e9e96",
-                "sha256:b1ffc8f58b81baed3f8962e28c30d99442079b82ce1ec836a1f67c0accad91c1",
-                "sha256:b5b38813eee5b4739f505d94247604c72eae626d5088a16dd77b08b8b1724ab3",
-                "sha256:b791beb17b32ac019a78cfbe6184f992b6273fdca31145b928ad2099435e2fcb",
-                "sha256:b82343a5bc51627b9d606f0b6b6b9551db7b6311a5dd920fa52a94beae2e8959",
-                "sha256:ba9af1218fa01b1f11c72271bc7290b701d11ad4dbc2ae97c445ecacf6858dba",
-                "sha256:bdbda870e0fda7dd0fe7db7135ca226ec4c1ade8aa76e96614829b56ca491012",
-                "sha256:bf76d79dfaea802f0f28f50153ffbc1a74ae1ee73e480baeda410b4f3e7ab25f",
-                "sha256:c1cee10662c25c94415bbb987f2ec0e6ba9e8fce786334b10be7e6a7ab958f69",
-                "sha256:c5648c7eec5cf1ba5db1cf2d6c10036a582d7f09e172990474a122e30c841361",
-                "sha256:c58cd6bb46dcb922e0d5792850aab5964433d511b3a020867650f8d930dde4f4",
-                "sha256:c5d9b480ebae60fc2cbc8d6865194136bc690538fa542ba58726433bed6e04cc",
-                "sha256:ca15308ef722f120967af7474ba6a453e0f5b6f331251e20b8145497cf1bc14a",
-                "sha256:d0df04495b76a885bfef009f45eebe8fe2fbf815ad7a83dabcf5aced62f33162",
-                "sha256:d5be4e93acce64f516bf4fd239c0e6118fc913c93fa1a3f52d15bdcc60d97b2d",
-                "sha256:d8249666c23683f74f8f93aeaa8794ac87cc61c40ff70374a825f3352a4371dc",
-                "sha256:e3f1cd1cd65695b1540b3cf7828d05b3515974a9d7c7530f762ac40f58a18161",
-                "sha256:e56fae4292e216b8deeee38ace84557b9fa85b52db005368a275427cdabb8192",
-                "sha256:e6dcc70a25cb95df0ae33dfc701de9b09c37f7dd9f00394d684a5b57257f8246",
-                "sha256:e89d5abf86c104de808108a25d171ad646c07eda96ca76c8b237b94b9c71e518",
-                "sha256:ed7c9debf7bfc63c9b9f8b595409237774ff4b061bf29fba6f53b287a2fdeab9",
-                "sha256:ef001a60e888f8741e42e5aa79ae55c91be73761e4df5e806efca1ddd62fd400",
-                "sha256:f30090e22a301952c5abd0e493a1c8358b4f0b368b49fa3e4568ed3ed68b8d1f",
-                "sha256:f79691335257d60951638dd43576b9bcd6f52baa5c1c2cd07a509bb003238372",
-                "sha256:f918e9ef4c98f477a5458238dde2a1643aed956c7213873ab6b6b82e32b8ef61",
-                "sha256:fd0a8aa431f9b7ad9eb8264f55ef83cbb254962af3775092fb6e93890dea9ca2"
+                "sha256:051afcbd6d2ac39298d62d340f94dbb6a1f31de06dfaf6fcef7b759dd3860c45",
+                "sha256:0a1890fca2962c4f1ad16551d660b46ea77291fba2cc21c024cd527b9d9c8809",
+                "sha256:0ee30375b409d9a7ea0f30c50645d436b6f5dfee254edffd27e45a980ad2c7f4",
+                "sha256:13250b1f0bd023e0c9f11838bdeb60214dd5b6aaf8e8d2f110c7e232a1bff83b",
+                "sha256:17e01dd8666c445025c29684d4aabf5a90dc6ef1ab25328aa52bedaa95b65ad7",
+                "sha256:19245c249aa711d954623d94f23cc94c0fd65865661f20b7781210cb97c471c0",
+                "sha256:1caed2367b32cc80a2b7f58a9f46658218a19c6cfe5bc234021966dc3daa01f0",
+                "sha256:1f66862d3a41674ebd8d1a7b6f5387fe5ce353f8719040a986551a545d7d83ea",
+                "sha256:220e3fa77d14c8a507b2d951e463b57a1f7810a6443a26f9b7591ef39047b1b2",
+                "sha256:276f4cd0001cd83b00817c8db76730938b1ee40f4993b6a905f40a7278103b3a",
+                "sha256:29de916ba1099ba2aab76aca101580006adfac5646de9b7c010a0f13867cba45",
+                "sha256:2a7f23bbaeb2a87f90f607730b45564076d870f1fb07b9318d0c21f36871932b",
+                "sha256:2c407b1950b2d2ffa091f4e225ca19a66a9bd81222f27c56bd12658fc5ca1209",
+                "sha256:30b5fec1d34cc932c1bc04017b538ce16bf84e239378b8f75220478645d11fca",
+                "sha256:3c2155943896ac78b9b0fd910fb381186d0c345911f5333ee46ac44c8f0e43ab",
+                "sha256:411d4ff9d041be08fdfc02adf62e89c735b9468f6d8f6427f8a14b6bb0a85095",
+                "sha256:436e103950d05b7d7f55e39beeb4d5be298ca3e119e0589c0227e6d0b01ee8c7",
+                "sha256:49640bda9bda35b057b0e65b7c43ba706fa2335c9a9896652aebe0fa399e80e6",
+                "sha256:4a950f83fd3f9bca23b77442f3a2b2ea4ac900944d8af9993743774c4fdc57af",
+                "sha256:50a6adc2be8edd7ee67d1abc3cd20678987c7b9d79cd265de55941e3d0d56499",
+                "sha256:52ab14b9e09ce052237dfe12d6892dd39b0401690856bcfe75d5baba4bfe2831",
+                "sha256:54f7e9705e14b2c9f6abdeb127c390f679f6dbe64ba732788d3015f7f76ef637",
+                "sha256:66e50680e888840c0995f2ad766e726ce71ca682e3c5f4eee82272c7671d38a2",
+                "sha256:790e4433962c9f454e213b21b0fd4b42310ade9c077e8edcb5113db0818450cb",
+                "sha256:7a38362528a9115a4e276e65eeabf67dcfaf57698e17ae388599568a78dcb029",
+                "sha256:7b05ed4b35bf6ee790832f68932baf1f00caa32283d66cc4d455c9e9d115aafc",
+                "sha256:7e109f1c9a3ece676597831874126555997c48f62bddbcace6ed17be3e372de8",
+                "sha256:949844af60ee96a376aac1ded2a27e134b8c8d35cc006a52903fc06c24a3296f",
+                "sha256:95304068686545aa368b35dfda1cdfbbdbe2f6fe43de4a2e9baa8ebd71be46e2",
+                "sha256:9e662e6fc4f513b79da5d10a23edd2b87685815b337b1a30cd11307a6679148d",
+                "sha256:a9fed35ca8c6e946e877893bbac022e8563b94404a605af1d1e6accc7eb73289",
+                "sha256:b69522b168a6b64edf0c33ba53eac491c0a8f5cc94fa4337f9c6f4c8f2f5296c",
+                "sha256:b78729038abea6a5df0d2708dce21e82073463b2d79d10884d7d591e0f385ded",
+                "sha256:b8c56bec53d6e3154eaff6ea941226e7bd7cc0d99f9b3756c2520fc7a94e6d96",
+                "sha256:b9727ac4f5cf2cbf87880a63870b5b9730a8ae3a4a360241a0fdaa2f71240ff0",
+                "sha256:ba3027deb7abf02859aca49c865ece538aee56dcb4871b4cced23ba4d5088904",
+                "sha256:be9fcf32c010da0ba40bf4ee01889d6c737658f4ddff160bd7eb9cac8f094b21",
+                "sha256:c18d47f314b950dbf24a41787ced1474e01ca816011925976d90a88b27c22b89",
+                "sha256:c76a3075e96b9c9ff00df8b5f7f560f5634dffd1658bafb79eb2682867e94f78",
+                "sha256:cbfcba14a3225b055a28b3199c3d81cd0ab37d2353ffd7f6fd64844cebab31ad",
+                "sha256:d254666d29540a72d17cc0175746cfb03d5123db33e67d1020e42dae611dc196",
+                "sha256:d66187792bfe56f8c18ba986a0e4ae44856b1c645336bd2c776e3386da91e1dd",
+                "sha256:d8d04e755934195bdc1db45ba9e040b8d20d046d04d6d77e71b3b34a8cc002d0",
+                "sha256:d8f3e2e0a1d6777e58e834fd5a04657f66affa615dae61dd67c35d1568c38882",
+                "sha256:e057e74e53db78122a3979f908973e171909a58ac20df05c33998d52e6d35757",
+                "sha256:e4ce984133b888cc3a46867c8b4372c7dee9cee300335e2925e197bcd45b9e16",
+                "sha256:ea76dbcad0b7b0deb265d8c36e0801abcddf6cc1395940a24e3595288b405ca0",
+                "sha256:ecb0f73954892f98611e183f50acdc9e21a4653f294dfbe079da73c6378a6f47",
+                "sha256:ef14d75d86f104f03dea66c13188487151760ef25dd6b2dbd541885185f05f40",
+                "sha256:f26648e1b3b03b6022b48a9b910d0ae209e2d51f50441db5dce5b530fad6d9b1",
+                "sha256:f67472c09a0c7486e27f3275f617c964d25e35727af952869dd496b9b5b7f6a3"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.0.3"
+            "version": "==7.0.5"
         },
         "cryptography": {
             "hashes": [
@@ -1595,11 +1617,11 @@
         },
         "cyclonedx-python-lib": {
             "hashes": [
-                "sha256:48ae942a892e8385f4e0193d2e295a338df9ab864652081406c26f58085d2b35",
-                "sha256:a03b8f79f23aa95d37180b5d7bca81ef393b569e2d29e02f4817cfe4488e1ba2"
+                "sha256:1ccd482024a30b95c4fffb3fe567a9df97b705f34c1075f8abde8537867600c3",
+                "sha256:8981ca462fba91469c268d684a03f72c89c7a807674d884f83a28d8c2822a9b6"
             ],
             "markers": "python_version >= '3.6' and python_version < '4.0'",
-            "version": "==3.1.1"
+            "version": "==3.1.5"
         },
         "execnet": {
             "hashes": [
@@ -1674,10 +1696,11 @@
         },
         "iniconfig": {
             "hashes": [
-                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
-                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+                "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
+                "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"
             ],
-            "version": "==1.1.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.0"
         },
         "isort": {
             "hashes": [
@@ -1857,11 +1880,11 @@
         },
         "pbr": {
             "hashes": [
-                "sha256:b97bc6695b2aff02144133c2e7399d5885223d42b7912ffaec2ca3898e673bfe",
-                "sha256:db2317ff07c84c4c63648c9064a79fe9d9f5c7ce85a9099d4b6258b3db83225a"
+                "sha256:567f09558bae2b3ab53cb3c1e2e33e726ff3338e7bae3db5dc954b3a44eef12b",
+                "sha256:aefc51675b0b533d56bb5fd1c8c6c0522fe31896679882e1c4c63d5e4a0fccb3"
             ],
             "markers": "python_version >= '2.6'",
-            "version": "==5.11.0"
+            "version": "==5.11.1"
         },
         "pip": {
             "hashes": [
@@ -2007,10 +2030,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:7ccfae7b4b2c067464a6733c6261673fdb8fd1be905460396b97a073e9fa683a",
-                "sha256:93007def75ae22f7cd991c84e02d434876818661f8df9ad5df9e950ff4e52cfd"
+                "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0",
+                "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"
             ],
-            "version": "==2022.7"
+            "version": "==2022.7.1"
         },
         "pyyaml": {
             "hashes": [
@@ -2049,11 +2072,11 @@
         },
         "requests": {
             "hashes": [
-                "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
-                "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
+                "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa",
+                "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"
             ],
             "markers": "python_version >= '3.7' and python_version < '4'",
-            "version": "==2.28.1"
+            "version": "==2.28.2"
         },
         "requests-mock": {
             "hashes": [
@@ -2080,11 +2103,11 @@
         },
         "rich": {
             "hashes": [
-                "sha256:25f83363f636995627a99f6e4abc52ed0970ebbd544960cc63cbb43aaac3d6f0",
-                "sha256:41fe1d05f433b0f4724cda8345219213d2bfa472ef56b2f64f415b5b94d51b04"
+                "sha256:81c73a30b144bbcdedc13f4ea0b6ffd7fdc3b0d3cc259a9402309c8e4aee1964",
+                "sha256:f846bff22a43e8508aebf3f0f2410ce1c6f4cde429098bd58d91fde038c57299"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==13.0.1"
+            "version": "==13.1.0"
         },
         "s3transfer": {
             "hashes": [
@@ -2096,11 +2119,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54",
-                "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"
+                "sha256:a78d01d1e2c175c474884671dde039962c9d74c7223db7369771fcf6e29ceeab",
+                "sha256:bd6eb2d6722568de6d14b87c44a96fac54b2a45ff5e940e639979a3d1792adb6"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==65.6.3"
+            "version": "==66.0.0"
         },
         "six": {
             "hashes": [
@@ -2158,11 +2181,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc",
-                "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"
+                "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72",
+                "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.26.13"
+            "version": "==1.26.14"
         },
         "webencodings": {
             "hashes": [
diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py
index 1c8d090e0..4abda5acd 100644
--- a/app/notifications/process_notifications.py
+++ b/app/notifications/process_notifications.py
@@ -105,13 +105,13 @@ def persist_notification(
     document_download_count=None,
     updated_at=None
 ):
-    current_app.logger.info('Presisting notification')
+    current_app.logger.info('Persisting notification')
 
     notification_created_at = created_at or datetime.utcnow()
     if not notification_id:
         notification_id = uuid.uuid4()
 
-    current_app.logger.info('Presisting notification with id {}'.format(notification_id))
+    current_app.logger.info('Persisting notification with id {}'.format(notification_id))
 
     notification = Notification(
         id=notification_id,
@@ -136,7 +136,7 @@ def persist_notification(
         updated_at=updated_at
     )
 
-    current_app.logger.info('Presisting notification with to address: {}'.format(notification.to))
+    current_app.logger.info('Persisting notification with to address: {}'.format(notification.to))
 
     if notification_type == SMS_TYPE:
         formatted_recipient = validate_and_format_phone_number(recipient, international=True)
@@ -146,9 +146,9 @@ def persist_notification(
         notification.phone_prefix = recipient_info.country_prefix
         notification.rate_multiplier = recipient_info.billable_units
     elif notification_type == EMAIL_TYPE:
-        current_app.logger.info('Presisting notification with type: {}'.format(EMAIL_TYPE))
+        current_app.logger.info('Persisting notification with type: {}'.format(EMAIL_TYPE))
         notification.normalised_to = format_email_address(notification.to)
-        current_app.logger.info('Presisting notification to formatted email: {}'.format(notification.normalised_to))
+        current_app.logger.info('Persisting notification to formatted email: {}'.format(notification.normalised_to))
     elif notification_type == LETTER_TYPE:
         notification.postage = postage
         notification.international = postage in INTERNATIONAL_POSTAGE_TYPES
diff --git a/newrelic.ini b/newrelic.ini
new file mode 100644
index 000000000..0ecb31567
--- /dev/null
+++ b/newrelic.ini
@@ -0,0 +1,219 @@
+# ---------------------------------------------------------------------------
+
+#
+# This file configures the New Relic Python Agent.
+#
+# The path to the configuration file should be supplied to the function
+# newrelic.agent.initialize() when the agent is being initialized.
+#
+# The configuration file follows a structure similar to what you would
+# find for Microsoft Windows INI files. For further information on the
+# configuration file format see the Python ConfigParser documentation at:
+#
+#    http://docs.python.org/library/configparser.html
+#
+# For further discussion on the behaviour of the Python agent that can
+# be configured via this configuration file see:
+#
+#    http://newrelic.com/docs/python/python-agent-configuration
+#
+
+# ---------------------------------------------------------------------------
+
+# Here are the settings that are common to all environments.
+
+[newrelic]
+
+# You must specify the license key associated with your New
+# Relic account. This key binds the Python Agent's data to your
+# account in the New Relic service.
+license_key = 88822380cb0c943af94c03098042a78b53d1NRAL
+
+host = gov-collector.newrelic.com
+
+# The application name. Set this to be the name of your
+# application as you would like it to show up in New Relic UI.
+# The UI will then auto-map instances of your application into a
+# entry on your home dashboard page.
+app_name = us-notify-api (Development)
+
+# New Relic offers distributed tracing for monitoring and analyzing modern
+# distributed systems.Enable distributed tracing.
+distributed_tracing.enabled = true
+
+# When "true", the agent collects performance data about your
+# application and reports this data to the New Relic UI at
+# newrelic.com. This global switch is normally overridden for
+# each environment below.
+monitor_mode = true
+
+# Sets the name of a file to log agent messages to. Useful for
+# debugging any issues with the agent. This is not set by
+# default as it is not known in advance what user your web
+# application processes will run as and where they have
+# permission to write to. Whatever you set this to you must
+# ensure that the permissions for the containing directory and
+# the file itself are correct, and that the user that your web
+# application runs as can write to the file. If not able to
+# write out a log file, it is also possible to say "stderr" and
+# output to standard error output. This would normally result in
+# output appearing in your web server log.
+#log_file = /tmp/newrelic-python-agent.log
+
+# Sets the level of detail of messages sent to the log file, if
+# a log file location has been provided. Possible values, in
+# increasing order of detail, are: "critical", "error", "warning",
+# "info" and "debug". When reporting any agent issues to New
+# Relic technical support, the most useful setting for the
+# support engineers is "debug". However, this can generate a lot
+# of information very quickly, so it is best not to keep the
+# agent at this level for longer than it takes to reproduce the
+# problem you are experiencing.
+log_level = info
+
+# The Python Agent communicates with the New Relic service using
+# SSL by default. Note that this does result in an increase in
+# CPU overhead, over and above what would occur for a non SSL
+# connection, to perform the encryption involved in the SSL
+# communication. This work is though done in a distinct thread
+# to those handling your web requests, so it should not impact
+# response times. You can if you wish revert to using a non SSL
+# connection, but this will result in information being sent
+# over a plain socket connection and will not be as secure.
+ssl = true
+
+# High Security Mode enforces certain security settings, and
+# prevents them from being overridden, so that no sensitive data
+# is sent to New Relic. Enabling High Security Mode means that
+# SSL is turned on, request parameters are not collected, and SQL
+# can not be sent to New Relic in its raw form. To activate High
+# Security Mode, it must be set to 'true' in this local .ini
+# configuration file AND be set to 'true' in the server-side
+# configuration in the New Relic user interface. For details, see
+# https://docs.newrelic.com/docs/subscriptions/high-security
+high_security = false
+
+# The Python Agent will attempt to connect directly to the New
+# Relic service. If there is an intermediate firewall between
+# your host and the New Relic service that requires you to use a
+# HTTP proxy, then you should set both the "proxy_host" and
+# "proxy_port" settings to the required values for the HTTP
+# proxy. The "proxy_user" and "proxy_pass" settings should
+# additionally be set if proxy authentication is implemented by
+# the HTTP proxy. The "proxy_scheme" setting dictates what
+# protocol scheme is used in talking to the HTTP proxy. This
+# would normally always be set as "http" which will result in the
+# agent then using a SSL tunnel through the HTTP proxy for end to
+# end encryption.
+# proxy_scheme = http
+# proxy_host = hostname
+# proxy_port = 8080
+# proxy_user =
+# proxy_pass =
+
+# Capturing request parameters is off by default. To enable the
+# capturing of request parameters, first ensure that the setting
+# "attributes.enabled" is set to "true" (the default value), and
+# then add "request.parameters.*" to the "attributes.include"
+# setting. For details about attributes configuration, please
+# consult the documentation.
+# attributes.include = request.parameters.*
+
+# The transaction tracer captures deep information about slow
+# transactions and sends this to the UI on a periodic basis. The
+# transaction tracer is enabled by default. Set this to "false"
+# to turn it off.
+transaction_tracer.enabled = true
+
+# Threshold in seconds for when to collect a transaction trace.
+# When the response time of a controller action exceeds this
+# threshold, a transaction trace will be recorded and sent to
+# the UI. Valid values are any positive float value, or (default)
+# "apdex_f", which will use the threshold for a dissatisfying
+# Apdex controller action - four times the Apdex T value.
+transaction_tracer.transaction_threshold = apdex_f
+
+# When the transaction tracer is on, SQL statements can
+# optionally be recorded. The recorder has three modes, "off"
+# which sends no SQL, "raw" which sends the SQL statement in its
+# original form, and "obfuscated", which strips out numeric and
+# string literals.
+transaction_tracer.record_sql = obfuscated
+
+# Threshold in seconds for when to collect stack trace for a SQL
+# call. In other words, when SQL statements exceed this
+# threshold, then capture and send to the UI the current stack
+# trace. This is helpful for pinpointing where long SQL calls
+# originate from in an application.
+transaction_tracer.stack_trace_threshold = 0.5
+
+# Determines whether the agent will capture query plans for slow
+# SQL queries. Only supported in MySQL and PostgreSQL. Set this
+# to "false" to turn it off.
+transaction_tracer.explain_enabled = true
+
+# Threshold for query execution time below which query plans
+# will not not be captured. Relevant only when "explain_enabled"
+# is true.
+transaction_tracer.explain_threshold = 0.5
+
+# Space separated list of function or method names in form
+# 'module:function' or 'module:class.function' for which
+# additional function timing instrumentation will be added.
+transaction_tracer.function_trace =
+
+# The error collector captures information about uncaught
+# exceptions or logged exceptions and sends them to UI for
+# viewing. The error collector is enabled by default. Set this
+# to "false" to turn it off.
+error_collector.enabled = true
+
+# To stop specific errors from reporting to the UI, set this to
+# a space separated list of the Python exception type names to
+# ignore. The exception name should be of the form 'module:class'.
+error_collector.ignore_errors =
+
+# Browser monitoring is the Real User Monitoring feature of the UI.
+# For those Python web frameworks that are supported, this
+# setting enables the auto-insertion of the browser monitoring
+# JavaScript fragments.
+browser_monitoring.auto_instrument = false
+
+# A thread profiling session can be scheduled via the UI when
+# this option is enabled. The thread profiler will periodically
+# capture a snapshot of the call stack for each active thread in
+# the application to construct a statistically representative
+# call tree.
+thread_profiler.enabled = true
+
+# ---------------------------------------------------------------------------
+
+#
+# The application environments. These are specific settings which
+# override the common environment settings. The settings related to a
+# specific environment will be used when the environment argument to the
+# newrelic.agent.initialize() function has been defined to be either
+# "development", "test", "staging" or "production".
+#
+
+[newrelic:development]
+app_name = us-notify-api (Development)
+monitor_mode = true
+
+[newrelic:test]
+app_name = us-notify-api (Test)
+monitor_mode = false
+
+[newrelic:staging]
+app_name = us-notify-api (Staging)
+monitor_mode = true
+
+[newrelic:demo]
+app_name = us-notify-api (Demo)
+monitor_mode = true
+
+[newrelic:production]
+app_name = us-notify-api
+monitor_mode = true
+
+# ---------------------------------------------------------------------------
diff --git a/sample.env b/sample.env
index a38f29e2a..8e02bbc3d 100644
--- a/sample.env
+++ b/sample.env
@@ -56,3 +56,8 @@ NOTIFY_APP_NAME=api
 FLASK_APP=application.py
 FLASK_ENV=development
 WERKZEUG_DEBUG_PIN=off
+
+#############################################################
+
+# New Relic
+NEW_RELIC_CONFIG_FILE=newrelic.ini

From bec3c531287b29532785e9ccf403635399822c90 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Wed, 18 Jan 2023 09:20:22 -0500
Subject: [PATCH 35/46] Setup newrelic for cloud.gov environments

---
 .github/workflows/checks.yml                            | 1 +
 .github/workflows/daily_checks.yml                      | 1 +
 .profile                                                | 1 +
 deploy-config/egress_proxy/notify-api-demo.allow.acl    | 1 +
 deploy-config/egress_proxy/notify-api-staging.allow.acl | 1 +
 manifest.yml                                            | 4 +++-
 newrelic.ini                                            | 4 ++++
 scripts/migrate_and_run_web.sh                          | 2 +-
 8 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 70fd96b2d..e5eb879b9 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -8,6 +8,7 @@ permissions:
 env:
   DEBUG: True
   NOTIFY_ENVIRONMENT: test
+  NEW_RELIC_ENVIRONMENT: test
   FLASK_APP: application.py
   FLASK_ENV: development
   WERKZEUG_DEBUG_PIN: off
diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml
index 34cef2173..2fec03578 100644
--- a/.github/workflows/daily_checks.yml
+++ b/.github/workflows/daily_checks.yml
@@ -12,6 +12,7 @@ permissions:
 env:
   DEBUG: True
   NOTIFY_ENVIRONMENT: test
+  NEW_RELIC_ENVIRONMENT: test
   FLASK_APP: application.py
   FLASK_ENV: development
   WERKZEUG_DEBUG_PIN: off
diff --git a/.profile b/.profile
index 9960f33a2..1c1f98b4b 100644
--- a/.profile
+++ b/.profile
@@ -4,3 +4,4 @@
 ##
 
 export https_proxy=$egress_proxy
+export NEW_RELIC_PROXY_HOST=$egress_proxy
diff --git a/deploy-config/egress_proxy/notify-api-demo.allow.acl b/deploy-config/egress_proxy/notify-api-demo.allow.acl
index 711a1adfd..13db9062b 100644
--- a/deploy-config/egress_proxy/notify-api-demo.allow.acl
+++ b/deploy-config/egress_proxy/notify-api-demo.allow.acl
@@ -1,2 +1,3 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
+gov-collector.newrelic.com
diff --git a/deploy-config/egress_proxy/notify-api-staging.allow.acl b/deploy-config/egress_proxy/notify-api-staging.allow.acl
index 711a1adfd..13db9062b 100644
--- a/deploy-config/egress_proxy/notify-api-staging.allow.acl
+++ b/deploy-config/egress_proxy/notify-api-staging.allow.acl
@@ -1,2 +1,3 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
+gov-collector.newrelic.com
diff --git a/manifest.yml b/manifest.yml
index 3a75d8c29..29a5c092a 100644
--- a/manifest.yml
+++ b/manifest.yml
@@ -22,7 +22,7 @@ applications:
       - type: worker
         instances: ((worker_instances))
         memory: ((worker_memory))
-        command: celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4
+        command: newrelic-admin run-program celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4
       - type: scheduler
         instances: 1
         memory: ((scheduler_memory))
@@ -32,6 +32,8 @@ applications:
       NOTIFY_APP_NAME: api
       FLASK_APP: application.py
       FLASK_ENV: production
+      NEW_RELIC_CONFIG_FILE: newrelic.ini
+      NEW_RELIC_ENVIRONMENT: ((env))
 
       NOTIFY_ENVIRONMENT: ((env))
       API_HOST_NAME: https://((public_api_route))
diff --git a/newrelic.ini b/newrelic.ini
index 0ecb31567..81762387c 100644
--- a/newrelic.ini
+++ b/newrelic.ini
@@ -204,6 +204,10 @@ monitor_mode = true
 app_name = us-notify-api (Test)
 monitor_mode = false
 
+[newrelic:sandbox]
+app_name = us-notify-api (Sandbox)
+monitor_mode = false
+
 [newrelic:staging]
 app_name = us-notify-api (Staging)
 monitor_mode = true
diff --git a/scripts/migrate_and_run_web.sh b/scripts/migrate_and_run_web.sh
index 89a4dd5ea..3e39dceb6 100755
--- a/scripts/migrate_and_run_web.sh
+++ b/scripts/migrate_and_run_web.sh
@@ -4,4 +4,4 @@ if [[ $CF_INSTANCE_INDEX -eq 0 ]]; then
   flask db upgrade
 fi
 
-exec gunicorn -c ${HOME}/gunicorn_config.py application
+exec newrelic-admin run-program gunicorn -c ${HOME}/gunicorn_config.py application

From 50bc1532a348e6cbed00bbb57efdb4edff2df4ae Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Wed, 18 Jan 2023 10:03:45 -0500
Subject: [PATCH 36/46] Turn off newrelic for celery-beat

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d041013e0..e09fba6e9 100644
--- a/Makefile
+++ b/Makefile
@@ -39,7 +39,7 @@ run-celery: ## Run celery, TODO remove purge for staging/prod
 
 .PHONY: run-celery-beat
 run-celery-beat: ## Run celery beat
-	pipenv run newrelic-admin run-program celery \
+	pipenv run celery \
 	-A run_celery.notify_celery beat \
 	--loglevel=INFO
 

From 0652b215219e5c3d33c7badf65444b3640597243 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Wed, 18 Jan 2023 10:49:39 -0500
Subject: [PATCH 37/46] Set new relic license key via env var

---
 .github/workflows/deploy-demo.yml | 2 ++
 .github/workflows/deploy.yml      | 2 ++
 manifest.yml                      | 1 +
 newrelic.ini                      | 4 ++--
 sample.env                        | 1 +
 5 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml
index ac800562b..a474ed8be 100644
--- a/.github/workflows/deploy-demo.yml
+++ b/.github/workflows/deploy-demo.yml
@@ -53,6 +53,7 @@ jobs:
         ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
@@ -65,6 +66,7 @@ jobs:
           --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
           --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"
           --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"
+          --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
 
     - name: Check for changes to egress config
       id: changed-egress-config
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 9349850df..70a1c5c2c 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -58,6 +58,7 @@ jobs:
         ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
       with:
         cf_username: ${{ secrets.CLOUDGOV_USERNAME }}
         cf_password: ${{ secrets.CLOUDGOV_PASSWORD }}
@@ -70,6 +71,7 @@ jobs:
           --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"
           --var AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID"
           --var AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY"
+          --var NEW_RELIC_LICENSE_KEY="$NEW_RELIC_LICENSE_KEY"
 
     - name: Check for changes to egress config
       id: changed-egress-config
diff --git a/manifest.yml b/manifest.yml
index 29a5c092a..130b2ff2e 100644
--- a/manifest.yml
+++ b/manifest.yml
@@ -34,6 +34,7 @@ applications:
       FLASK_ENV: production
       NEW_RELIC_CONFIG_FILE: newrelic.ini
       NEW_RELIC_ENVIRONMENT: ((env))
+      NEW_RELIC_LICENSE_KEY: ((NEW_RELIC_LICENSE_KEY))
 
       NOTIFY_ENVIRONMENT: ((env))
       API_HOST_NAME: https://((public_api_route))
diff --git a/newrelic.ini b/newrelic.ini
index 81762387c..996fa6ed5 100644
--- a/newrelic.ini
+++ b/newrelic.ini
@@ -27,7 +27,7 @@
 # You must specify the license key associated with your New
 # Relic account. This key binds the Python Agent's data to your
 # account in the New Relic service.
-license_key = 88822380cb0c943af94c03098042a78b53d1NRAL
+# license_key is set via NEW_RELIC_LICENSE_KEY env var
 
 host = gov-collector.newrelic.com
 
@@ -45,7 +45,7 @@ distributed_tracing.enabled = true
 # application and reports this data to the New Relic UI at
 # newrelic.com. This global switch is normally overridden for
 # each environment below.
-monitor_mode = true
+monitor_mode = false
 
 # Sets the name of a file to log agent messages to. Useful for
 # debugging any issues with the agent. This is not set by
diff --git a/sample.env b/sample.env
index 8e02bbc3d..70d1fa779 100644
--- a/sample.env
+++ b/sample.env
@@ -61,3 +61,4 @@ WERKZEUG_DEBUG_PIN=off
 
 # New Relic
 NEW_RELIC_CONFIG_FILE=newrelic.ini
+NEW_RELIC_LICENSE_KEY="don't write secrets to the sample file"

From b9a53b7b54248b61ba17042e61e4483ea4ff31b5 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 19 Jan 2023 09:33:33 -0500
Subject: [PATCH 38/46] Validate new relic config in github actions

---
 .github/workflows/checks.yml       | 14 ++++++++++++++
 .github/workflows/daily_checks.yml |  1 +
 newrelic.ini                       |  2 +-
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index e5eb879b9..146abfaf6 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -8,6 +8,7 @@ permissions:
 env:
   DEBUG: True
   NOTIFY_ENVIRONMENT: test
+  NEW_RELIC_CONFIG_FILE: newrelic.ini
   NEW_RELIC_ENVIRONMENT: test
   FLASK_APP: application.py
   FLASK_ENV: development
@@ -53,6 +54,19 @@ jobs:
       env:
         SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
 
+  validate-new-relic-config:
+    runs-on: ubuntu-latest
+    environment: staging
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup-project
+      - name: Validate NewRelic config
+        env:
+          NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+          # Need to set a NEW_RELIC_ENVIRONMENT with monitor_mode: true
+          NEW_RELIC_ENVIRONMENT: staging
+        run: pipenv run newrelic-admin validate-config $NEW_RELIC_CONFIG_FILE
+
   pip-audit:
     runs-on: ubuntu-latest
     steps:
diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml
index 2fec03578..e4968f723 100644
--- a/.github/workflows/daily_checks.yml
+++ b/.github/workflows/daily_checks.yml
@@ -12,6 +12,7 @@ permissions:
 env:
   DEBUG: True
   NOTIFY_ENVIRONMENT: test
+  NEW_RELIC_CONFIG_FILE: newrelic.ini
   NEW_RELIC_ENVIRONMENT: test
   FLASK_APP: application.py
   FLASK_ENV: development
diff --git a/newrelic.ini b/newrelic.ini
index 996fa6ed5..ee0d090dc 100644
--- a/newrelic.ini
+++ b/newrelic.ini
@@ -171,7 +171,7 @@ error_collector.enabled = true
 # To stop specific errors from reporting to the UI, set this to
 # a space separated list of the Python exception type names to
 # ignore. The exception name should be of the form 'module:class'.
-error_collector.ignore_errors =
+error_collector.ignore_classes =
 
 # Browser monitoring is the Real User Monitoring feature of the UI.
 # For those Python web frameworks that are supported, this

From c3faf3df4d4f3b0f11a9eac26d828e80bbe6b29f Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Thu, 19 Jan 2023 09:38:18 -0500
Subject: [PATCH 39/46] Install newrelic before attempting to validate config

---
 .github/workflows/checks.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 146abfaf6..c5f2a3d0c 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -60,6 +60,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-project
+      - name: Install pipenv packages
+        run: pipenv install --dev
       - name: Validate NewRelic config
         env:
           NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}

From 35fdd6132df28d4096019d97fa401b5e3a6561c1 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Mon, 23 Jan 2023 12:00:58 -0500
Subject: [PATCH 40/46] Add onboarding step for new relic to infra-overview.md

---
 docs/infra-overview.md | 9 +++++++--
 newrelic.ini           | 3 +--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/docs/infra-overview.md b/docs/infra-overview.md
index db017f8f3..301d618d4 100644
--- a/docs/infra-overview.md
+++ b/docs/infra-overview.md
@@ -22,16 +22,21 @@ In SNS, we have 3 topics for SMS receipts. These are not currently functional, s
 
 Through Pinpoint, the API needs at least one number so that the application itself can send SMS for authentication codes.
 
-The API also has access to AWS S3 buckets for storing CSVs of messages and contact lists. It does not access a third S3 bucket that stores agency logos. 
+The API also has access to AWS S3 buckets for storing CSVs of messages and contact lists. It does not access a third S3 bucket that stores agency logos.
 
 We may be able to provision these services through cloud.gov, as well. In addition to [s3 support](https://cloud.gov/docs/services/s3/), there is [an SES brokerpak](https://github.com/GSA-TTS/datagov-brokerpak-smtp) and work on an SNS brokerpak.
 
+## New Relic
+
+We are using [New Relic](https://one.newrelic.com/nr1-core?account=3389907) for application monitoring and error reporting. When requesting access to New Relic, ask to be added to the Benefits-Studio subaccount.
+
 ## Onboarding
 
 - [ ] Join [the GSA GitHub org](https://github.com/GSA/GitHub-Administration#join-the-gsa-organization)
 - [ ] Get permissions for the repos
 - [ ] Get access to the cloud.gov org && space
 - [ ] Get [access to AWS](https://handbook.tts.gsa.gov/launching-software/infrastructure/#cloud-service-provider-csp-sandbox-accounts), if necessary
+- [ ] Get [access to New Relic](https://handbook.tts.gsa.gov/tools/new-relic/#how-do-i-get-access-to-new-relic), if necessary
 - [ ] Pull down creds from cloud.gov and create the local .env file
 - [ ] Do stuff!
 
@@ -54,4 +59,4 @@ TODO: create env vars for these origin and destination email addresses for the r
 6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)"
 7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number)
 
-At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! 🎉
\ No newline at end of file
+At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! 🎉
diff --git a/newrelic.ini b/newrelic.ini
index ee0d090dc..638dbf84e 100644
--- a/newrelic.ini
+++ b/newrelic.ini
@@ -45,7 +45,7 @@ distributed_tracing.enabled = true
 # application and reports this data to the New Relic UI at
 # newrelic.com. This global switch is normally overridden for
 # each environment below.
-monitor_mode = false
+monitor_mode = true
 
 # Sets the name of a file to log agent messages to. Useful for
 # debugging any issues with the agent. This is not set by
@@ -198,7 +198,6 @@ thread_profiler.enabled = true
 
 [newrelic:development]
 app_name = us-notify-api (Development)
-monitor_mode = true
 
 [newrelic:test]
 app_name = us-notify-api (Test)

From 914a7cc969c665676dc1d0b84551775c20dc18c7 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 24 Jan 2023 13:43:37 -0500
Subject: [PATCH 41/46] Updates for sandbox env

---
 deploy-config/sandbox.yml | 6 +++---
 newrelic.ini              | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/deploy-config/sandbox.yml b/deploy-config/sandbox.yml
index 70e1da125..aafa9b491 100644
--- a/deploy-config/sandbox.yml
+++ b/deploy-config/sandbox.yml
@@ -6,6 +6,6 @@ worker_memory: 512M
 scheduler_memory: 256M
 public_api_route: notify-api-sandbox.app.cloud.gov
 admin_base_url: https://notify-sandbox.app.cloud.gov
-ADMIN_CLIENT_SECRET: dev-notify-secret-key
-DANGEROUS_SALT: dev-notify-salt
-SECRET_KEY: dev-notify-secret-key
+ADMIN_CLIENT_SECRET: sandbox-notify-secret-key
+DANGEROUS_SALT: sandbox-notify-salt
+SECRET_KEY: sandbox-notify-secret-key
diff --git a/newrelic.ini b/newrelic.ini
index 638dbf84e..394ae3c1a 100644
--- a/newrelic.ini
+++ b/newrelic.ini
@@ -205,7 +205,7 @@ monitor_mode = false
 
 [newrelic:sandbox]
 app_name = us-notify-api (Sandbox)
-monitor_mode = false
+monitor_mode = true
 
 [newrelic:staging]
 app_name = us-notify-api (Staging)

From c54a207c5755abaab101f954578f705a97605376 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 24 Jan 2023 13:48:38 -0500
Subject: [PATCH 42/46] Update proxy settings to enable sending data to new
 relic

---
 .github/actions/deploy-proxy/action.yml                 | 4 ++--
 deploy-config/egress_proxy/notify-api-demo.allow.acl    | 1 +
 deploy-config/egress_proxy/notify-api-staging.allow.acl | 1 +
 manifest.yml                                            | 1 +
 4 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/actions/deploy-proxy/action.yml b/.github/actions/deploy-proxy/action.yml
index 2c1f4e882..13a773fa0 100644
--- a/.github/actions/deploy-proxy/action.yml
+++ b/.github/actions/deploy-proxy/action.yml
@@ -9,10 +9,10 @@ inputs:
     required: true
   proxy_repo:
     description: git repo for cg-egress-proxy
-    default: https://github.com/GSA/cg-egress-proxy.git
+    default: https://github.com/rahearn/cg-egress-proxy.git
   proxy_version:
     description: git ref to be deployed
-    default: main
+    default: new-relic-connection
 runs:
   using: composite
   steps:
diff --git a/deploy-config/egress_proxy/notify-api-demo.allow.acl b/deploy-config/egress_proxy/notify-api-demo.allow.acl
index 13db9062b..7061eef91 100644
--- a/deploy-config/egress_proxy/notify-api-demo.allow.acl
+++ b/deploy-config/egress_proxy/notify-api-demo.allow.acl
@@ -1,3 +1,4 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
 gov-collector.newrelic.com
+*.apps.internal
diff --git a/deploy-config/egress_proxy/notify-api-staging.allow.acl b/deploy-config/egress_proxy/notify-api-staging.allow.acl
index 13db9062b..7061eef91 100644
--- a/deploy-config/egress_proxy/notify-api-staging.allow.acl
+++ b/deploy-config/egress_proxy/notify-api-staging.allow.acl
@@ -1,3 +1,4 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
 gov-collector.newrelic.com
+*.apps.internal
diff --git a/manifest.yml b/manifest.yml
index 130b2ff2e..ae64daf6c 100644
--- a/manifest.yml
+++ b/manifest.yml
@@ -51,3 +51,4 @@ applications:
       AWS_US_TOLL_FREE_NUMBER: +18446120782
 
       REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt"
+      NEW_RELIC_CA_BUNDLE_PATH: "/etc/ssl/certs/ca-certificates.crt"

From b46dfe98d82d65a1e819881f28ea5df33416f6a6 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Tue, 24 Jan 2023 15:57:43 -0500
Subject: [PATCH 43/46] Remove wildcard matcher on egress allow

---
 deploy-config/egress_proxy/notify-api-demo.allow.acl    | 2 +-
 deploy-config/egress_proxy/notify-api-staging.allow.acl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/deploy-config/egress_proxy/notify-api-demo.allow.acl b/deploy-config/egress_proxy/notify-api-demo.allow.acl
index 7061eef91..fcd95f284 100644
--- a/deploy-config/egress_proxy/notify-api-demo.allow.acl
+++ b/deploy-config/egress_proxy/notify-api-demo.allow.acl
@@ -1,4 +1,4 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
 gov-collector.newrelic.com
-*.apps.internal
+egress-proxy-notify-api-demo.apps.internal
diff --git a/deploy-config/egress_proxy/notify-api-staging.allow.acl b/deploy-config/egress_proxy/notify-api-staging.allow.acl
index 7061eef91..9f16b1edf 100644
--- a/deploy-config/egress_proxy/notify-api-staging.allow.acl
+++ b/deploy-config/egress_proxy/notify-api-staging.allow.acl
@@ -1,4 +1,4 @@
 email.us-west-2.amazonaws.com
 sns.us-west-2.amazonaws.com
 gov-collector.newrelic.com
-*.apps.internal
+egress-proxy-notify-api-staging.apps.internal

From d84393a242c191c054af399362597adab4ba0437 Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Fri, 27 Jan 2023 14:27:22 -0500
Subject: [PATCH 44/46] Tweak tail & jq usage to parse cf-cli@8 output

---
 terraform/create_service_account.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/create_service_account.sh b/terraform/create_service_account.sh
index d69a90796..0ac5b0844 100755
--- a/terraform/create_service_account.sh
+++ b/terraform/create_service_account.sh
@@ -67,7 +67,7 @@ cf create-service cloud-gov-service-account $role $service 1>&2
 cf create-service-key $service service-account-key 1>&2
 
 # output service key to stdout in secrets.auto.tfvars format
-creds=`cf service-key $service service-account-key | tail -n 4`
+creds=`cf service-key $service service-account-key | tail -n +2 | jq '.credentials'`
 username=`echo $creds | jq -r '.username'`
 password=`echo $creds | jq -r '.password'`
 

From a1ea9281a27bd48b09212e0808c4fa950c57846d Mon Sep 17 00:00:00 2001
From: Ryan Ahearn <ryan.ahearn@gsa.gov>
Date: Fri, 27 Jan 2023 14:54:14 -0500
Subject: [PATCH 45/46] Add cf-cli version check to create_service_account

---
 terraform/create_service_account.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/terraform/create_service_account.sh b/terraform/create_service_account.sh
index 0ac5b0844..cee07591e 100755
--- a/terraform/create_service_account.sh
+++ b/terraform/create_service_account.sh
@@ -18,9 +18,16 @@ Options:
 -o <ORG NAME>: configure the organization to act on. Default: $org
 
 Notes:
-OrgManager is required for terraform to create <env>-egress spaces
+* OrgManager is required for terraform to create <env>-egress spaces
+* Requires cf-cli@8
 "
 
+cf_version=`cf --version | cut -d " " -f 3`
+if [[ $cf_version != 8.* ]]; then
+  echo "$usage"
+  exit 1
+fi
+
 set -e
 set -o pipefail
 

From dc06b411ca1b2b60fc75dfa184603c43173fb6dd Mon Sep 17 00:00:00 2001
From: Steven Reilly <stvnrlly@users.noreply.github.com>
Date: Fri, 3 Feb 2023 10:11:21 -0500
Subject: [PATCH 46/46] Update default templates and add update command (#166)

This commit adds config_files for default data and, using that, creates a new way to update our default templates without needing to hardcode a migration.

---------

Co-authored-by: Ryan Ahearn <ryan.ahearn@gsa.gov>
---
 app/commands.py                               |  27 ++
 app/config.py                                 |   5 +-
 app/config_files/templates.json               | 274 ++++++++++++++++++
 migrations/alembic.ini                        |   4 +-
 .../versions/0383_update_default_templates.py |  55 ++++
 tests/app/test_commands.py                    |  20 +-
 6 files changed, 381 insertions(+), 4 deletions(-)
 create mode 100644 app/config_files/templates.json
 create mode 100644 migrations/versions/0383_update_default_templates.py

diff --git a/app/commands.py b/app/commands.py
index 1718ea82d..5f0ffb91b 100644
--- a/app/commands.py
+++ b/app/commands.py
@@ -65,6 +65,8 @@ from app.models import (
     Notification,
     Organisation,
     Service,
+    Template,
+    TemplateHistory,
     User,
 )
 from app.utils import get_local_midnight_in_utc
@@ -765,3 +767,28 @@ def create_user_jwt(token):
     service_id = token[-73:-37]
     api_key = token[-36:]
     print(create_jwt_token(api_key, service_id))
+
+
+def _update_template(id, name, template_type, content, subject):
+
+    template = Template.query.filter_by(id=id).first()
+    template.name = name
+    template.template_type = template_type
+    template.content = '\n'.join(content)
+    template.subject = subject
+
+    history = TemplateHistory.query.filter_by(id=id).first()
+    history.name = name
+    history.template_type = template_type
+    history.content = '\n'.join(content)
+    history.subject = subject
+
+    db.session.commit()
+
+
+@notify_command(name='update-templates')
+def update_templates():
+    with open(current_app.config['CONFIG_FILES'] + '/templates.json') as f:
+        data = json.load(f)
+        for d in data:
+            _update_template(d['id'], d['name'], d['type'], d['content'], d['subject'])
diff --git a/app/config.py b/app/config.py
index 10d4220c8..039da6a43 100644
--- a/app/config.py
+++ b/app/config.py
@@ -1,6 +1,6 @@
 import json
 from datetime import timedelta
-from os import getenv
+from os import getenv, path
 
 from celery.schedules import crontab
 from kombu import Exchange, Queue
@@ -147,6 +147,9 @@ class Config(object):
     MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024  # 40mb
     MAX_LETTER_PDF_COUNT_PER_ZIP = 500
 
+    # Default data
+    CONFIG_FILES = path.dirname(__file__) + '/config_files/'
+
     NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
     NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6'
     INVITATION_EMAIL_TEMPLATE_ID = '4f46df42-f795-4cc4-83bb-65ca312f49cc'
diff --git a/app/config_files/templates.json b/app/config_files/templates.json
new file mode 100644
index 000000000..ae208541b
--- /dev/null
+++ b/app/config_files/templates.json
@@ -0,0 +1,274 @@
+[
+    {
+        "id": "42a23d19-504e-49bb-a95e-4976baff4757",
+        "name": "Example text message template",
+        "type": "sms",
+        "subject":  "",
+        "content": ["Hi, I’m trying out U.S. Notify. Today is ((day of week)) and my favorite color is ((color))."]
+    },
+    {
+        "id": "4f46df42-f795-4cc4-83bb-65ca312f49cc",
+        "name": "Notify invitation email",
+        "type": "email",
+        "subject":  "((user_name)) has invited you to collaborate on ((service_name)) on U.S. Notify",
+        "content": ["((user_name)) has invited you to collaborate on ((service_name)) on U.S. Notify.",
+            "",
+            "",
+            "U.S. Notify makes it easy to keep people updated by helping you send text messages, emails and letters.",
+            "",
+            "",
+            "Click this link to create an account on U.S. Notify:",
+            "",
+            "((url))",
+            "",
+            "",
+            "This invitation will stop working at midnight tomorrow. This is to keep ((service_name)) secure."]
+    },
+    {
+        "id": "36fb0730-6259-4da1-8a80-c8de22ad4246",
+        "name": "Notify SMS verify code",
+        "type": "sms",
+        "subject": "",
+        "content": ["((verify_code)) is your U.S. Notify authentication code"]
+    },
+    {
+        "id": "474e9242-823b-4f99-813d-ed392e7f1201",
+        "name": "Notify password reset email",
+        "type": "email",
+        "subject": "Reset your U.S. Notify password",
+        "content": ["Hi ((user_name)),",
+            "",
+            "",
+            "We received a request to reset your password on U.S. Notify.",
+            "",
+            "",
+            "If you didn’t request this email, you can ignore it – your password has not been changed.",
+            "",
+            "",
+            "To reset your password, click this link:","","","((url))"]
+    },
+    {
+        "id": "299726d2-dba6-42b8-8209-30e1d66ea164",
+        "name": "Notify email verify code",
+        "type": "email",
+        "subject": "Sign in to U.S. Notify",
+        "content": ["Hi ((name)),",
+        "",
+        "",
+        "To sign in to U.S. Notify please open this link:",
+        "",
+        "",
+        "((url))"]
+    },
+    {
+        "id": "ece42649-22a8-4d06-b87f-d52d5d3f0a27",
+        "name": "Notify email verification code",
+        "type": "email",
+        "subject": "Confirm U.S. Notify registration",
+        "content": ["Hi ((name)),",
+        "",
+        "",
+        "To complete your registration for U.S. Notify please click the link below",
+        "",
+        "",
+        "((url))"]
+    },
+    {
+        "id": "0880fbb1-a0c6-46f0-9a8e-36c986381ceb",
+        "name": "Your U.S. Notify account",
+        "type": "email",
+        "subject": "Your U.S. Notify account",
+        "content": ["You already have a U.S. Notify account with this email address.",
+        "",
+        "",
+        "Sign in here: ((signin_url))",
+        "",
+        "",
+        "If you’ve forgotten your password, you can reset it here: ((forgot_password_url))",
+        "",
+        "",
+        "",
+        "",
+        "If you didn’t try to register for a U.S. Notify account recently, please let us know here: ((feedback_url))"]
+    },
+    {
+        "id": "eb4d9930-87ab-4aef-9bce-786762687884",
+        "name": "Confirm new email address",
+        "type": "email",
+        "subject": "Confirm your email address for U.S. Notify",
+        "content": ["Hi ((name)),","","","Click this link to confirm your new email address:",
+            "",
+            "",
+            "((url))",
+            "",
+            "",
+            "If you didn’t try to change the email address for your U.S. Notify account, let us know here:",
+            "",
+            "",
+            "((feedback_url))"]
+    },
+    {
+        "id": "618185c6-3636-49cd-b7d2-6f6f5eb3bdde",
+        "name": "Automated \"You’re now live\" message",
+        "type": "email",
+        "subject": "((service name)) is now live on U.S. Notify",
+        "content": ["Hi ((name)),",
+        "",
+        "",
+        "((service name)) is now live on U.S. Notify.",
+        "",
+        "",
+        "You can send up to ((message limit)) messages per day.",
+        "",
+        "",
+        "As a live service, you’ll need to know who to contact if you have a question, or something goes wrong.",
+        "",
+        "",
+        "If our system status page shows a problem, then we’ve been alerted and are working on it – you don’t need to contact us.",
+        "",
+        "",
+        "#Problems or questions out of hours",
+        "",
+        "",
+        "We offer out of hours support for emergencies.",
+        "",
+        "",
+        "It’s only an emergency if:",
+        "",
+        "*   no one in your team can log in",
+        "",
+        "*   a ‘technical difficulties’ error appears when you try to upload a file",
+        "",
+        "*   a 500 response code appears when you try to send messages using the API",
+        "",
+        "",
+        "If you have one of these emergencies, email details to:",
+        "",
+        "notify-support@gsa.gov",
+        "",
+        "",
+        "^Only use this email address for out of hours emergencies. Don’t share this address with people outside of your team.",
+        "",
+        "",
+        "We’ll get back to you within 30 minutes and give you hourly updates until the problem’s fixed.",
+        "",
+        "",
+        "For non-emergency problems or questions, use our support page and we’ll reply in office hours.",
+        "",
+        "",
+        "Thanks",
+        "",
+        "U.S. Notify team"]
+    },
+    {
+        "id": "203566f0-d835-47c5-aa06-932439c86573",
+        "name": "Notify organization invitation email",
+        "type": "email",
+        "subject": "((user_name)) has invited you to collaborate on ((organisation_name)) on U.S. Notify",
+        "content": ["((user_name)) has invited you to collaborate on ((organisation_name)) on U.S. Notify.","","","U.S. Notify makes it easy to keep people updated by helping you send text messages, emails and letters.","","","Open this link to create an account on U.S. Notify:","","((url))","","","This invitation will stop working at midnight tomorrow. This is to keep ((organisation_name)) secure."]
+    },
+    {
+        "id": "c73f1d71-4049-46d5-a647-d013bdeca3f0",
+        "name": "Email address changed by service manager",
+        "type": "email",
+        "subject": "Your U.S. Notify email address has changed",
+        "content":  ["Dear ((name)),","","","((servicemanagername)) changed your Notify account email address to:","","","((email address))","","","You’ll need to use this email address next time you sign in.","","","Thanks","","","U.S. Notify team"]
+    },
+    {
+        "id": "8a31520f-4751-4789-8ea1-fe54496725eb",
+        "name": "Phone number changed by service manager",
+        "type": "sms",
+        "subject": "",
+        "content": ["Your mobile number was changed by ((servicemanagername)). Next time you sign in, your U.S. Notify authentication code will be sent to this phone."]
+    },
+    {
+        "id": "a42f1d17-9404-46d5-a647-d013bdfca3e1",
+        "name": "Verify email reply-to address for a service",
+        "type": "email",
+        "subject": "Your U.S. Notify reply-to email address",
+        "content": ["Hi,","","","This address has been provided as a reply-to email address for a U.S. Notify account.","","Any replies from users to emails they receive through U.S. Notify will come back to this email address.","","","This is just a quick check to make sure the address is valid.","","","No need to reply.","","","Thanks","","","U.S. Notify team"]
+    },
+    {
+        "id": "4fd2e43c-309b-4e50-8fb8-1955852d9d71",
+        "name": "MOU Signed By Receipt",
+        "type": "email",
+        "subject": "You’ve accepted the U.S. Notify data sharing and financial agreement",
+        "content": [
+            "Hi ((signed_by_name)),",
+            "",
+            "((org_name)) has accepted the U.S. Notify data sharing and financial agreement. ",
+            "",
+            "If you need another copy of the agreement you can download it here: ((mou_link))",
+            "",
+            "",
+            "Thanks,",
+            "U.S. Notify team"
+        ]
+    },
+    {
+        "id": "c20206d5-bf03-4002-9a90-37d5032d9e84",
+        "name": "MOU Signed On Behalf Of Receipt - Signed by",
+        "type": "email",
+        "subject": "You’ve accepted the U.S. Notify data sharing and financial agreement",
+        "content": [
+            "Hi ((signed_by_name)),",
+            "",
+            "((org_name)) has accepted the U.S. Notify data sharing and financial agreement. We’ve emailed ((on_behalf_of_name)) to let them know too.",
+            "",
+            "If you need another copy of the agreement you can download it here: ((mou_link))",
+            "",
+            "",
+            "Thanks,",
+            "U.S. Notify team"
+        ]
+    },
+    {
+        "id": "522b6657-5ca5-4368-a294-6b527703bd0b",
+        "name": "MOU Signed On Behalf Of Receipt - On Behalf Of",
+        "type": "email",
+        "subject": "((org_name)) has accepted the U.S. Notify data sharing and financial agreement",
+        "content": [
+            "Hi ((on_behalf_of_name)),",
+            "",
+            "((signed_by_name)) has accepted the U.S. Notify data sharing and financial agreement on your behalf, for ((org_name)).",
+            "",
+            "U.S. Notify lets teams in the public sector send emails, text messages and letters. It’s built and run by a team in the TTS Public Benefits Studio (part of GSA).",
+            "",
+            "If you need another copy of the agreement you can download it here: ((mou_link))",
+            "",
+            "",
+            "Thanks,",
+            "U.S. Notify team"
+        ]
+    },
+    {
+        "id": "d0e66c4c-0c50-43f0-94f5-f85b613202d4",
+        "name": "MOU Signed Notify Team Alert",
+        "type": "email",
+        "subject": "Someone signed an MOU for an org on Notify",
+        "content": [
+            "What’s up Notifiers,",
+            "",
+            "((signed_by_name)) just accepted the data sharing and financial agreement for ((org_name)).",
+            "",
+            "See how ((org_name)) is using Notify here: ((org_dashboard_link))"
+        ]
+    },
+    {
+        "id": "11fad854-fd38-4a7c-bd17-805fb13dfc12",
+        "name": "Notify daily letter volumes",
+        "type": "email",
+        "subject": "Notify letter volume for ((date)): ((total_volume)) letters, ((total_sheets)) sheets",
+        "content": [
+            "((total_volume)) letters (((total_sheets)) sheets) sent via Notify are coming in today’s batch. These include: ",
+            "",
+            "((first_class_volume)) first class letters (((first_class_sheets)) sheets).",
+            "((second_class_volume)) second class letters (((second_class_sheets)) sheets).",
+            "((international_volume)) international letters (((international_sheets)) sheets).",
+            "",
+            "Thanks",
+            "",
+            "U.S. Notify team"
+        ]
+    }
+]
diff --git a/migrations/alembic.ini b/migrations/alembic.ini
index 8c9d61375..199797e0a 100644
--- a/migrations/alembic.ini
+++ b/migrations/alembic.ini
@@ -20,12 +20,12 @@ keys = console
 keys = generic
 
 [logger_root]
-level = WARN
+level = INFO
 handlers =
 qualname =
 
 [logger_sqlalchemy]
-level = WARN
+level = INFO
 handlers =
 qualname = sqlalchemy.engine
 
diff --git a/migrations/versions/0383_update_default_templates.py b/migrations/versions/0383_update_default_templates.py
new file mode 100644
index 000000000..3313abfa4
--- /dev/null
+++ b/migrations/versions/0383_update_default_templates.py
@@ -0,0 +1,55 @@
+"""
+
+Revision ID: 0383_update_default_templates.py
+Revises: 0382_remove_old_providers
+Create Date: 2023-01-10 11:42:25.633265
+
+"""
+import json
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from flask import current_app
+
+revision = '0383_update_default_templates.py'
+down_revision = '0382_remove_old_providers'
+
+
+def upgrade():
+    update = """
+        UPDATE {} SET name = '{}', template_type = '{}', content = '{}', subject = '{}'
+        WHERE id = '{}'
+    """
+
+    with open(current_app.config['CONFIG_FILES'] + '/templates.json') as f:
+        data = json.load(f)
+        for d in data:
+            for table_name in 'templates', 'templates_history':
+                op.execute(
+                    update.format(
+                        table_name,
+                        d['name'],
+                        d['type'],
+                        '\n'.join(d['content']),
+                        d.get('subject'),
+                        d['id']
+                    )
+                )
+
+            # op.execute(
+            #     """
+            #     INSERT INTO template_redacted
+            #     (
+            #         template_id,
+            #         redact_personalisation,
+            #         updated_at,
+            #         updated_by_id
+            #     ) VALUES ( '{}', false, current_timestamp, '{}' )
+            #     """.format(d['id'], current_app.config['NOTIFY_USER_ID'])
+            # )
+   
+
+def downgrade():
+    # with associated code changes, edits to templates should no longer be made via migration.
+    # instead, update the fixture and run the flask command to update.
+    pass
diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py
index 6f50f30f7..fc6c85695 100644
--- a/tests/app/test_commands.py
+++ b/tests/app/test_commands.py
@@ -1,12 +1,13 @@
 import pytest
 
 from app.commands import (
+    _update_template,
     create_test_user,
     insert_inbound_numbers_from_file,
     populate_annual_billing_with_defaults,
 )
 from app.dao.inbound_numbers_dao import dao_get_available_inbound_numbers
-from app.models import AnnualBilling, User
+from app.models import AnnualBilling, Template, User
 from tests.app.db import create_annual_billing, create_service
 
 
@@ -88,3 +89,20 @@ def test_populate_annual_billing_with_defaults_sets_free_allowance_to_zero_if_pr
 
     assert len(results) == 1
     assert results[0].free_sms_fragment_limit == 0
+
+
+def test_update_template(
+    notify_db_session, email_2fa_code_template
+):
+
+    _update_template(
+        "299726d2-dba6-42b8-8209-30e1d66ea164",
+        "Example text message template!",
+        "sms",
+        ["Hi, I’m trying out U.S. Notify! Today is ((day of week)) and my favorite color is ((color))."],
+        ""
+    )
+
+    t = Template.query.all()
+
+    assert t[0].name == "Example text message template!"