Merge branch 'main' of https://github.com/GSA/notifications-api into notify-260

Pipfile.lock

@@ -500,6 +500,7 @@
                 "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
                 "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
             ],
+            "markers": "python_full_version >= '3.5.0'",
             "version": "==3.4"
         },
         "importlib-metadata": {
@@ -847,7 +848,7 @@
                 "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b",
                 "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==3.0.38"
         },
         "psycopg2-binary": {
@@ -1062,11 +1063,11 @@
         },
         "requests": {
             "hashes": [
-                "sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294",
+                "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
-                "sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4"
+                "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
             ],
-            "markers": "python_version >= '3.7'",
+            "index": "pypi",
-            "version": "==2.30.0"
+            "version": "==2.31.0"
         },
         "rfc3339-validator": {
             "hashes": [
@@ -1087,7 +1088,7 @@
                 "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2",
                 "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9"
             ],
-            "markers": "python_version >= '3.5' and python_version < '4'",
+            "markers": "python_version < '4' and python_full_version >= '3.5.0'",
             "version": "==4.7.2"
         },
         "s3transfer": {
@@ -1100,11 +1101,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b",
+                "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f",
-                "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"
+                "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==67.7.2"
+            "version": "==67.8.0"
         },
         "shapely": {
             "hashes": [
@@ -1216,11 +1217,11 @@
         },
         "typing-extensions": {
             "hashes": [
-                "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb",
+                "sha256:6ad00b63f849b7dcc313b70b6b304ed67b2b2963b3098a33efe18056b1a9a223",
-                "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"
+                "sha256:ff6b238610c747e44c268aa4bb23c8c735d665a63726df3f9431ce707f2aa768"
             ],
             "markers": "python_version < '3.10'",
-            "version": "==4.5.0"
+            "version": "==4.6.0"
         },
         "uri-template": {
             "hashes": [
@@ -1847,6 +1848,7 @@
                 "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
                 "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
             ],
+            "markers": "python_full_version >= '3.5.0'",
             "version": "==3.4"
         },
         "iniconfig": {
@@ -2194,7 +2196,7 @@
                 "sha256:4659bc2a667783e7a15d190f6fccf8b2486685b6dba4c19c3876314769c57526",
                 "sha256:b4fa3a7a0be38243123cf9d1f3518da10c51bdb165a2b2985566247f9155a7d3"
             ],
-            "markers": "python_full_version >= '3.6.0'",
+            "markers": "python_version >= '3.6'",
             "version": "==32.0.1"
         },
         "pluggy": {
@@ -2214,22 +2216,22 @@
         },
         "protobuf": {
             "hashes": [
-                "sha256:03eee35b60317112a72d19c54d0bff7bc58ff12fea4cd7b018232bd99758ffdf",
+                "sha256:2036a3a1e7fc27f973fa0a7888dce712393af644f4695385f117886abc792e39",
-                "sha256:2b94bd6df92d71bd1234a2ffe7ce96ddf6d10cf637a18d6b55ad0a89fbb7fc21",
+                "sha256:32e78beda26d7a101fecf15d7a4a792278a0d26a31bc327ff05564a9d68ab8ee",
-                "sha256:36f5370a930cb77c8ad2f4135590c672d0d2c72d4a707c7d0058dce4b4b4a598",
+                "sha256:346990f634272caac1f09efbcfbbacb23098b1f606d172534c6fa2d9758bb436",
-                "sha256:5f1eba1da2a2f3f7df469fccddef3cc060b8a16cfe3cc65961ad36b4dbcf59c5",
+                "sha256:3b8905eafe4439076e1f58e9d1fa327025fd2777cf90f14083092ae47f77b0aa",
-                "sha256:6c16657d6717a0c62d5d740cb354fbad1b0d8cb811669e06fc1caa0ff4799ddd",
+                "sha256:3ce113b3f3362493bddc9069c2163a38f240a9ed685ff83e7bcb756b05e1deb0",
-                "sha256:6fe180b56e1169d72ecc4acbd39186339aed20af5384531b8e8979b02bbee159",
+                "sha256:410bcc0a5b279f634d3e16082ce221dfef7c3392fac723500e2e64d1806dd2be",
-                "sha256:7cb5b9a05ce52c6a782bb97de52679bd3438ff2b7460eff5da348db65650f227",
+                "sha256:5b9cd6097e6acae48a68cb29b56bc79339be84eca65b486910bb1e7a30e2b7c1",
-                "sha256:9744e934ea5855d12191040ea198eaf704ac78665d365a89d9572e3b627c2688",
+                "sha256:65f0ac96ef67d7dd09b19a46aad81a851b6f85f89725577f16de38f2d68ad477",
-                "sha256:9f5a0fbfcdcc364f3986f9ed9f8bb1328fb84114fd790423ff3d7fdb0f85c2d1",
+                "sha256:91fac0753c3c4951fbb98a93271c43cc7cf3b93cf67747b3e600bb1e5cc14d61",
-                "sha256:baca40d067dddd62141a129f244703160d278648b569e90bb0e3753067644711",
+                "sha256:95789b569418a3e32a53f43d7763be3d490a831e9c08042539462b6d972c2d7e",
-                "sha256:d5a35ff54e3f62e8fc7be02bb0d2fbc212bba1a5a9cc2748090690093996f07b",
+                "sha256:ac50be82491369a9ec3710565777e4da87c6d2e20404e0abb1f3a8f10ffd20f0",
-                "sha256:e62fb869762b4ba18666370e2f8a18f17f8ab92dd4467295c6d38be6f8fef60b",
+                "sha256:decf119d54e820f298ee6d89c72d6b289ea240c32c521f00433f9dc420595f38",
-                "sha256:ebde3a023b8e11bfa6c890ef34cd6a8b47d586f26135e86c21344fe433daf2e2"
+                "sha256:f9510cac91e764e86acd74e2b7f7bc5e6127a7f3fb646d7c8033cfb84fd1176a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.23.0"
+            "version": "==4.23.1"
         },
         "py": {
             "hashes": [
@@ -2385,11 +2387,11 @@
         },
         "requests": {
             "hashes": [
-                "sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294",
+                "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
-                "sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4"
+                "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
             ],
-            "markers": "python_version >= '3.7'",
+            "index": "pypi",
-            "version": "==2.30.0"
+            "version": "==2.31.0"
         },
         "requests-mock": {
             "hashes": [
@@ -2419,7 +2421,7 @@
                 "sha256:2d11b9b8dd03868f09b4fffadc84a6a8cda574e40dc90821bd845720ebb8e89c",
                 "sha256:69cdf53799e63f38b95b9bf9c875f8c90e78dd62b2f00c13a911c7a3b9fa4704"
             ],
-            "markers": "python_full_version >= '3.7.0'",
+            "markers": "python_version >= '3.7'",
             "version": "==13.3.5"
         },
         "s3transfer": {
@@ -2455,11 +2457,11 @@
         },
         "stevedore": {
             "hashes": [
-                "sha256:2c428d2338976279e8eb2196f7a94910960d9f7ba2f41f3988511e95ca447021",
+                "sha256:8cc040628f3cea5d7128f2e76cf486b2251a4e543c7b938f58d9a377f6694a2d",
-                "sha256:bd5a71ff5e5e5f5ea983880e4a1dd1bb47f8feebbb3d95b592398e2f02194771"
+                "sha256:a54534acf9b89bc7ed264807013b505bf07f74dbe4bcfa37d32bd063870b087c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.0.0"
+            "version": "==5.1.0"
         },
         "toml": {
             "hashes": [
@@ -2479,10 +2481,10 @@
         },
         "types-pyyaml": {
             "hashes": [
-                "sha256:5aed5aa66bd2d2e158f75dda22b059570ede988559f030cf294871d3b647e3e8",
+                "sha256:662fa444963eff9b68120d70cda1af5a5f2aa57900003c2006d7626450eaae5f",
-                "sha256:c51b1bd6d99ddf0aa2884a7a328810ebf70a4262c292195d3f4f9a0005f9eeb6"
+                "sha256:ebab3d0700b946553724ae6ca636ea932c1b0868701d4af121630e78d695fc97"
             ],
-            "version": "==6.0.12.9"
+            "version": "==6.0.12.10"
         },
         "urllib3": {
             "hashes": [
@@ -2501,11 +2503,11 @@
         },
         "websocket-client": {
             "hashes": [
-                "sha256:3f09e6d8230892547132177f575a4e3e73cfdf06526e20cc02aa1c3b47184d40",
+                "sha256:c7d67c13b928645f259d9b847ab5b57fd2d127213ca41ebd880de1f553b7c23b",
-                "sha256:cdf5877568b7e83aa7cf2244ab56a3213de587bbe0ce9d8b9600fc77b455d89e"
+                "sha256:f8c64e28cd700e7ba1f04350d66422b6833b82a796b525a51e740b8cc8dab4b1"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.5.1"
+            "version": "==1.5.2"
         },
         "werkzeug": {
             "hashes": [

app/aws/s3.py

@@ -65,3 +65,14 @@ def remove_job_from_s3(service_id, job_id):
 def remove_s3_object(bucket_name, object_key, access_key, secret_key, region):
     obj = get_s3_object(bucket_name, object_key, access_key, secret_key, region)
     return obj.delete()
+
+
+def remove_csv_object(object_key):
+    obj = get_s3_object(
+        current_app.config['CSV_UPLOAD_BUCKET']['bucket'],
+        object_key,
+        current_app.config['CSV_UPLOAD_BUCKET']['access_key_id'],
+        current_app.config['CSV_UPLOAD_BUCKET']['secret_access_key'],
+        current_app.config['CSV_UPLOAD_BUCKET']['region']
+    )
+    return obj.delete()

app/celery/nightly_tasks.py

@@ -5,6 +5,7 @@ from sqlalchemy.exc import SQLAlchemyError
 
 from app import notify_celery
 from app.aws import s3
+from app.aws.s3 import remove_csv_object
 from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
 from app.config import QueueNames
 from app.cronitor import cronitor
@@ -13,6 +14,7 @@ from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
     dao_archive_job,
     dao_get_jobs_older_than_data_retention,
+    dao_get_unfinished_jobs,
 )
 from app.dao.notifications_dao import (
     dao_get_notifications_processing_time_stats,
@@ -41,6 +43,19 @@ def _remove_csv_files(job_types):
         current_app.logger.info("Job ID {} has been removed from s3.".format(job.id))
 
 
+@notify_celery.task(name="cleanup-unfinished-jobs")
+def cleanup_unfinished_jobs():
+    now = datetime.utcnow()
+    jobs = dao_get_unfinished_jobs()
+    for job in jobs:
+        # The query already checks that the processing_finished time is null, so here we are saying
+        # if it started more than 4 hours ago, that's too long
+        acceptable_finish_time = job.processing_started + timedelta(minutes=5)
+        if now > acceptable_finish_time:
+            remove_csv_object(job.original_file_name)
+            dao_archive_job(job)
+
+
 @notify_celery.task(name="delete-notifications-older-than-retention")
 def delete_notifications_older_than_retention():
     delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING)
@@ -158,6 +173,7 @@ def delete_inbound_sms():
 @notify_celery.task(name='save-daily-notification-processing-time')
 @cronitor("save-daily-notification-processing-time")
 def save_daily_notification_processing_time(local_date=None):
+
     # local_date is a string in the format of "YYYY-MM-DD"
     if local_date is None:
         # if a date is not provided, we run against yesterdays data

app/celery/provider_tasks.py

@@ -11,7 +11,10 @@ from app.clients.email.aws_ses import AwsSesClientThrottlingSendRateException
 from app.clients.sms import SmsClientResponseException
 from app.config import QueueNames
 from app.dao import notifications_dao
-from app.dao.notifications_dao import update_notification_status_by_id
+from app.dao.notifications_dao import (
+    insert_notification_history_delete_notifications_by_id,
+    update_notification_status_by_id,
+)
 from app.delivery import send_to_providers
 from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
@@ -40,6 +43,10 @@ def check_sms_delivery_receipt(self, message_id, notification_id, sent_at):
     update_notification_status_by_id(notification_id, status, provider_response=provider_response)
     current_app.logger.info(f"Updated notification {notification_id} with response '{provider_response}'")
 
+    if status == NOTIFICATION_SENT:
+        insert_notification_history_delete_notifications_by_id(notification_id)
+        current_app.logger.info(f"Archived notification {notification_id} that was successfully sent")
+
 
 @notify_celery.task(bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300)
 def deliver_sms(self, notification_id):

app/celery/research_mode_tasks.py

@@ -5,6 +5,7 @@ from requests import HTTPError, request
 
 from app.celery.process_ses_receipts_tasks import process_ses_results
 from app.config import QueueNames
+from app.dao.notifications_dao import get_notification_by_id
 from app.models import SMS_TYPE
 
 temp_fail = "2028675303"
@@ -16,8 +17,8 @@ perm_fail_email = "perm-fail@simulator.notify"
 temp_fail_email = "temp-fail@simulator.notify"
 
 
-def send_sms_response(provider, reference, to):
+def send_sms_response(provider, reference):
-    body = sns_callback(reference, to)
+    body = sns_callback(reference)
     headers = {"Content-type": "application/json"}
 
     make_request(SMS_TYPE, provider, body, headers)
@@ -59,25 +60,16 @@ def make_request(notification_type, provider, data, headers):
     return response.json()
 
 
-def sns_callback(notification_id, to):
+def sns_callback(notification_id):
-    raise Exception("Need to update for SNS callback format along with test_send_to_providers")
+    notification = get_notification_by_id(notification_id)
 
-    # example from mmg_callback
+    # This will only work if all notifications, including successful ones, are in the notifications table
-    # if to.strip().endswith(temp_fail):
+    # If we decide to delete successful notifications, we will have to get this from notifications history
-    #     # status: 4 - expired (temp failure)
+    return json.dumps({
-    #     status = "4"
+        "CID": str(notification_id),
-    # elif to.strip().endswith(perm_fail):
+        "status": notification.status,
-    #     # status: 5 - rejected (perm failure)
+        # "deliverytime": notification.completed_at
-    #     status = "5"
+    })
-    # else:
-    #     # status: 3 - delivered
-    #     status = "3"
-
-    # return json.dumps({"reference": "mmg_reference",
-    #                    "CID": str(notification_id),
-    #                    "MSISDN": to,
-    #                    "status": status,
-    #                    "deliverytime": "2016-04-05 16:01:07"})
 
 
 def ses_notification_callback(reference):

app/commands.py

@@ -18,6 +18,7 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from app import db
 from app.aws import s3
+from app.celery.nightly_tasks import cleanup_unfinished_jobs
 from app.celery.tasks import process_row
 from app.dao.annual_billing_dao import (
     dao_create_or_update_annual_billing_for_year,
@@ -464,6 +465,12 @@ def fix_billable_units():
     print("End fix_billable_units")
 
 
+@notify_command(name='delete-unfinished-jobs')
+def delete_unfinished_jobs():
+    cleanup_unfinished_jobs()
+    print("End cleanup_unfinished_jobs")
+
+
 @notify_command(name='process-row-from-job')
 @click.option('-j', '--job_id', required=True, help='Job id')
 @click.option('-n', '--job_row_number', type=int, required=True, help='Job id')

app/config.py

@@ -240,6 +240,11 @@ class Config(object):
                 'schedule': crontab(hour=2, minute=0),
                 'options': {'queue': QueueNames.PERIODIC}
             },
+            'cleanup-unfinished-jobs': {
+                'task': 'cleanup-unfinished-jobs',
+                'schedule': crontab(hour=0, minute=5),
+                'options': {'queue': QueueNames.PERIODIC}
+            },
             'remove_sms_email_jobs': {
                 'task': 'remove_sms_email_jobs',
                 'schedule': crontab(hour=4, minute=0),

app/dao/jobs_dao.py

@@ -43,6 +43,10 @@ def dao_get_job_by_service_id_and_job_id(service_id, job_id):
     return Job.query.filter_by(service_id=service_id, id=job_id).one()
 
 
+def dao_get_unfinished_jobs():
+    return Job.query.filter(Job.processing_finished.is_(None)).all()
+
+
 def dao_get_jobs_by_service_id(
     service_id,
     *,

app/dao/notifications_dao.py

@@ -271,6 +271,37 @@ def _filter_query(query, filter_dict=None):
     return query
 
 
+@autocommit
+def insert_notification_history_delete_notifications_by_id(
+    notification_id
+):
+    """
+    Deletes one notification after it has run successfully and moves it to the notification_history
+    table.
+    """
+    input_params = {
+        "notification_id": notification_id
+    }
+    # Insert into NotificationHistory if the row already exists do nothing.
+    insert_query = """
+        insert into notification_history
+         SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id,
+             key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units,
+             client_reference, international, phone_prefix, rate_multiplier, notification_status,
+              created_by_id, document_download_count
+          from NOTIFICATIONS WHERE id= :notification_id
+          ON CONFLICT ON CONSTRAINT notification_history_pkey
+          DO NOTHING
+    """
+    delete_query = """
+        DELETE FROM notifications
+        where id= :notification_id
+    """
+
+    db.session.execute(insert_query, input_params)
+    db.session.execute(delete_query, input_params)
+
+
 @autocommit
 def insert_notification_history_delete_notifications(
     notification_type, service_id, timestamp_to_delete_backwards_from, qry_limit=50000

app/delivery/send_to_providers.py

@@ -61,7 +61,7 @@ def send_sms_to_provider(notification):
         )
         if service.research_mode or notification.key_type == KEY_TYPE_TEST:
             update_notification_to_sending(notification, provider)
-            send_sms_response(provider.name, str(notification.id), notification.to)
+            send_sms_response(provider.name, str(notification.id))
 
         else:
             try:

app/models.py

@@ -108,7 +108,7 @@ class User(db.Model):
     platform_admin = db.Column(db.Boolean, nullable=False, default=False)
     current_session_id = db.Column(UUID(as_uuid=True), nullable=True)
     auth_type = db.Column(
-        db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=EMAIL_AUTH_TYPE
+        db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=SMS_AUTH_TYPE
     )
     email_access_validated_at = db.Column(
         db.DateTime, index=False, unique=False, nullable=False, default=datetime.datetime.utcnow
@@ -1651,7 +1651,7 @@ class InvitedUser(db.Model):
         db.ForeignKey('auth_type.name'),
         index=True,
         nullable=False,
-        default=EMAIL_AUTH_TYPE
+        default=SMS_AUTH_TYPE
     )
     folder_permissions = db.Column(JSONB(none_as_null=True), nullable=False, default=[])
 

docs/infra-overview.md

@@ -117,8 +117,8 @@ Steps for deploying production from scratch. These can be updated for a new clou
     1. Update `terraform-production.yml` and `deploy-prod.yml` to point to the correct space and git branch.
     1. Ensure that the `api_network_route` and `domain` modules are commented out in `terraform/production/main.tf`
     1. Run CI/CD pipeline on the `production` branch by opening a PR from `main` to `production`
-    1. Uncomment the `api_network_route` and `domain` modules and re-trigger a deploy
     1. Create DNS records for `domain` module within https://github.com/18f/dns
+    1. Uncomment the `api_network_route` and `domain` modules and re-trigger a deploy
 
 ### Steps to prepare SES
 

migrations/versions/0395_remove_intl_letters_perm.py

@@ -0,0 +1,28 @@
+"""
+
+Revision ID: 0395_remove_international_letters_permission
+Revises: 0394_remove_contact_list
+Create Date: 2023-05-23 10:03:10.485368
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision = '0395_remove_intl_letters_perm'
+down_revision = '0394_remove_contact_list'
+
+
+def upgrade():
+    sql = """
+        DELETE
+        FROM service_permissions
+        WHERE permission = 'international_letters'
+    """
+
+    conn = op.get_bind()
+    conn.execute(sql)
+
+
+def downgrade():
+    pass

tests/app/celery/test_nightly_tasks.py

@@ -7,6 +7,7 @@ from freezegun import freeze_time
 from app.celery import nightly_tasks
 from app.celery.nightly_tasks import (
     _delete_notifications_older_than_retention_by_type,
+    cleanup_unfinished_jobs,
     delete_email_notifications_older_than_retention,
     delete_inbound_sms,
     delete_sms_notifications_older_than_retention,
@@ -15,7 +16,7 @@ from app.celery.nightly_tasks import (
     save_daily_notification_processing_time,
     timeout_notifications,
 )
-from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime
+from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime, Job
 from tests.app.db import (
     create_job,
     create_notification,
@@ -313,3 +314,17 @@ def test_delete_notifications_task_calls_task_for_services_that_have_sent_notifi
             'datetime_to_delete_before': date(2021, 3, 26)
         }),
     ])
+
+
+def test_cleanup_unfinished_jobs(mocker):
+    mock_s3 = mocker.patch('app.celery.nightly_tasks.remove_csv_object')
+    mock_dao_archive = mocker.patch('app.celery.nightly_tasks.dao_archive_job')
+    mock_dao = mocker.patch('app.celery.nightly_tasks.dao_get_unfinished_jobs')
+    mock_job_unfinished = Job()
+    mock_job_unfinished.processing_started = datetime(2023, 1, 1, 0, 0, 0)
+    mock_job_unfinished.original_file_name = "blah"
+
+    mock_dao.return_value = [mock_job_unfinished]
+    cleanup_unfinished_jobs()
+    mock_s3.assert_called_once_with('blah')
+    mock_dao_archive.assert_called_once_with(mock_job_unfinished)

tests/app/celery/test_provider_tasks.py

@@ -67,11 +67,11 @@ def test_should_retry_and_log_warning_if_SmsClientResponseException_for_deliver_
     )
     mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
     mock_logger_warning = mocker.patch('app.celery.tasks.current_app.logger.warning')
+    assert sample_notification.status == 'created'
 
     deliver_sms(sample_notification.id)
 
     assert provider_tasks.deliver_sms.retry.called is True
-    assert sample_notification.status == 'created'
     assert mock_logger_warning.called
 
 
@@ -82,14 +82,13 @@ def test_should_retry_and_log_exception_for_non_SmsClientResponseException_excep
     mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
     mock_logger_exception = mocker.patch('app.celery.tasks.current_app.logger.exception')
 
+    assert sample_notification.status == 'created'
     deliver_sms(sample_notification.id)
 
     assert provider_tasks.deliver_sms.retry.called is True
-    assert sample_notification.status == 'created'
     assert mock_logger_exception.called
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(sample_notification, mocker):
     mocker.patch('app.delivery.send_to_providers.send_sms_to_provider', side_effect=Exception("EXPECTED"))
     mocker.patch('app.celery.provider_tasks.deliver_sms.retry', side_effect=MaxRetriesExceededError())
@@ -128,7 +127,6 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_ta
     app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks")
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 @pytest.mark.parametrize(
     'exception_class', [
         Exception(),
@@ -150,7 +148,6 @@ def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_email_task
     assert sample_notification.status == 'technical-failure'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_should_technical_error_and_not_retry_if_EmailClientNonRetryableException(sample_notification, mocker):
     mocker.patch(
         'app.delivery.send_to_providers.send_email_to_provider',

tests/app/celery/test_reporting_tasks.py

@@ -108,10 +108,9 @@ def test_create_nightly_notification_status_triggers_relevant_tasks(
     assert types == expected_types_aggregated
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_create_nightly_billing_for_day_checks_history(
     sample_service,
-    sample_sms_template,
+    sample_template,
     mocker
 ):
     yesterday = datetime.now() - timedelta(days=1)
@@ -119,13 +118,13 @@ def test_create_nightly_billing_for_day_checks_history(
 
     create_notification(
         created_at=yesterday,
-        template=sample_sms_template,
+        template=sample_template,
         status='sending',
     )
 
     create_notification_history(
         created_at=yesterday,
-        template=sample_sms_template,
+        template=sample_template,
         status='delivered',
     )
 
@@ -141,7 +140,6 @@ def test_create_nightly_billing_for_day_checks_history(
     assert record.notifications_sent == 2
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 @pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier',
                          [(1.0, 1, 2, [1]),
                           (2.0, 2, 1, [1, 2])])
@@ -193,7 +191,6 @@ def test_create_nightly_billing_for_day_sms_rate_multiplier(
         assert record.rate_multiplier == multiplier[i]
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_create_nightly_billing_for_day_different_templates(
         sample_service,
         sample_template,
@@ -240,8 +237,7 @@ def test_create_nightly_billing_for_day_different_templates(
         assert record.rate_multiplier == multiplier[i]
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
+def test_create_nightly_billing_for_day_same_sent_by(
-def test_create_nightly_billing_for_day_different_sent_by(
     sample_service,
     sample_template,
     sample_email_template,
@@ -276,16 +272,15 @@ def test_create_nightly_billing_for_day_different_sent_by(
     create_nightly_billing_for_day(str(yesterday.date()))
 
     records = FactBilling.query.order_by('rate_multiplier').all()
-    assert len(records) == 2
+    assert len(records) == 1
 
     for _, record in enumerate(records):
         assert record.local_date == datetime.date(yesterday)
         assert record.rate == Decimal(1.33)
-        assert record.billable_units == 1
+        assert record.billable_units == 2
         assert record.rate_multiplier == 1.0
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_create_nightly_billing_for_day_null_sent_by_sms(
     sample_service,
     sample_template,
@@ -497,7 +492,6 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
     assert sms_delivered_row.key_type == KEY_TYPE_NORMAL
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_create_nightly_notification_status_for_service_and_day_overwrites_old_data(notify_db_session):
     first_service = create_service(service_name='First Service')
     first_template = create_template(service=first_service)

tests/app/celery/test_research_mode_tasks.py

@@ -12,6 +12,7 @@ from app.celery.research_mode_tasks import (
     sns_callback,
 )
 from app.config import QueueNames
+from app.models import NOTIFICATION_DELIVERED, NOTIFICATION_FAILED, Notification
 from tests.conftest import Matcher
 
 dvla_response_file_matcher = Matcher(
@@ -20,24 +21,33 @@ dvla_response_file_matcher = Matcher(
 )
 
 
-@pytest.mark.skip(reason="Re-enable when SMS receipts exist")
+def test_make_sns_callback(notify_api, rmock, mocker):
-def test_make_sns_callback(notify_api, rmock):
     endpoint = "http://localhost:6011/notifications/sms/sns"
+    get_notification_by_id = mocker.patch('app.celery.research_mode_tasks.get_notification_by_id')
+    n = Notification()
+    n.id = 1234
+    n.status = NOTIFICATION_DELIVERED
+    get_notification_by_id.return_value = n
     rmock.request(
         "POST",
         endpoint,
         json={"status": "success"},
         status_code=200)
-    send_sms_response("sns", "1234", "2028675309")
+    send_sms_response("sns", "1234")
 
     assert rmock.called
     assert rmock.request_history[0].url == endpoint
-    assert json.loads(rmock.request_history[0].text)['MSISDN'] == '2028675309'
+    assert json.loads(rmock.request_history[0].text)['status'] == 'delivered'
 
 
-@pytest.mark.skip(reason="Re-enable when SMS receipts exist")
 def test_callback_logs_on_api_call_failure(notify_api, rmock, mocker):
     endpoint = "http://localhost:6011/notifications/sms/sns"
+    get_notification_by_id = mocker.patch('app.celery.research_mode_tasks.get_notification_by_id')
+    n = Notification()
+    n.id = 1234
+    n.status = NOTIFICATION_FAILED
+    get_notification_by_id.return_value = n
+
     rmock.request(
         "POST",
         endpoint,
@@ -46,12 +56,12 @@ def test_callback_logs_on_api_call_failure(notify_api, rmock, mocker):
     mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
 
     with pytest.raises(HTTPError):
-        send_sms_response("mmg", "1234", "07700900001")
+        send_sms_response("sns", "1234")
 
     assert rmock.called
     assert rmock.request_history[0].url == endpoint
     mock_logger.assert_called_once_with(
-        'API POST request on http://localhost:6011/notifications/sms/mmg failed with status 500'
+        'API POST request on http://localhost:6011/notifications/sms/sns failed with status 500'
     )
 
 
@@ -65,31 +75,13 @@ def test_make_ses_callback(notify_api, mocker):
     assert mock_task.apply_async.call_args[0][0][0] == ses_notification_callback(some_ref)
 
 
-@pytest.mark.skip(reason="Re-enable when SNS delivery receipts exist")
+def test_delivered_sns_callback(mocker):
-def test_delievered_sns_callback():
+    get_notification_by_id = mocker.patch('app.celery.research_mode_tasks.get_notification_by_id')
-    phone_number = "2028675309"
+    n = Notification()
-    data = json.loads(sns_callback("1234", phone_number))
+    n.id = 1234
-    assert data['MSISDN'] == phone_number
+    n.status = NOTIFICATION_DELIVERED
-    assert data['status'] == "3"
+    get_notification_by_id.return_value = n
-    assert data['reference'] == "sns_reference"
+
-    assert data['CID'] == "1234"
+    data = json.loads(sns_callback("1234"))
-
+    assert data['status'] == "delivered"
-
-@pytest.mark.skip(reason="Re-enable when SNS delivery receipts exist")
-def test_perm_failure_sns_callback():
-    phone_number = "2028675302"
-    data = json.loads(sns_callback("1234", phone_number))
-    assert data['MSISDN'] == phone_number
-    assert data['status'] == "5"
-    assert data['reference'] == "sns_reference"
-    assert data['CID'] == "1234"
-
-
-@pytest.mark.skip(reason="Re-enable when SNS delivery receipts exist")
-def test_temp_failure_sns_callback():
-    phone_number = "2028675303"
-    data = json.loads(sns_callback("1234", phone_number))
-    assert data['MSISDN'] == phone_number
-    assert data['status'] == "4"
-    assert data['reference'] == "sns_reference"
     assert data['CID'] == "1234"

tests/app/clients/test_sms.py

@@ -15,9 +15,8 @@ def fake_client(notify_api):
     return fake_client
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: New SMS client")
 def test_send_sms(fake_client, mocker):
-    mock_send = mocker.patch.object(fake_client, 'try_send_sms')
+    mock_send = mocker.patch.object(fake_client, 'send_sms')
 
     fake_client.send_sms(
         to='to',
@@ -28,14 +27,13 @@ def test_send_sms(fake_client, mocker):
     )
 
     mock_send.assert_called_with(
-        'to', 'content', 'reference', False, 'testing'
+        to='to', content='content', reference='reference', international=False, sender='testing'
     )
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: New SMS client")
 def test_send_sms_error(fake_client, mocker):
     mocker.patch.object(
-        fake_client, 'try_send_sms', side_effect=SmsClientResponseException('error')
+        fake_client, 'send_sms', side_effect=SmsClientResponseException('error')
     )
 
     with pytest.raises(SmsClientResponseException):

tests/app/dao/test_fact_billing_dao.py

@@ -210,8 +210,6 @@ def test_fetch_billing_data_for_day_returns_empty_list(notify_db_session):
     assert results == []
 
 
-# TODO: ready for reactivation?
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_fetch_billing_data_for_day_uses_correct_table(notify_db_session):
     service = create_service()
     create_service_data_retention(service, notification_type='email', days_of_retention=3)

tests/app/dao/test_jobs_dao.py

@@ -116,7 +116,6 @@ def test_get_jobs_for_service(sample_template):
     assert one_job_from_db != other_job_from_db
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_get_jobs_for_service_with_limit_days_param(sample_template):
     one_job = create_job(sample_template)
     old_job = create_job(sample_template, created_at=datetime.now() - timedelta(days=8))

tests/app/dao/test_organisation_dao.py

@@ -90,10 +90,6 @@ def test_update_organisation(notify_db_session):
     (['ABC', 'DEF'], {'abc', 'def'}),
     ([], set()),
     (None, {'123', '456'}),
-    pytest.param(
-        ['abc', 'ABC'], {'abc'},
-        marks=pytest.mark.xfail(raises=IntegrityError)
-    ),
 ))
 def test_update_organisation_domains_lowercases(
     notify_db_session,
@@ -113,6 +109,29 @@ def test_update_organisation_domains_lowercases(
     assert {domain.domain for domain in organisation.domains} == expected_domains
 
 
+@pytest.mark.parametrize('domain_list, expected_domains', (
+        (['abc', 'ABC'], {'abc'}),
+))
+def test_update_organisation_domains_lowercases_integrity_error(
+    notify_db_session,
+    domain_list,
+    expected_domains,
+):
+    create_organisation()
+
+    organisation = Organisation.query.one()
+
+    # Seed some domains
+    dao_update_organisation(organisation.id, domains=['123', '456'])
+
+    with pytest.raises(expected_exception=IntegrityError):
+
+        # This should overwrite the seeded domains
+        dao_update_organisation(organisation.id, domains=domain_list)
+
+        assert {domain.domain for domain in organisation.domains} == expected_domains
+
+
 def test_update_organisation_does_not_update_the_service_if_certain_attributes_not_provided(
     sample_service,
     sample_organisation,

tests/app/dao/test_provider_details_dao.py

@@ -193,44 +193,6 @@ def test_get_sms_providers_for_update_returns_nothing_if_recent_updates(restore_
     assert not resp
 
 
-@pytest.mark.skip(reason="Reenable if/when we add a second SMS provider")
-@pytest.mark.parametrize(['starting_priorities', 'expected_priorities'], [
-    ({'sns': 50, 'other': 50}, {'sns': 40, 'other': 60}),
-    ({'sns': 0, 'other': 20}, {'sns': 0, 'other': 30}),  # lower bound respected
-    ({'sns': 50, 'other': 100}, {'sns': 40, 'other': 100}),  # upper bound respected
-
-    # document what happens if they have unexpected values outside of the 0 - 100 range (due to manual setting from
-    # the admin app). the code never causes further issues, but sometimes doesn't actively reset the vaues to 0-100.
-    ({'sns': 150, 'other': 50}, {'sns': 140, 'other': 60}),
-    ({'sns': 50, 'other': 150}, {'sns': 40, 'other': 100}),
-
-    ({'sns': -100, 'other': 50}, {'sns': 0, 'other': 60}),
-    ({'sns': 50, 'other': -100}, {'sns': 40, 'other': -90}),
-])
-def test_reduce_sms_provider_priority_adjusts_provider_priorities(
-    mocker,
-    restore_provider_details,
-    notify_user,
-    starting_priorities,
-    expected_priorities,
-):
-    mock_adjust = mocker.patch('app.dao.provider_details_dao._adjust_provider_priority')
-
-    sns = get_provider_details_by_identifier('sns')
-    other = get_provider_details_by_identifier('other')
-
-    sns.priority = starting_priorities['sns']
-    other.priority = starting_priorities['other']
-    # need to update these manually to avoid triggering the `onupdate` clause of the updated_at column
-    ProviderDetails.query.filter(ProviderDetails.notification_type == 'sms').update({'updated_at': datetime.min})
-
-    # switch away from sns. currently both 50/50
-    dao_reduce_sms_provider_priority('sns', time_threshold=timedelta(minutes=10))
-
-    mock_adjust.assert_any_call(other, expected_priorities['other'])
-    mock_adjust.assert_any_call(sns, expected_priorities['sns'])
-
-
 def test_reduce_sms_provider_priority_does_nothing_if_providers_have_recently_changed(
     mocker,
     restore_provider_details,
@@ -255,38 +217,6 @@ def test_reduce_sms_provider_priority_does_nothing_if_there_is_only_one_active_p
     assert mock_adjust.called is False
 
 
-@pytest.mark.skip(reason="Reenable if/when we add a second SMS provider")
-@pytest.mark.parametrize('existing_sns, existing_other, new_sns, new_other', [
-    (50, 50, 60, 40),  # not just 50/50 - 60/40 specifically
-    (65, 35, 60, 40),  # doesn't overshoot if there's less than 10 difference
-    (0, 100, 10, 90),  # only adjusts by 10
-    (100, 100, 90, 90),  # it tries to fix weird data - it will reduce both if needs be
-])
-def test_adjust_provider_priority_back_to_resting_points_updates_all_providers(
-    restore_provider_details,
-    mocker,
-    existing_sns,
-    existing_other,
-    new_sns,
-    new_other
-):
-    sns = get_provider_details_by_identifier('sns')
-    other = get_provider_details_by_identifier('other')
-    sns.priority = existing_sns
-    other.priority = existing_other
-
-    mock_adjust = mocker.patch('app.dao.provider_details_dao._adjust_provider_priority')
-    mock_get_providers = mocker.patch('app.dao.provider_details_dao._get_sms_providers_for_update', return_value=[
-        sns, other
-    ])
-
-    dao_adjust_provider_priority_back_to_resting_points()
-
-    mock_get_providers.assert_called_once_with(timedelta(hours=1))
-    mock_adjust.assert_any_call(sns, new_sns)
-    mock_adjust.assert_any_call(other, new_other)
-
-
 def test_adjust_provider_priority_back_to_resting_points_does_nothing_if_theyre_already_at_right_values(
     restore_provider_details,
     mocker,

tests/app/dao/test_services_dao.py

@@ -3,6 +3,7 @@ from datetime import datetime, timedelta
 from unittest import mock
 
 import pytest
+import sqlalchemy
 from freezegun import freeze_time
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
@@ -585,7 +586,6 @@ def test_update_service_permission_creates_a_history_record_with_current_data(no
     assert history[2].version == 3
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_create_service_and_history_is_transactional(notify_db_session):
     user = create_user()
     assert Service.query.count() == 0
@@ -596,10 +596,11 @@ def test_create_service_and_history_is_transactional(notify_db_session):
                       restricted=False,
                       created_by=user)
 
-    with pytest.raises(IntegrityError) as excinfo:
+    try:
         dao_create_service(service, user)
+    except sqlalchemy.exc.IntegrityError as seeei:
+        assert 'null value in column "name" of relation "services_history" violates not-null constraint' in str(seeei)
 
-    assert 'column "name" violates not-null constraint' in str(excinfo.value)
     assert Service.query.count() == 0
     assert Service.get_history_model().query.count() == 0
 
@@ -826,7 +827,6 @@ def test_dao_fetch_todays_stats_for_service_only_includes_today_during_bst(notif
     assert not stats.get('permanent-failure')
 
 
-@pytest.mark.skip(reason="Need a better way to test variable DST date")
 def test_dao_fetch_todays_stats_for_service_only_includes_today_when_clocks_fall_back(notify_db_session):
     template = create_template(service=create_service())
     with freeze_time('2021-10-30T22:59:59'):
@@ -871,7 +871,6 @@ def test_dao_fetch_todays_stats_for_service_only_includes_during_utc(notify_db_s
     assert not stats.get('permanent-failure')
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_dao_fetch_todays_stats_for_all_services_includes_all_services(notify_db_session):
     # two services, each with an email and sms notification
     service1 = create_service(service_name='service 1', email_from='service.1')
@@ -910,7 +909,6 @@ def test_dao_fetch_todays_stats_for_all_services_only_includes_today(notify_db_s
     assert stats['failed'] == 1
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_dao_fetch_todays_stats_for_all_services_groups_correctly(notify_db_session):
     service1 = create_service(service_name='service 1', email_from='service.1')
     service2 = create_service(service_name='service 2', email_from='service.2')
@@ -937,7 +935,6 @@ def test_dao_fetch_todays_stats_for_all_services_groups_correctly(notify_db_sess
             service2.created_at, 'sms', 'created', 1) in stats
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_dao_fetch_todays_stats_for_all_services_includes_all_keys_by_default(notify_db_session):
     template = create_template(service=create_service())
     create_notification(template=template, key_type=KEY_TYPE_NORMAL)
@@ -950,7 +947,6 @@ def test_dao_fetch_todays_stats_for_all_services_includes_all_keys_by_default(no
     assert stats[0].count == 3
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_dao_fetch_todays_stats_for_all_services_can_exclude_from_test_key(notify_db_session):
     template = create_template(service=create_service())
     create_notification(template=template, key_type=KEY_TYPE_NORMAL)

tests/app/dao/test_uploads_dao.py

@@ -1,6 +1,5 @@
 from datetime import datetime, timedelta
 
-import pytest
 from freezegun import freeze_time
 
 from app.dao.uploads_dao import dao_get_uploads_by_service_id
@@ -38,39 +37,21 @@ def create_uploaded_template(service):
     )
 
 
-@pytest.mark.skip(reason="Investigate what remains after removing letters")
 @freeze_time("2020-02-02 09:00")  # GMT time
 def test_get_uploads_for_service(sample_template):
     create_service_data_retention(sample_template.service, 'sms', days_of_retention=9)
     job = create_job(sample_template, processing_started=datetime.utcnow())
-    letter_template = create_uploaded_template(sample_template.service)
-    letter = create_uploaded_letter(letter_template, sample_template.service)
 
     other_service = create_service(service_name="other service")
     other_template = create_template(service=other_service)
     other_job = create_job(other_template, processing_started=datetime.utcnow())
-    other_letter_template = create_uploaded_template(other_service)
-    create_uploaded_letter(other_letter_template, other_service)
 
     uploads_from_db = dao_get_uploads_by_service_id(job.service_id).items
     other_uploads_from_db = dao_get_uploads_by_service_id(other_job.service_id).items
 
-    assert len(uploads_from_db) == 2
+    assert len(uploads_from_db) == 1
 
     assert uploads_from_db[0] == (
-        None,
-        'Uploaded letters',
-        1,
-        'letter',
-        None,
-        letter.created_at.replace(hour=22, minute=30, second=0, microsecond=0),
-        None,
-        letter.created_at.replace(hour=22, minute=30, second=0, microsecond=0),
-        None,
-        'letter_day',
-        None,
-    )
-    assert uploads_from_db[1] == (
         job.id,
         job.original_file_name,
         job.notification_count,
@@ -84,21 +65,8 @@ def test_get_uploads_for_service(sample_template):
         None,
     )
 
-    assert len(other_uploads_from_db) == 2
+    assert len(other_uploads_from_db) == 1
-    assert other_uploads_from_db[0] == (
+    assert other_uploads_from_db[0] == (other_job.id,
-        None,
-        'Uploaded letters',
-        1,
-        'letter',
-        None,
-        letter.created_at.replace(hour=22, minute=30, second=0, microsecond=0),
-        None,
-        letter.created_at.replace(hour=22, minute=30, second=0, microsecond=0),
-        None,
-        "letter_day",
-        None,
-    )
-    assert other_uploads_from_db[1] == (other_job.id,
                                         other_job.original_file_name,
                                         other_job.notification_count,
                                         other_job.template.template_type,
@@ -110,81 +78,9 @@ def test_get_uploads_for_service(sample_template):
                                         "job",
                                         None)
 
-    assert uploads_from_db[1] != other_uploads_from_db[1]
+    assert uploads_from_db[0] != other_uploads_from_db[0]
 
 
-@pytest.mark.skip(reason="Investigate what remains after removing letters")
-@freeze_time("2020-02-02 18:00")
-def test_get_uploads_for_service_groups_letters(sample_template):
-    letter_template = create_uploaded_template(sample_template.service)
-
-    # Just gets into yesterday’s print run
-    create_uploaded_letter(letter_template, sample_template.service, created_at=(
-        datetime(2020, 2, 1, 22, 29, 59)
-    ))
-
-    # Yesterday but in today’s print run
-    create_uploaded_letter(letter_template, sample_template.service, created_at=(
-        datetime(2020, 2, 1, 22, 30)
-    ))
-    # First thing today
-    create_uploaded_letter(letter_template, sample_template.service, created_at=(
-        datetime(2020, 2, 2, 5, 0)
-    ))
-    # Just before today’s print deadline
-    create_uploaded_letter(letter_template, sample_template.service, created_at=(
-        datetime(2020, 2, 2, 22, 29, 59)
-    ))
-
-    # Just missed today’s print deadline
-    create_uploaded_letter(letter_template, sample_template.service, created_at=(
-        datetime(2020, 2, 2, 22, 30)
-    ))
-
-    uploads_from_db = dao_get_uploads_by_service_id(sample_template.service_id).items
-
-    assert [
-        (upload.notification_count, upload.created_at)
-        for upload in uploads_from_db
-    ] == [
-        (1, datetime(2020, 2, 3, 22, 30)),
-        (3, datetime(2020, 2, 2, 22, 30)),
-        (1, datetime(2020, 2, 1, 22, 30)),
-    ]
-
-
-@pytest.mark.skip(reason="Investigate what remains after removing letters")
-def test_get_uploads_does_not_return_cancelled_jobs_or_letters(sample_template):
-    create_job(sample_template, job_status='scheduled')
-    create_job(sample_template, job_status='cancelled')
-    letter_template = create_uploaded_template(sample_template.service)
-    create_uploaded_letter(letter_template, sample_template.service, status='cancelled')
-
-    assert len(dao_get_uploads_by_service_id(sample_template.service_id).items) == 0
-
-
-@pytest.mark.skip(reason="Investigate what remains after removing letters")
-def test_get_uploads_orders_by_created_at_desc(sample_template):
-    letter_template = create_uploaded_template(sample_template.service)
-
-    upload_1 = create_job(sample_template, processing_started=datetime.utcnow(),
-                          job_status=JOB_STATUS_IN_PROGRESS)
-    upload_2 = create_job(sample_template, processing_started=datetime.utcnow(),
-                          job_status=JOB_STATUS_IN_PROGRESS)
-    create_uploaded_letter(letter_template, sample_template.service, status='delivered')
-
-    results = dao_get_uploads_by_service_id(service_id=sample_template.service_id).items
-
-    assert [
-        (result.id, result.upload_type) for result in results
-    ] == [
-        (None, 'letter_day'),
-        (upload_2.id, 'job'),
-        (upload_1.id, 'job'),
-    ]
-
-
-@pytest.mark.skip(reason="Investigate what remains after removing letters")
 def test_get_uploads_orders_by_processing_started_desc(sample_template):
     days_ago = datetime.utcnow() - timedelta(days=3)
     upload_1 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=1),
@@ -201,121 +97,6 @@ def test_get_uploads_orders_by_processing_started_desc(sample_template):
     assert results[1].id == upload_2.id
 
 
-@pytest.mark.skip(reason="Investigate what remains after removing letters")
-@freeze_time("2020-10-27 16:15")  # GMT time
-def test_get_uploads_orders_by_processing_started_and_created_at_desc(sample_template):
-    letter_template = create_uploaded_template(sample_template.service)
-
-    days_ago = datetime.utcnow() - timedelta(days=4)
-    create_uploaded_letter(letter_template, service=letter_template.service)
-    upload_2 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=1),
-                          created_at=days_ago,
-                          job_status=JOB_STATUS_IN_PROGRESS)
-    upload_3 = create_job(sample_template, processing_started=datetime.utcnow() - timedelta(days=2),
-                          created_at=days_ago,
-                          job_status=JOB_STATUS_IN_PROGRESS)
-    create_uploaded_letter(letter_template, service=letter_template.service,
-                           created_at=datetime.utcnow() - timedelta(days=3))
-
-    results = dao_get_uploads_by_service_id(service_id=sample_template.service_id).items
-
-    assert len(results) == 4
-    assert results[0].id is None
-    assert results[1].id == upload_2.id
-    assert results[2].id == upload_3.id
-    assert results[3].id is None
-
-
-@pytest.mark.skip(reason="Investigate what remains after removing letters")
-@freeze_time('2020-04-02 14:00')  # Few days after the clocks go forward
-def test_get_uploads_only_gets_uploads_within_service_retention_period(sample_template):
-    letter_template = create_uploaded_template(sample_template.service)
-    create_service_data_retention(sample_template.service, 'sms', days_of_retention=3)
-
-    days_ago = datetime.utcnow() - timedelta(days=4)
-    upload_1 = create_uploaded_letter(letter_template, service=letter_template.service)
-    upload_2 = create_job(
-        sample_template, processing_started=datetime.utcnow() - timedelta(days=1), created_at=days_ago,
-        job_status=JOB_STATUS_IN_PROGRESS
-    )
-    # older than custom retention for sms:
-    create_job(
-        sample_template, processing_started=datetime.utcnow() - timedelta(days=5), created_at=days_ago,
-        job_status=JOB_STATUS_IN_PROGRESS
-    )
-    upload_3 = create_uploaded_letter(
-        letter_template, service=letter_template.service, created_at=datetime.utcnow() - timedelta(days=3)
-    )
-
-    # older than retention for sms but within letter retention:
-    upload_4 = create_uploaded_letter(
-        letter_template, service=letter_template.service, created_at=datetime.utcnow() - timedelta(days=6)
-    )
-
-    # older than default retention for letters:
-    create_uploaded_letter(
-        letter_template, service=letter_template.service, created_at=datetime.utcnow() - timedelta(days=8)
-    )
-
-    results = dao_get_uploads_by_service_id(service_id=sample_template.service_id).items
-
-    assert len(results) == 4
-
-    # Uploaded letters get their `created_at` shifted time of printing
-    # 21:30 EST == 16:30 UTC
-    assert results[0].created_at == upload_1.created_at.replace(hour=21, minute=30, second=0, microsecond=0)
-
-    # Jobs keep their original `created_at`
-    assert results[1].created_at == upload_2.created_at.replace(hour=14, minute=00, second=0, microsecond=0)
-
-    # Still in BST here…
-    assert results[2].created_at == upload_3.created_at.replace(hour=21, minute=30, second=0, microsecond=0)
-
-    # Now we’ve gone far enough back to be in GMT
-    # 17:30 GMT == 17:30 UTC
-    assert results[3].created_at == upload_4.created_at.replace(hour=21, minute=30, second=0, microsecond=0)
-
-
-@pytest.mark.skip(reason="Investigate what remains after removing letters")
-@freeze_time('2020-02-02 14:00')
-def test_get_uploads_is_paginated(sample_template):
-    letter_template = create_uploaded_template(sample_template.service)
-
-    create_uploaded_letter(
-        letter_template, sample_template.service, status='delivered',
-        created_at=datetime.utcnow() - timedelta(minutes=3),
-    )
-    create_job(
-        sample_template, processing_started=datetime.utcnow() - timedelta(minutes=2),
-        job_status=JOB_STATUS_IN_PROGRESS,
-    )
-    create_uploaded_letter(
-        letter_template, sample_template.service, status='delivered',
-        created_at=datetime.utcnow() - timedelta(minutes=1),
-    )
-    create_job(
-        sample_template, processing_started=datetime.utcnow(),
-        job_status=JOB_STATUS_IN_PROGRESS,
-    )
-
-    results = dao_get_uploads_by_service_id(sample_template.service_id, page=1, page_size=1)
-
-    assert results.per_page == 1
-    assert results.total == 3
-    assert len(results.items) == 1
-    assert results.items[0].created_at == datetime.utcnow().replace(hour=22, minute=30, second=0, microsecond=0)
-    assert results.items[0].notification_count == 2
-    assert results.items[0].upload_type == 'letter_day'
-
-    results = dao_get_uploads_by_service_id(sample_template.service_id, page=2, page_size=1)
-
-    assert len(results.items) == 1
-    assert results.items[0].created_at == datetime.utcnow().replace(hour=14, minute=0, second=0, microsecond=0)
-    assert results.items[0].notification_count == 1
-    assert results.items[0].upload_type == 'job'
-
-
-@pytest.mark.skip(reason="Investigate what remains after removing letters")
 def test_get_uploads_returns_empty_list(sample_service):
     items = dao_get_uploads_by_service_id(sample_service.id).items
     assert items == []

tests/app/delivery/test_send_to_providers.py

@@ -45,36 +45,6 @@ def setup_function(_function):
     send_to_providers.provider_cache.clear()
 
 
-@pytest.mark.skip(reason="Reenable when we have more than 1 SMS provider")
-def test_provider_to_use_should_return_random_provider(mocker, notify_db_session):
-    sns = get_provider_details_by_identifier('sns')
-    other = get_provider_details_by_identifier('other')
-    sns.priority = 60
-    other.priority = 40
-    mock_choices = mocker.patch('app.delivery.send_to_providers.random.choices', return_value=[sns])
-
-    ret = send_to_providers.provider_to_use('sms', international=True)
-
-    mock_choices.assert_called_once_with([sns, other], weights=[60, 40])
-    assert ret.name == 'sns'
-
-
-@pytest.mark.skip(reason="Reenable when we have more than 1 SMS provider")
-def test_provider_to_use_should_cache_repeated_calls(mocker, notify_db_session):
-    mock_choices = mocker.patch(
-        'app.delivery.send_to_providers.random.choices',
-        wraps=send_to_providers.random.choices,
-    )
-
-    results = [
-        send_to_providers.provider_to_use('sms', international=False)
-        for _ in range(10)
-    ]
-
-    assert all(result == results[0] for result in results)
-    assert len(mock_choices.call_args_list) == 1
-
-
 @pytest.mark.parametrize('international_provider_priority', (
     # Since there’s only one international provider it should always
     # be used, no matter what its priority is set to
@@ -93,18 +63,6 @@ def test_provider_to_use_should_only_return_sns_for_international(
     assert ret.name == 'sns'
 
 
-@pytest.mark.skip(reason="Reenable when we have more than 1 SMS provider")
-def test_provider_to_use_should_only_return_active_providers(mocker, restore_provider_details):
-    sns = get_provider_details_by_identifier('sns')
-    other = get_provider_details_by_identifier('other')
-    sns.active = False
-    other.active = True
-
-    ret = send_to_providers.provider_to_use('sms')
-
-    assert ret.name == 'other'
-
-
 def test_provider_to_use_raises_if_no_active_providers(mocker, restore_provider_details):
     sns = get_provider_details_by_identifier('sns')
     sns.active = False
@@ -261,7 +219,7 @@ def test_should_call_send_sms_response_task_if_research_mode(
         notify_db_session, sample_service, sample_notification, mocker, research_mode, key_type
 ):
     mocker.patch('app.aws_sns_client.send_sms')
-    mocker.patch('app.delivery.send_to_providers.send_sms_response')
+    send_sms_response = mocker.patch('app.delivery.send_to_providers.send_sms_response')
 
     if research_mode:
         sample_service.research_mode = True
@@ -275,8 +233,8 @@ def test_should_call_send_sms_response_task_if_research_mode(
     )
     assert not aws_sns_client.send_sms.called
 
-    app.delivery.send_to_providers.send_sms_response.assert_called_once_with(
+    send_sms_response.assert_called_once_with(
-        'sns', str(sample_notification.id), sample_notification.to
+        'sns', str(sample_notification.id)
     )
 
     persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id)
@@ -288,17 +246,15 @@ def test_should_call_send_sms_response_task_if_research_mode(
     assert not persisted_notification.personalisation
 
 
-@pytest.mark.skip(reason="Needs updating when we get SMS delivery receipts done")
 def test_should_have_sending_status_if_fake_callback_function_fails(sample_notification, mocker):
     mocker.patch('app.delivery.send_to_providers.send_sms_response', side_effect=HTTPError)
 
     sample_notification.key_type = KEY_TYPE_TEST
-
     with pytest.raises(HTTPError):
         send_to_providers.send_sms_to_provider(
             sample_notification
         )
-    assert sample_notification.status == 'sending'
+    assert sample_notification.status == 'sent'
     assert sample_notification.sent_by == 'sns'
 
 
@@ -389,14 +345,14 @@ def test_send_email_to_provider_should_call_research_mode_task_response_task_if_
     reference = uuid.uuid4()
     mocker.patch('app.uuid.uuid4', return_value=reference)
     mocker.patch('app.aws_ses_client.send_email')
-    mocker.patch('app.delivery.send_to_providers.send_email_response')
+    send_email_response = mocker.patch('app.delivery.send_to_providers.send_email_response')
 
     send_to_providers.send_email_to_provider(
         notification
     )
 
     assert not app.aws_ses_client.send_email.called
-    app.delivery.send_to_providers.send_email_response.assert_called_once_with(str(reference), 'john@smith.com')
+    send_email_response.assert_called_once_with(str(reference), 'john@smith.com')
     persisted_notification = Notification.query.filter_by(id=notification.id).one()
     assert persisted_notification.to == 'john@smith.com'
     assert persisted_notification.template_id == sample_email_template.id

tests/app/inbound_sms/test_rest.py

@@ -76,7 +76,6 @@ def test_post_to_get_inbound_sms_filters_international_user_number(admin_request
     )['data']
 
     assert len(sms) == 2
-    print(f'sms is: {sms}')
     assert sms[1]['id'] == str(one.id)
     assert sms[1]['user_number'] == str(one.user_number)
 

tests/app/notifications/test_notifications_ses_callback.py

@@ -86,7 +86,6 @@ def test_check_and_queue_callback_task(mocker, sample_notification):
 
     # callback_api doesn't match by equality for some
     # reason, so we need to take this approach instead
-    print(f'mock_create.mock_calls is: {mock_create.mock_calls}')
     mock_create_args = mock_create.mock_calls[0][1]
     assert mock_create_args[0] == sample_notification
     assert mock_create_args[1].id == callback_api.id

tests/app/notifications/test_validators.py

@@ -587,7 +587,6 @@ def test_check_reply_to_sms_type(sample_service):
     assert check_reply_to(sample_service.id, sms_sender.id, SMS_TYPE) == '123456'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_check_if_service_can_send_files_by_email_raises_if_no_contact_link_set(sample_service):
     with pytest.raises(BadRequestError) as e:
         check_if_service_can_send_files_by_email(

tests/app/organisation/test_invite_rest.py

@@ -10,7 +10,6 @@ from tests import create_admin_authorization_header
 from tests.app.db import create_invited_org_user
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 @pytest.mark.parametrize('platform_admin, expected_invited_by', (
     (True, 'The GOV.UK Notify team'),
     (False, 'Test User')

tests/app/organisation/test_rest.py

@@ -127,10 +127,7 @@ def test_get_organisation_by_id_returns_domains(admin_request, notify_db_session
     ('foo.gov.uk', 200),
     ('bar.gov.uk', 200),
     ('oof.gov.uk', 404),
-    pytest.param(
+    ('rab.gov.uk', 200),
-        'rab.gov.uk', 200,
-        marks=pytest.mark.xfail(raises=AssertionError),
-    ),
     (None, 400),
     ('personally.identifying.information@example.com', 400),
 ))
@@ -152,7 +149,9 @@ def test_get_organisation_by_domain(
         domain=domain,
     )
 
-    if expected_status == 200:
+    if domain == 'rab.gov.uk' and expected_status == 200:
+        assert response['id'] == str(other_org.id)
+    elif expected_status == 200:
         assert response['id'] == str(org.id)
     else:
         assert response['result'] == 'error'
@@ -490,7 +489,6 @@ def test_post_update_organisation_set_mou_doesnt_email_if_no_signed_by(
     assert queue_mock.called is False
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 @pytest.mark.parametrize('on_behalf_of_name, on_behalf_of_email_address, templates_and_recipients', [
     (
         None,

tests/app/performance_dashboard/test_rest.py

@@ -1,7 +1,5 @@
 from datetime import date
 
-import pytest
-
 from tests.app.db import (
     create_ft_notification_status,
     create_process_time,
@@ -9,11 +7,9 @@ from tests.app.db import (
 )
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Needs updating for new providers")
 def test_performance_dashboard(sample_service, admin_request):
     template_sms = create_template(service=sample_service, template_type='sms', template_name='a')
     template_email = create_template(service=sample_service, template_type='email', template_name='b')
-    template_letter = create_template(service=sample_service, template_type='letter', template_name='c')
     create_ft_notification_status(local_date=date(2021, 2, 28),
                                   service=template_email.service,
                                   template=template_email,
@@ -22,10 +18,6 @@ def test_performance_dashboard(sample_service, admin_request):
                                   service=template_sms.service,
                                   template=template_sms,
                                   count=5)
-    create_ft_notification_status(local_date=date(2021, 2, 28),
-                                  service=template_letter.service,
-                                  template=template_letter,
-                                  count=2)
     create_ft_notification_status(local_date=date(2021, 3, 1),
                                   service=template_email.service,
                                   template=template_email,
@@ -34,10 +26,6 @@ def test_performance_dashboard(sample_service, admin_request):
                                   service=template_sms.service,
                                   template=template_sms,
                                   count=20)
-    create_ft_notification_status(local_date=date(2021, 3, 1),
-                                  service=template_letter.service,
-                                  template=template_letter,
-                                  count=3)
     create_ft_notification_status(local_date=date(2021, 3, 2),
                                   service=template_email.service,
                                   template=template_email,
@@ -46,10 +34,6 @@ def test_performance_dashboard(sample_service, admin_request):
                                   service=template_sms.service,
                                   template=template_sms,
                                   count=30)
-    create_ft_notification_status(local_date=date(2021, 3, 2),
-                                  service=template_letter.service,
-                                  template=template_letter,
-                                  count=10)
     create_ft_notification_status(local_date=date(2021, 3, 3),
                                   service=template_email.service,
                                   template=template_email,
@@ -58,10 +42,6 @@ def test_performance_dashboard(sample_service, admin_request):
                                   service=template_sms.service,
                                   template=template_sms,
                                   count=35)
-    create_ft_notification_status(local_date=date(2021, 3, 3),
-                                  service=template_letter.service,
-                                  template=template_letter,
-                                  count=15)
 
     create_process_time(local_date='2021-02-28', messages_total=15, messages_within_10_secs=14)
     create_process_time(local_date='2021-03-01', messages_total=35, messages_within_10_secs=34)
@@ -72,13 +52,12 @@ def test_performance_dashboard(sample_service, admin_request):
                                 start_date='2021-03-01',
                                 end_date='2021-03-02')
 
-    assert results['total_notifications'] == 10+5+2+15+20+3+25+30+10+45+35+15
+    assert results['total_notifications'] == 185
     assert results['email_notifications'] == 10+15+25+45
     assert results['sms_notifications'] == 5+20+30+35
-    assert results['letter_notifications'] == 2+3+10+15
+    assert results['notifications_by_type'] == [{"date": '2021-03-01', "emails": 15, "sms": 20},
-    assert results['notifications_by_type'] == [{"date": '2021-03-01', "emails": 15, "sms": 20, "letters": 3},
+                                                {"date": '2021-03-02', "emails": 25, "sms": 30}]
-                                                {"date": '2021-03-02', "emails": 25, "sms": 30, "letters": 10}]
+    assert results['processing_time'] == [{"date": "2021-03-01", "percentage_under_10_seconds": 97.14285714285714},
-    assert results['processing_time'] == [{"date": "2021-03-01", "percentage_under_10_seconds": 97.1428571428571},
                                           {"date": "2021-03-02", "percentage_under_10_seconds": 80.0}]
     assert results["live_service_count"] == 1
     assert results["services_using_notify"][0]["service_name"] == sample_service.name

tests/app/service/send_notification/test_send_notification.py

@@ -169,12 +169,6 @@ def test_send_notification_with_placeholders_replaced(notify_api, sample_email_t
         ),
         '6',
     ),
-    pytest.param(
-        None,
-        ('we consider None equivalent to missing personalisation'),
-        '',
-        marks=pytest.mark.xfail
-    ),
 ])
 def test_send_notification_with_placeholders_replaced_with_unusual_types(
     client,
@@ -209,6 +203,43 @@ def test_send_notification_with_placeholders_replaced_with_unusual_types(
     assert response_data['subject'] == expected_subject
 
 
+@pytest.mark.parametrize('personalisation, expected_body, expected_subject', [
+    (
+        None,
+        ('we consider None equivalent to missing personalisation'),
+        '',
+    ),
+])
+def test_send_notification_with_placeholders_replaced_with_unusual_types_no_personalization(
+    client,
+    sample_email_template_with_placeholders,
+    mocker,
+    personalisation,
+    expected_body,
+    expected_subject,
+):
+    mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+
+    response = client.post(
+        path='/notifications/email',
+        data=json.dumps(
+            {
+                'to': 'ok@ok.com',
+                'template': str(sample_email_template_with_placeholders.id),
+                'personalisation': {
+                    'name': personalisation
+                }
+            }
+        ),
+        headers=[
+            ('Content-Type', 'application/json'),
+            create_service_authorization_header(service_id=sample_email_template_with_placeholders.service.id)
+        ]
+    )
+
+    assert response.status_code == 400
+
+
 def test_should_not_send_notification_for_archived_template(notify_api, sample_template):
     with notify_api.test_request_context():
         with notify_api.test_client() as client:

tests/app/service/test_rest.py

@@ -1944,7 +1944,6 @@ def test_get_detailed_service(sample_template, client, sample_service, today_onl
     assert service['statistics'][SMS_TYPE] == stats
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_get_services_with_detailed_flag(client, sample_template):
     notifications = [
         create_notification(sample_template),
@@ -1967,7 +1966,6 @@ def test_get_services_with_detailed_flag(client, sample_template):
     }
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_get_services_with_detailed_flag_excluding_from_test_key(client, sample_template):
     create_notification(sample_template, key_type=KEY_TYPE_NORMAL)
     create_notification(sample_template, key_type=KEY_TYPE_TEAM)
@@ -2023,7 +2021,6 @@ def test_get_services_with_detailed_flag_defaults_to_today(client, mocker):
     assert resp.status_code == 200
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_get_detailed_services_groups_by_service(notify_db_session):
     from app.service.rest import get_detailed_services
 
@@ -2054,7 +2051,6 @@ def test_get_detailed_services_groups_by_service(notify_db_session):
     }
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
 def test_get_detailed_services_includes_services_with_no_notifications(notify_db_session):
     from app.service.rest import get_detailed_services
 

tests/app/service_invite/test_service_invite_rest.py

@@ -11,7 +11,6 @@ from tests import create_admin_authorization_header
 from tests.app.db import create_invited_user
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 @pytest.mark.parametrize('extra_args, expected_start_of_invite_url', [
     (
         {},
@@ -73,7 +72,6 @@ def test_create_invited_user(
     mocked.assert_called_once_with([(str(notification.id))], queue="notify-internal-tasks")
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_create_invited_user_without_auth_type(admin_request, sample_service, mocker, invitation_email_template):
     mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
     email_address = 'invited_user@service.gov.uk'
@@ -126,13 +124,11 @@ def test_create_invited_user_invalid_email(client, sample_service, mocker, fake_
     assert mocked.call_count == 0
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_get_all_invited_users_by_service(client, notify_db_session, sample_service):
     invites = []
     for i in range(0, 5):
         email = 'invited_user_{}@service.gov.uk'.format(i)
         invited_user = create_invited_user(sample_service, to_email_address=email)
-
         invites.append(invited_user)
 
     url = '/service/{}/invite'.format(sample_service.id)

tests/app/test_route_authentication.py

@@ -1,8 +1,3 @@
-
-import pytest
-
-
-@pytest.mark.skip(reason="Needs updating for TTS")
 def test_all_routes_have_authentication(client):
     # This tests that each blueprint registered on the application has a before_request function registered.
     # The None row is removed from the comparison as that is not blueprint specific but app specific.

tests/app/user/test_rest.py

@@ -47,7 +47,6 @@ def test_get_user_list(admin_request, sample_service):
     assert sorted(expected_permissions) == sorted(fetched['permissions'][str(sample_service.id)])
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_get_user(admin_request, sample_service, sample_organisation):
     """
     Tests GET endpoint '/<user_id>' to retrieve a single service.
@@ -123,7 +122,6 @@ def test_post_user(admin_request, notify_db_session):
     assert user.auth_type == EMAIL_AUTH_TYPE
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_post_user_without_auth_type(admin_request, notify_db_session):
     User.query.delete()
     data = {
@@ -618,7 +616,6 @@ def test_send_user_reset_password_should_send_reset_password_link(admin_request,
     assert notification.reply_to_text == notify_service.get_default_reply_to_email_address()
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 @pytest.mark.parametrize('data, expected_url', (
     ({
         'email': 'notify@digital.cabinet-office.gov.uk',
@@ -820,7 +817,6 @@ def test_activate_user_fails_if_already_active(admin_request, sample_user):
     assert sample_user.state == 'active'
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 def test_update_user_auth_type(admin_request, sample_user):
     assert sample_user.auth_type == 'sms_auth'
     resp = admin_request.post(

tests/app/user/test_rest_verify.py

@@ -292,7 +292,6 @@ def test_send_sms_code_returns_204_when_too_many_codes_already_created(client, s
     assert VerifyCode.query.count() == 5
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 @pytest.mark.parametrize('post_data, expected_url_starts_with', (
     (
         {},
@@ -387,7 +386,6 @@ def test_reset_failed_login_count_returns_404_when_user_does_not_exist(client):
     assert resp.status_code == 404
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason")
 # we send sms_auth users and webauthn_auth users email code to validate their email access
 @pytest.mark.parametrize('auth_type', USER_AUTH_TYPES)
 @pytest.mark.parametrize('data, expected_auth_url', (

tests/app/v2/notifications/test_post_notifications.py

@@ -238,7 +238,6 @@ def test_should_cache_template_lookups_in_memory(mocker, client, sample_template
     assert Notification.query.count() == 5
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: cloud.gov redis fails, local docker works, mock redis fails")
 def test_should_cache_template_and_service_in_redis(mocker, client, sample_template):
 
     from app.schemas import service_schema, template_schema
@@ -289,7 +288,6 @@ def test_should_cache_template_and_service_in_redis(mocker, client, sample_templ
     assert templates_call[1]['ex'] == 604_800
 
 
-@pytest.mark.skip(reason="Needs updating for TTS: cloud.gov redis fails, local docker works, mock redis fails")
 def test_should_return_template_if_found_in_redis(mocker, client, sample_template):
 
     from app.schemas import service_schema, template_schema
@@ -313,7 +311,7 @@ def test_should_return_template_if_found_in_redis(mocker, client, sample_templat
     mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
 
     data = {
-        'phone_number': '+447700900855',
+        'phone_number': '+16615555555',
         'template_id': str(sample_template.id),
     }