Merge branch 'main' into update-marshmallow-deps

Bump setuptools from 80.6.0 to 80.7.1

.ds.baseline

@@ -127,262 +127,16 @@
     }
   ],
   "results": {
-    ".github/workflows/checks.yml": [
-      {
-        "type": "Secret Keyword",
-        "filename": ".github/workflows/checks.yml",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 28,
-        "is_secret": false
-      },
-      {
-        "type": "Basic Auth Credentials",
-        "filename": ".github/workflows/checks.yml",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 45,
-        "is_secret": false
-      }
-    ],
-    ".github/workflows/daily_checks.yml": [
-      {
-        "type": "Secret Keyword",
-        "filename": ".github/workflows/daily_checks.yml",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 61,
-        "is_secret": false
-      },
-      {
-        "type": "Basic Auth Credentials",
-        "filename": ".github/workflows/daily_checks.yml",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 77,
-        "is_secret": false
-      }
-    ],
-    "app/enums.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "app/enums.py",
-        "hashed_secret": "12322e07b94ee3c7cd65a2952ece441538b53eb3",
-        "is_verified": false,
-        "line_number": 123,
-        "is_secret": false
-      }
-    ],
-    "app/notifications/receive_notifications.py": [
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "app/notifications/receive_notifications.py",
-        "hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
-        "is_verified": false,
-        "line_number": 29,
-        "is_secret": false
-      }
-    ],
-    "deploy-config/sandbox.yml": [
-      {
-        "type": "Secret Keyword",
-        "filename": "deploy-config/sandbox.yml",
-        "hashed_secret": "113151dd10316fcb0d5507b6215d78e2f3fe9e54",
-        "is_verified": false,
-        "line_number": 11,
-        "is_secret": false
-      }
-    ],
-    "sample.env": [
-      {
-        "type": "Basic Auth Credentials",
-        "filename": "sample.env",
-        "hashed_secret": "5b98cf4c3d794c8af1fcd7991e89cd4e52fb42a4",
-        "is_verified": false,
-        "line_number": 16,
-        "is_secret": false
-      }
-    ],
-    "tests/app/aws/test_s3.py": [
+    ".github/actions/deploy-proxy/action.yml": [
       {
         "type": "Hex High Entropy String",
-        "filename": "tests/app/aws/test_s3.py",
-        "hashed_secret": "67a74306b06d0c01624fe0d0249a570f4d093747",
+        "filename": ".github/actions/deploy-proxy/action.yml",
+        "hashed_secret": "a6c13f5da3788e8d654cd24001dc79a238723248",
         "is_verified": false,
-        "line_number": 42,
-        "is_secret": false
-      }
-    ],
-    "tests/app/clients/test_document_download.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/clients/test_document_download.py",
-        "hashed_secret": "3acfb2c2b433c0ea7ff107e33df91b18e52f960f",
-        "is_verified": false,
-        "line_number": 14,
-        "is_secret": false
-      }
-    ],
-    "tests/app/clients/test_performance_platform.py": [
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "tests/app/clients/test_performance_platform.py",
-        "hashed_secret": "76bb66c38ac4046bf73cd4a2c35a2b0af94aeb61",
-        "is_verified": false,
-        "line_number": 84,
-        "is_secret": false
-      }
-    ],
-    "tests/app/dao/test_services_dao.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/dao/test_services_dao.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 289,
-        "is_secret": false
-      }
-    ],
-    "tests/app/dao/test_users_dao.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/dao/test_users_dao.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 69,
-        "is_secret": false
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/dao/test_users_dao.py",
-        "hashed_secret": "f2c57870308dc87f432e5912d4de6f8e322721ba",
-        "is_verified": false,
-        "line_number": 199,
-        "is_secret": false
-      }
-    ],
-    "tests/app/db.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/db.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 90,
-        "is_secret": false
-      }
-    ],
-    "tests/app/notifications/test_receive_notification.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/notifications/test_receive_notification.py",
-        "hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
-        "is_verified": false,
-        "line_number": 27,
-        "is_secret": false
-      },
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "tests/app/notifications/test_receive_notification.py",
-        "hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
-        "is_verified": false,
-        "line_number": 57,
-        "is_secret": false
-      }
-    ],
-    "tests/app/notifications/test_validators.py": [
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "tests/app/notifications/test_validators.py",
-        "hashed_secret": "6c1a8443963d02d13ffe575a71abe19ea731fb66",
-        "is_verified": false,
-        "line_number": 672,
-        "is_secret": false
-      }
-    ],
-    "tests/app/service/test_rest.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/service/test_rest.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 1285,
-        "is_secret": false
-      }
-    ],
-    "tests/app/test_cloudfoundry_config.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/test_cloudfoundry_config.py",
-        "hashed_secret": "e5e178db7317356946d13e5d2da037d39ac61c71",
-        "is_verified": false,
-        "line_number": 12,
-        "is_secret": false
-      },
-      {
-        "type": "Basic Auth Credentials",
-        "filename": "tests/app/test_cloudfoundry_config.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 14,
-        "is_secret": false
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/test_cloudfoundry_config.py",
-        "hashed_secret": "cfd48edeb81ba7d48cbddcf1eeede25ba67057e8",
-        "is_verified": false,
-        "line_number": 33,
-        "is_secret": false
-      }
-    ],
-    "tests/app/user/test_rest.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/user/test_rest.py",
-        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
-        "is_verified": false,
-        "line_number": 110,
-        "is_secret": false
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/app/user/test_rest.py",
-        "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
-        "is_verified": false,
-        "line_number": 864,
-        "is_secret": false
-      }
-    ],
-    "tests/notifications_utils/clients/antivirus/test_antivirus_client.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/notifications_utils/clients/antivirus/test_antivirus_client.py",
-        "hashed_secret": "932b25270abe1301c22c709a19082dff07d469ff",
-        "is_verified": false,
-        "line_number": 16,
-        "is_secret": false
-      }
-    ],
-    "tests/notifications_utils/clients/encryption/test_encryption_client.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/notifications_utils/clients/encryption/test_encryption_client.py",
-        "hashed_secret": "f1e923a9667de11be6a210849a8651c1bfd81605",
-        "is_verified": false,
-        "line_number": 13,
-        "is_secret": false
-      }
-    ],
-    "tests/notifications_utils/clients/zendesk/test_zendesk_client.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/notifications_utils/clients/zendesk/test_zendesk_client.py",
-        "hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
-        "is_verified": false,
-        "line_number": 16,
+        "line_number": 18,
         "is_secret": false
       }
     ]
   },
-  "generated_at": "2025-02-27T21:09:45Z"
+  "generated_at": "2025-05-12T16:45:34Z"
 }

.github/actions/deploy-proxy/action.yml

@@ -15,7 +15,7 @@ inputs:
     default: https://github.com/GSA-TTS/cg-egress-proxy.git
   proxy_version:
     description: git ref to be deployed
-    default: main
+    default: 1500c67157c1a7a6fbbda7a2de172b3d0a67e703
 runs:
   using: composite
   steps:

.profile

@@ -6,4 +6,4 @@
 export http_proxy=$egress_proxy
 export https_proxy=$egress_proxy
 export NEW_RELIC_PROXY_HOST=$egress_proxy
-export no_proxy="apps.internal"
+export no_proxy="apps.internal,s3-fips.us-gov-west-1.amazonaws.com"

Makefile

@@ -110,6 +110,10 @@ test: ## Run tests and create coverage report
 	poetry run coverage report -m --fail-under=93
 	poetry run coverage html -d .coverage_cache
 
+.PHONY: test-debug
+test-debug:
+	poetry run pytest --pdb -x
+
 .PHONY: py-lock
 py-lock: ## Syncs dependencies and updates lock file without performing recursive internal updates
 	poetry lock --no-update

app/aws/s3.py

@@ -2,10 +2,10 @@ import csv
 import datetime
 import re
 import time
-from concurrent.futures import ThreadPoolExecutor
 from io import StringIO
 
 import botocore
+import eventlet
 from boto3 import Session
 from flask import current_app
 
@@ -24,8 +24,15 @@ s3_client = None
 s3_resource = None
 
 
+def get_service_id_from_key(key):
+    key = key.replace("service-", "")
+    key = key.split("/")
+    key = key[0].replace("-notify", "")
+    return key
+
+
 def set_job_cache(key, value):
-    current_app.logger.debug(f"Setting {key} in the job_cache.")
+    current_app.logger.debug(f"Setting {key} in the job_cache to {value}.")
     job_cache = current_app.config["job_cache"]
     job_cache[key] = (value, time.time() + 8 * 24 * 60 * 60)
 
@@ -36,7 +43,7 @@ def get_job_cache(key):
     if ret is None:
         current_app.logger.warning(f"Could not find {key} in the job_cache.")
     else:
-        current_app.logger.debug(f"Got {key} from job_cache.")
+        current_app.logger.debug(f"Got {key} from job_cache with value {ret}.")
     return ret
 
 
@@ -116,9 +123,9 @@ def list_s3_objects():
                 )
             else:
                 break
-    except Exception:
+    except Exception as e:
         current_app.logger.exception(
-            "An error occurred while regenerating cache #notify-debug-admin-1200",
+            f"An error occurred while regenerating cache #notify-debug-admin-1200: {str(e)}",
         )
 
 
@@ -184,18 +191,20 @@ def read_s3_file(bucket_name, object_key, s3res):
     """
     try:
         job_id = get_job_id_from_s3_object_key(object_key)
+        service_id = get_service_id_from_key(object_key)
+
         if get_job_cache(job_id) is None:
-            object = (
+            job = (
                 s3res.Object(bucket_name, object_key)
                 .get()["Body"]
                 .read()
                 .decode("utf-8")
             )
-            set_job_cache(job_id, object)
-            set_job_cache(f"{job_id}_phones", extract_phones(object))
+            set_job_cache(job_id, job)
+            set_job_cache(f"{job_id}_phones", extract_phones(job, service_id, job_id))
             set_job_cache(
                 f"{job_id}_personalisation",
-                extract_personalisation(object),
+                extract_personalisation(job),
             )
 
     except LookupError:
@@ -216,8 +225,8 @@ def get_s3_files():
         f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200"
     )
     try:
-        with ThreadPoolExecutor() as executor:
-            executor.map(lambda key: read_s3_file(bucket_name, key, s3res), object_keys)
+        for object_key in object_keys:
+            read_s3_file(bucket_name, object_key, s3res)
     except Exception:
         current_app.logger.exception("Connection pool issue")
 
@@ -375,7 +384,7 @@ def get_job_from_s3(service_id, job_id):
                 )
                 retries += 1
                 sleep_time = backoff_factor * (2**retries)  # Exponential backoff
-                time.sleep(sleep_time)
+                eventlet.sleep(sleep_time)
                 continue
             else:
                 # Typically this is "NoSuchKey"

app/celery/tasks.py

@@ -1,6 +1,6 @@
 import json
-from time import sleep
 
+import eventlet
 from celery.signals import task_postrun
 from flask import current_app
 from requests import HTTPError, RequestException, request
@@ -83,7 +83,7 @@ def process_job(job_id, sender_id=None):
         process_row(row, template, job, service, sender_id=sender_id)
         count = count + 1
         if count % 3 == 0:
-            sleep(1)
+            eventlet.sleep(1)
 
     # End point/Exit point for message send flow.
     job_complete(job, start=start)

app/clients/cloudwatch/aws_cloudwatch.py

@@ -151,7 +151,7 @@ class AwsCloudwatchClient(Client):
     #         result = temp_client.get_query_results(queryId=query_id)
     #         if result['status'] == 'Complete':
     #             break
-    #         time.sleep(1)
+    #         eventlet.sleep(1)
 
     #     delivery_receipts = []
     #     for log in result['results']:

app/notifications/rest.py

@@ -15,9 +15,9 @@ from app.notifications.validators import (
     service_has_permission,
     validate_template,
 )
+from app.public_schemas.public import PublicNotificationResponseSchema
 from app.schemas import (
     email_notification_schema,
-    notification_with_personalisation_schema,
     notifications_filter_schema,
     sms_template_notification_schema,
 )
@@ -35,6 +35,7 @@ def get_notification_by_id(notification_id):
     notification = notifications_dao.get_notification_with_personalisation(
         str(authenticated_service.id), notification_id, key_type=None
     )
+
     if notification.job_id is not None:
         notification.personalisation = get_personalisation_from_s3(
             notification.service_id,
@@ -48,16 +49,9 @@ def get_notification_by_id(notification_id):
         )
         notification.to = recipient
         notification.normalised_to = recipient
-    return (
-        jsonify(
-            data={
-                "notification": notification_with_personalisation_schema.dump(
-                    notification
-                )
-            }
-        ),
-        200,
-    )
+
+    serialized = PublicNotificationResponseSchema().dump(notification)
+    return jsonify(data={"notification": serialized}), 200
 
 
 @notifications.route("/notifications", methods=["GET"])
@@ -99,7 +93,7 @@ def get_all_notifications():
             notification.normalised_to = recipient
 
     result = jsonify(
-        notifications=notification_with_personalisation_schema.dump(
+        notifications=PublicNotificationResponseSchema().dump(
             pagination.items, many=True
         ),
         page_size=page_size,

app/public_schemas/public.py

@@ -0,0 +1,100 @@
+from datetime import timezone, datetime
+from uuid import UUID
+
+from marshmallow import EXCLUDE, Schema, fields, post_dump, pre_dump
+
+from app.schemas import FlexibleDateTime, JobSchema, TemplateSchema
+from app import ma
+
+class PublicTemplateSchema(Schema):
+    id = fields.UUID(required=True)
+    name = fields.String(required=True)
+    template_type = fields.String(required=True)
+    version = fields.Integer(required=True)
+
+
+class PublicJobSchema(Schema):
+    id = fields.UUID(required=True)
+    original_file_name = fields.String(required=True)
+
+
+class PublicNotificationSchema(Schema):
+    id = fields.UUID(required=True)
+    to = fields.String(required=True)
+    job_row_number = fields.Integer(allow_none=True)
+    template_version = fields.Integer(required=True)
+    billable_units = fields.Integer(required=True)
+    notification_type = fields.String(required=True)
+    created_at = fields.String(required=True)
+    sent_at = fields.String(allow_none=True)
+    updated_at = fields.String(allow_none=True)
+    sent_by = fields.String(allow_none=True)
+    status = fields.String(required=True)
+    reference = fields.String(allow_none=True)
+    template = fields.Nested(PublicTemplateSchema, required=True)
+    service = fields.UUID(required=True)
+    job = fields.Nested(PublicJobSchema, allow_none=True)
+    api_key = fields.UUID(allow_none=True)
+    body = fields.String(required=True)
+    content_char_count = fields.Integer(required=True)
+
+
+class PublicNotificationResponseSchema(PublicNotificationSchema):
+    class Meta:
+        unknown = EXCLUDE
+
+    @post_dump
+    def transform(self, data, **kwargs):
+        def to_rfc3339(dt):
+            if dt is None:
+                return None
+            if isinstance(dt, str):
+                try:
+                    dt = datetime.fromisoformat(dt)
+                except ValueError:
+                    return dt  # fallback, might already be valid
+            if dt.tzinfo is None:
+                dt = dt.replace(tzinfo=timezone.utc)
+            return dt.isoformat().replace("+00:00", "Z")
+
+        data["created_at"] = to_rfc3339(data.get("created_at"))
+        data["sent_at"] = to_rfc3339(data.get("sent_at"))
+        data["updated_at"] = to_rfc3339(data.get("updated_at"))
+
+        # Fallback content
+        template = data.get("template", {})
+        body = data.get("body") or (template.get("content") if isinstance(template, dict) else "")
+        data["body"] = body or ""
+        data["content_char_count"] = len(data["body"])
+
+        # Extract UUID string for service
+        service = data.get("service")
+        if hasattr(service, "id"):
+            data["service"] = str(service.id)
+        elif isinstance(service, UUID):
+            data["service"] = str(service)
+        elif isinstance(service, str) and service.startswith("<Service "):
+            # fallback if __str__ was called on the SQLAlchemy object
+            data["service"] = service.split("<Service ")[1].rstrip(">")
+        else:
+            data["service"] = str(service)  # best effort fallback
+
+        # Extract UUID string for api_key
+        api_key = data.get("api_key")
+        if hasattr(api_key, "id"):
+            data["api_key"] = str(api_key.id)
+        elif isinstance(api_key, UUID):
+            data["api_key"] = str(api_key)
+        elif isinstance(api_key, str) and api_key.startswith("<ApiKey "):
+            data["api_key"] = api_key.split("<ApiKey ")[1].rstrip(">")
+        else:
+            data["api_key"] = str(api_key) if api_key else None
+
+        # Fix job dict
+        job = data.get("job")
+        if isinstance(job, dict) and "id" in job:
+            job_id = job.get("id")
+            job["id"] = str(job_id) if job_id else None
+            data["job"] = job
+
+        return data

app/schemas.py

@@ -2,6 +2,7 @@ from datetime import timedelta
 from uuid import UUID
 
 from dateutil.parser import parse
+from flask import current_app
 from flask_marshmallow.fields import fields
 from marshmallow import (
     EXCLUDE,
@@ -17,7 +18,7 @@ from marshmallow_sqlalchemy import auto_field, field_for
 
 from app import ma, models
 from app.dao.permissions_dao import permission_dao
-from app.enums import ServicePermissionType, TemplateType
+from app.enums import NotificationStatus, ServicePermissionType, TemplateType
 from app.models import ServicePermission
 from app.utils import DATETIME_FORMAT_NO_TIMEZONE, get_template_instance, utc_now
 from notifications_utils.recipients import (
@@ -113,7 +114,7 @@ class UserSchema(BaseSchema):
     )
     updated_at = FlexibleDateTime()
     logged_in_at = FlexibleDateTime()
-    auth_type = field_for(models.User, "auth_type")
+    auth_type = auto_field(by_value=True)
     password = fields.String(required=True, load_only=True)
 
     def user_permissions(self, usr):
@@ -136,28 +137,31 @@ class UserSchema(BaseSchema):
         )
 
     @validates("name")
-    def validate_name(self, value):
+    def validate_name(self, value, data_key):
         if not value:
+            current_app.logger.exception(f"{data_key}: Invalid name")
             raise ValidationError("Invalid name")
 
     @validates("email_address")
-    def validate_email_address(self, value):
+    def validate_email_address(self, value, data_key):
         try:
             validate_email_address(value)
         except InvalidEmailError as e:
+            current_app.logger.exception(f"{data_key}: {str(e)}")
             raise ValidationError(str(e))
 
     @validates("mobile_number")
-    def validate_mobile_number(self, value):
+    def validate_mobile_number(self, value, data_key):
         try:
             if value is not None:
                 validate_phone_number(value, international=True)
         except InvalidPhoneError as error:
-            raise ValidationError(f"Invalid phone number: {error}")
+            current_app.logger.exception(f"{data_key}: {str(error)}")
+            raise ValidationError(f"Invalid phone number: {str(error)}")
 
 
 class UserUpdateAttributeSchema(BaseSchema):
-    auth_type = field_for(models.User, "auth_type")
+    auth_type = auto_field(by_value=True)
     email_access_validated_at = FlexibleDateTime()
 
     class Meta(BaseSchema.Meta):
@@ -176,24 +180,29 @@ class UserUpdateAttributeSchema(BaseSchema):
         )
 
     @validates("name")
-    def validate_name(self, value):
+    def validate_name(self, value, data_key):
         if not value:
+            current_app.logger.exception(f"{data_key}: Invalid name")
             raise ValidationError("Invalid name")
 
     @validates("email_address")
-    def validate_email_address(self, value):
+    def validate_email_address(self, value, data_key):
         try:
             validate_email_address(value)
         except InvalidEmailError as e:
+            current_app.logger.exception(f"{data_key}: {str(e)}")
             raise ValidationError(str(e))
 
     @validates("mobile_number")
-    def validate_mobile_number(self, value):
+    def validate_mobile_number(self, value, data_key):
         try:
             if value is not None:
                 validate_phone_number(value, international=True)
         except InvalidPhoneError as error:
-            raise ValidationError(f"Invalid phone number: {error}")
+            current_app.logger.exception(
+                f"{data_key}: Invalid phone number ({str(error)})"
+            )
+            raise ValidationError(f"Invalid phone number: {str(error)}")
 
     @validates_schema(pass_original=True)
     def check_unknown_fields(self, data, original_data, **kwargs):
@@ -282,14 +291,20 @@ class ServiceSchema(BaseSchema, UUIDsAsStringsMixin):
         )
 
     @validates("permissions")
-    def validate_permissions(self, value):
+    def validate_permissions(self, value, data_key):
         permissions = [v.permission for v in value]
         for p in permissions:
             if p not in {e for e in ServicePermissionType}:
+                current_app.logger.exception(
+                    f"{data_key}: Invalid Service Permission: '{p}'"
+                )
                 raise ValidationError(f"Invalid Service Permission: '{p}'")
 
         if len(set(permissions)) != len(permissions):
             duplicates = list(set([x for x in permissions if permissions.count(x) > 1]))
+            current_app.logger.exception(
+                f"{data_key}: Duplicate Service Permission: {duplicates}"
+            )
             raise ValidationError(f"Duplicate Service Permission: {duplicates}")
 
     @pre_load()
@@ -350,7 +365,7 @@ class NotificationModelSchema(BaseSchema):
             "api_key",
         )
 
-    status = fields.String(required=False)
+    status = auto_field(by_value=True)
     created_at = FlexibleDateTime()
     sent_at = FlexibleDateTime()
     updated_at = FlexibleDateTime()
@@ -359,6 +374,7 @@ class NotificationModelSchema(BaseSchema):
 class BaseTemplateSchema(BaseSchema):
     reply_to = fields.Method("get_reply_to", allow_none=True)
     reply_to_text = fields.Method("get_reply_to_text", allow_none=True)
+    template_type = auto_field(by_value=True)
 
     def get_reply_to(self, template):
         return template.reply_to
@@ -373,7 +389,7 @@ class BaseTemplateSchema(BaseSchema):
 
 class TemplateSchema(BaseTemplateSchema, UUIDsAsStringsMixin):
     created_by = field_for(models.Template, "created_by", required=True)
-    process_type = field_for(models.Template, "process_type")
+    process_type = auto_field(by_value=True)
     redact_personalisation = fields.Method("redact")
     created_at = FlexibleDateTime()
     updated_at = FlexibleDateTime()
@@ -419,6 +435,7 @@ class TemplateHistorySchema(BaseSchema):
     reply_to = fields.Method("get_reply_to", allow_none=True)
     reply_to_text = fields.Method("get_reply_to_text", allow_none=True)
     process_type = field_for(models.Template, "process_type")
+    template_type = auto_field(by_value=True)
 
     created_by = fields.Nested(
         UserSchema, only=["id", "name", "email_address"], dump_only=True
@@ -440,7 +457,7 @@ class TemplateHistorySchema(BaseSchema):
 
 class ApiKeySchema(BaseSchema):
     created_by = field_for(models.ApiKey, "created_by", required=True)
-    key_type = field_for(models.ApiKey, "key_type", required=True)
+    key_type = auto_field(by_value=True)
     expiry_date = FlexibleDateTime()
     created_at = FlexibleDateTime()
     updated_at = FlexibleDateTime()
@@ -464,7 +481,7 @@ class JobSchema(BaseSchema):
     processing_started = FlexibleDateTime()
     processing_finished = FlexibleDateTime()
 
-    job_status = auto_field()
+    job_status = auto_field(by_value=True)
 
     scheduled_for = FlexibleDateTime()
     service_name = fields.Nested(
@@ -482,10 +499,10 @@ class JobSchema(BaseSchema):
         return job.template.name
 
     def get_template_type(self, job):
-        return job.template.template_type
+        return job.template.template_type.value
 
     @validates("scheduled_for")
-    def validate_scheduled_for(self, value):
+    def validate_scheduled_for(self, value, data_key):
         _validate_datetime_not_in_past(value)
         _validate_datetime_not_more_than_96_hours_in_future(value)
 
@@ -503,7 +520,7 @@ class NotificationSchema(ma.Schema):
     class Meta:
         unknown = EXCLUDE
 
-    status = fields.String(required=False)
+    status = fields.Enum(NotificationStatus, by_value=True, required=False)
     personalisation = fields.Dict(required=False)
 
 
@@ -511,11 +528,14 @@ class SmsNotificationSchema(NotificationSchema):
     to = fields.Str(required=True)
 
     @validates("to")
-    def validate_to(self, value):
+    def validate_to(self, value, data_key):
         try:
             validate_phone_number(value, international=True)
         except InvalidPhoneError as error:
-            raise ValidationError("Invalid phone number: {}".format(error))
+            current_app.logger.exception(
+                f"{data_key}: Invalid phone number ({str(error)}"
+            )
+            raise ValidationError(f"Invalid phone number: {str(error)}")
 
     @post_load
     def format_phone_number(self, item, **kwargs):
@@ -528,10 +548,11 @@ class EmailNotificationSchema(NotificationSchema):
     template = fields.Str(required=True)
 
     @validates("to")
-    def validate_to(self, value):
+    def validate_to(self, value, data_key):
         try:
             validate_email_address(value)
         except InvalidEmailError as e:
+            current_app.logger.exception(f"{data_key}: {str(e)}")
             raise ValidationError(str(e))
 
 
@@ -542,6 +563,7 @@ class SmsTemplateNotificationSchema(SmsNotificationSchema):
 
 class NotificationWithTemplateSchema(BaseSchema):
     class Meta(BaseSchema.Meta):
+        unknown = EXCLUDE
         model = models.Notification
         exclude = ("_personalisation",)
 
@@ -558,13 +580,15 @@ class NotificationWithTemplateSchema(BaseSchema):
         ],
         dump_only=True,
     )
+    template_version = fields.Integer()
     job = fields.Nested(JobSchema, only=["id", "original_file_name"], dump_only=True)
     created_by = fields.Nested(
         UserSchema, only=["id", "name", "email_address"], dump_only=True
     )
-    status = fields.String(required=False)
+    status = auto_field(by_value=True)
     personalisation = fields.Dict(required=False)
-    key_type = field_for(models.Notification, "key_type", required=True)
+    notification_type = auto_field(by_value=True)
+    key_type = auto_field(by_value=True)
     key_name = fields.String()
     created_at = FlexibleDateTime()
     updated_at = FlexibleDateTime()
@@ -579,80 +603,20 @@ class NotificationWithTemplateSchema(BaseSchema):
         return in_data
 
 
-class NotificationWithPersonalisationSchema(NotificationWithTemplateSchema):
-    template_history = fields.Nested(
-        TemplateHistorySchema,
-        attribute="template",
-        only=["id", "name", "template_type", "content", "subject", "version"],
-        dump_only=True,
-    )
-
-    class Meta(NotificationWithTemplateSchema.Meta):
-        # mark as many fields as possible as required since this is a public api.
-        # WARNING: Does _not_ reference fields computed in handle_template_merge, such as
-        # 'body', 'subject' [for emails], and 'content_char_count'
-        fields = (
-            # db rows
-            "billable_units",
-            "created_at",
-            "id",
-            "job_row_number",
-            "notification_type",
-            "reference",
-            "sent_at",
-            "sent_by",
-            "status",
-            "template_version",
-            "to",
-            "updated_at",
-            # computed fields
-            "personalisation",
-            # relationships
-            "api_key",
-            "job",
-            "service",
-            "template_history",
-        )
-        # Overwrite the `NotificationWithTemplateSchema` base class to not exclude `_personalisation`, which
-        # isn't a defined field for this class
-        exclude = ()
-
-    @pre_dump
-    def handle_personalisation_property(self, in_data, **kwargs):
-        self.personalisation = in_data.personalisation
-        return in_data
-
-    @post_dump
-    def handle_template_merge(self, in_data, **kwargs):
-        in_data["template"] = in_data.pop("template_history")
-        template = get_template_instance(
-            in_data["template"], in_data["personalisation"]
-        )
-        in_data["body"] = template.content_with_placeholders_filled_in
-        if in_data["template"]["template_type"] != TemplateType.SMS:
-            in_data["subject"] = template.subject
-            in_data["content_char_count"] = None
-        else:
-            in_data["content_char_count"] = template.content_count
-
-        in_data.pop("personalisation", None)
-        in_data["template"].pop("content", None)
-        in_data["template"].pop("subject", None)
-        return in_data
-
-
 class InvitedUserSchema(BaseSchema):
-    auth_type = field_for(models.InvitedUser, "auth_type")
+    auth_type = auto_field(by_value=True)
     created_at = FlexibleDateTime()
+    status = auto_field(by_value=True)
 
     class Meta(BaseSchema.Meta):
         model = models.InvitedUser
 
     @validates("email_address")
-    def validate_to(self, value):
+    def validate_to(self, value, data_key):
         try:
             validate_email_address(value)
         except InvalidEmailError as e:
+            current_app.logger.exception(f"{data_key}: {str(e)}")
             raise ValidationError(str(e))
 
 
@@ -669,12 +633,13 @@ class EmailDataSchema(ma.Schema):
         self.partial_email = partial_email
 
     @validates("email")
-    def validate_email(self, value):
+    def validate_email(self, value, data_key):
         if self.partial_email:
             return
         try:
             validate_email_address(value)
         except InvalidEmailError as e:
+            current_app.logger.exception(f"{data_key}: {str(e)}")
             raise ValidationError(str(e))
 
 
@@ -719,11 +684,11 @@ class NotificationsFilterSchema(ma.Schema):
         return in_data
 
     @validates("page")
-    def validate_page(self, value):
+    def validate_page(self, value, data_key):
         _validate_positive_number(value)
 
     @validates("page_size")
-    def validate_page_size(self, value):
+    def validate_page_size(self, value, data_key):
         _validate_positive_number(value)
 
 
@@ -789,7 +754,6 @@ sms_template_notification_schema = SmsTemplateNotificationSchema()
 email_notification_schema = EmailNotificationSchema()
 notification_schema = NotificationModelSchema()
 notification_with_template_schema = NotificationWithTemplateSchema()
-notification_with_personalisation_schema = NotificationWithPersonalisationSchema()
 invited_user_schema = InvitedUserSchema()
 email_data_request_schema = EmailDataSchema()
 partial_email_data_request_schema = EmailDataSchema(partial_email=True)

pyproject.toml

@@ -18,26 +18,26 @@ celery = {version = "==5.5.2", extras = ["redis"]}
 certifi = ">=2022.12.7"
 cffi = "==1.17.1"
 charset-normalizer = "^3.4.2"
-click = "==8.1.8"
+click = "==8.2.0"
 click-datetime = "==0.4.0"
 click-didyoumean = "==0.3.1"
 click-plugins = "==1.1.1"
 click-repl = "==0.3.0"
 deprecated = "==1.2.14"
-eventlet = "==0.39.1"
+eventlet = "==0.36.1"
 expiringdict = "==1.2.2"
-flask = "~=3.0"
+flask = "~=3.1"
 flask-bcrypt = "==1.0.1"
-flask-marshmallow = "==1.2.1"
+flask-marshmallow = "^1.3.0"
 flask-migrate = "==4.1.0"
 flask-redis = "==0.4.0"
-flask-sqlalchemy = "==3.1.1"
+flask-sqlalchemy = "^3.1.1"
 gunicorn = {version = "==23.0.0", extras = ["eventlet"]}
 iso8601 = "==2.1.0"
 jsonschema = {version = "==4.23.0", extras = ["format"]}
 lxml = "==5.4.0"
-marshmallow = "==3.26.1"
-marshmallow-sqlalchemy = "==1.0.0"
+marshmallow = "^4.0.0"
+marshmallow-sqlalchemy = "^1.4.2"
 newrelic = "*"
 notifications-python-client = "==10.0.1"
 oscrypto = { git = "https://github.com/wbond/oscrypto.git", rev = "1547f53" }
@@ -46,15 +46,15 @@ poetry-dotenv-plugin = "==0.2.0"
 psycopg2-binary = "==2.9.9"
 pyjwt = "==2.10.1"
 python-dotenv = "==1.1.0"
-sqlalchemy = "==2.0.40"
+sqlalchemy = "==2.0.41"
 werkzeug = "^3.0.6"
-faker = "^37.1.0"
+faker = "^37.3.0"
 async-timeout = "^5.0.1"
 bleach = "^6.1.0"
 geojson = "^3.2.0"
 numpy = "^2.2.5"
 ordered-set = "^4.1.0"
-phonenumbers = "^9.0.4"
+phonenumbers = "^9.0.5"
 python-json-logger = "^3.3.0"
 regex = "^2024.11.6"
 shapely = "^2.0.5"
@@ -85,7 +85,7 @@ awscli = "^1.33.32"
 bandit = "*"
 black = "^25.1.0"
 cloudfoundry-client = "*"
-exceptiongroup = "==1.2.2"
+exceptiongroup = "==1.3.0"
 flake8 = "^7.2.0"
 flake8-bugbear = "^24.12.12"
 freezegun = "^1.5.1"
@@ -102,7 +102,7 @@ pytest-cov = "^6.1.1"
 pytest-xdist = "^3.5.0"
 radon = "^6.0.1"
 requests-mock = "^1.11.0"
-setuptools = "^80.3.1"
+setuptools = "^80.7.1"
 sqlalchemy-utils = "^0.41.2"
 vulture = "^2.10"
 detect-secrets = "^1.5.0"

tests/app/aws/test_s3.py

@@ -39,7 +39,7 @@ default_region = getenv("CSV_AWS_REGION")
 
 def single_s3_object_stub(key="foo", last_modified=None):
     return {
-        "ETag": '"d41d8cd98f00b204e9800998ecf8427e"',
+        "ETag": '"d"',
         "Key": key,
         "LastModified": last_modified or utc_now(),
     }
@@ -420,29 +420,17 @@ def test_get_s3_files_success(client, mocker):
         "CSV_UPLOAD_BUCKET": {"bucket": "test-bucket"},
         "job_cache": {},
     }
-    mock_thread_pool_executor = mocker.patch("app.aws.s3.ThreadPoolExecutor")
     mock_read_s3_file = mocker.patch("app.aws.s3.read_s3_file")
     mock_list_s3_objects = mocker.patch("app.aws.s3.list_s3_objects")
     mock_get_s3_resource = mocker.patch("app.aws.s3.get_s3_resource")
     mock_list_s3_objects.return_value = ["file1.csv", "file2.csv"]
     mock_s3_resource = MagicMock()
     mock_get_s3_resource.return_value = mock_s3_resource
-    mock_executor = MagicMock()
-
-    def mock_map(func, iterable):
-        for item in iterable:
-            func(item)
-
-    mock_executor.map.side_effect = mock_map
-    mock_thread_pool_executor.return_value.__enter__.return_value = mock_executor
 
     get_s3_files()
 
     # mock_current_app.config.__getitem__.assert_called_once_with("CSV_UPLOAD_BUCKET")
     mock_list_s3_objects.assert_called_once()
-    mock_thread_pool_executor.assert_called_once()
-
-    mock_executor.map.assert_called_once()
 
     calls = [
         (("test-bucket", "file1.csv", mock_s3_resource),),
@@ -615,15 +603,6 @@ def test_get_s3_files_handles_exception(mocker):
     mock_read_s3_file = mocker.patch(
         "app.aws.s3.read_s3_file", side_effect=[None, Exception("exception here")]
     )
-
-    mock_thread_pool_executor = mocker.patch("app.aws.s3.ThreadPoolExecutor")
-    mock_executor = mock_thread_pool_executor.return_value.__enter__.return_value
-
-    def mock_map(func, iterable):
-        for item in iterable:
-            func(item)
-
-    mock_executor.map.side_effect = mock_map
     get_s3_files()
 
     calls = [

tests/app/notifications/test_rest.py

@@ -558,8 +558,8 @@ def test_get_notification_by_id_returns_merged_template_content(
 def test_get_notification_by_id_returns_merged_template_content_for_email(
     client, sample_email_template_with_placeholders, mocker
 ):
-    mock_s3 = mocker.patch("app.notifications.rest.get_personalisation_from_s3")
-    mock_s3.return_value = {"name": "foo"}
+    # mock_s3 = mocker.patch("app.notifications.rest.get_personalisation_from_s3")
+    # mock_s3.return_value = {"name": "foo"}
     sample_notification = create_notification(
         sample_email_template_with_placeholders, personalisation={"name": "world"}
     )

tests/app/public_contracts/test_GET_notification.py

@@ -24,8 +24,6 @@ def _get_notification(client, notification, url):
 
 
 # v0
-
-
 def test_get_api_sms_contract(client, sample_notification):
     response_json = return_json_from_response(
         _get_notification(
@@ -37,18 +35,6 @@ def test_get_api_sms_contract(client, sample_notification):
     validate_v0(response_json, "GET_notification_return_sms.json")
 
 
-@pytest.mark.skip(reason="Update to fetch email from s3")
-def test_get_api_email_contract(client, sample_email_notification):
-    response_json = return_json_from_response(
-        _get_notification(
-            client,
-            sample_email_notification,
-            "/notifications/{}".format(sample_email_notification.id),
-        )
-    )
-    validate_v0(response_json, "GET_notification_return_email.json")
-
-
 def test_get_job_sms_contract(client, sample_notification):
     response_json = return_json_from_response(
         _get_notification(
@@ -60,22 +46,12 @@ def test_get_job_sms_contract(client, sample_notification):
     validate_v0(response_json, "GET_notification_return_sms.json")
 
 
-@pytest.mark.skip(reason="Update to fetch email from s3")
-def test_get_job_email_contract(client, sample_email_notification):
-    response_json = return_json_from_response(
-        _get_notification(
-            client,
-            sample_email_notification,
-            "/notifications/{}".format(sample_email_notification.id),
-        )
-    )
-    validate_v0(response_json, "GET_notification_return_email.json")
 
-
-def test_get_notifications_contract(
-    client, sample_notification, sample_email_notification
-):
+def test_get_notifications_contract(client, sample_notification, sample_email_notification):
     response_json = return_json_from_response(
         _get_notification(client, sample_notification, "/notifications")
     )
+    notifications = response_json["notifications"]
+    assert notifications, "No notifications returned"
+    assert notifications[0]["template"]["template_type"] == "sms"
     validate_v0(response_json, "GET_notifications_return.json")

tests/app/test_schemas.py

@@ -48,7 +48,7 @@ def test_notification_schema_adds_api_key_name(sample_notification):
         "notification_with_template_schema",
         "notification_schema",
         "notification_with_template_schema",
-        "notification_with_personalisation_schema",
+        "public_notification_response_schema",
     ],
 )
 def test_notification_schema_has_correct_status(sample_notification, schema_name):