Merge pull request #1745 from GSA/pythonupgrade

python upgrade to 3.13.2

.ds.baseline

@@ -127,6 +127,16 @@
     }
   ],
   "results": {
+    ".github/actions/deploy-proxy/action.yml": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": ".github/actions/deploy-proxy/action.yml",
+        "hashed_secret": "a6c13f5da3788e8d654cd24001dc79a238723248",
+        "is_verified": false,
+        "line_number": 18,
+        "is_secret": false
+      }
+    ],
     ".github/workflows/checks.yml": [
       {
         "type": "Secret Keyword",
@@ -151,7 +161,7 @@
         "filename": ".github/workflows/daily_checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 61,
+        "line_number": 71,
         "is_secret": false
       },
       {
@@ -159,7 +169,7 @@
         "filename": ".github/workflows/daily_checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 77,
+        "line_number": 87,
         "is_secret": false
       }
     ],
@@ -203,16 +213,6 @@
         "is_secret": false
       }
     ],
-    "tests/app/aws/test_s3.py": [
-      {
-        "type": "Hex High Entropy String",
-        "filename": "tests/app/aws/test_s3.py",
-        "hashed_secret": "67a74306b06d0c01624fe0d0249a570f4d093747",
-        "is_verified": false,
-        "line_number": 42,
-        "is_secret": false
-      }
-    ],
     "tests/app/clients/test_document_download.py": [
       {
         "type": "Secret Keyword",
@@ -384,5 +384,5 @@
       }
     ]
   },
-  "generated_at": "2025-02-27T21:09:45Z"
+  "generated_at": "2025-06-02T13:22:36Z"
 }

.github/actions/deploy-proxy/action.yml

@@ -15,7 +15,7 @@ inputs:
     default: https://github.com/GSA-TTS/cg-egress-proxy.git
   proxy_version:
     description: git ref to be deployed
-    default: main
+    default: 1500c67157c1a7a6fbbda7a2de172b3d0a67e703
 runs:
   using: composite
   steps:

.github/actions/setup-project/action.yml

@@ -9,13 +9,16 @@ runs:
         sudo apt-get update \
         && sudo apt-get install -y --no-install-recommends \
         libcurl4-openssl-dev
-    - name: Set up Python 3.12.3
+    - name: Set up Python 3.13.2
       uses: actions/setup-python@v4
       with:
-        python-version: "3.12.3"
+        python-version: "3.13.2"
     - name: Install poetry
       shell: bash
-      run: pip install poetry==1.8.5
+      run: pip install poetry==2.1.3
+    - name: Install poetry export
+      shell: bash
+      run: poetry self add poetry-plugin-export
     - name: Downgrade virtualenv to compatible version
       shell: bash
       run: pip install "virtualenv<20.30"

.github/dependabot.yml

@@ -11,3 +11,7 @@ updates:
       interval: "daily"
     labels:
       - "dependabot" # Custom label to identify Dependabot PRs
+    assignees:
+      - "alexjanousekGSA"
+    reviewers:
+      - "alexjanousekGSA"

.github/workflows/checks.yml

@@ -87,12 +87,20 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
-        run: poetry export --without-hashes --format=requirements.txt > requirements.txt
-      - uses: pypa/gh-action-pip-audit@v1.0.8
+        run: poetry export --output requirements_tmp.txt --without-hashes
+      - name: Filter requirements.txt
+        run: grep -v "oscrypto@ git" requirements_tmp.txt > requirements.txt
+      - name: Verify requirements.txt
+        run: ls -l requirements.txt
+      - name: Print requirements.txt
+        run: |
+          echo "Contents of requirements.txt:"
+          cat requirements.txt
+      - uses: pypa/gh-action-pip-audit@v1.1.0
         with:
           inputs: requirements.txt
           ignore-vulns: |
-            PYSEC-2022-43162
+            PYSEC-2023-312
 
   static-scan:
     runs-on: ubuntu-latest

.github/workflows/daily_checks.yml

@@ -26,10 +26,20 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
-        run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+        run: poetry export --output requirements_tmp.txt --without-hashes
+      - name: Filter requirements.txt
+        run: grep -v "oscrypto@ git" requirements_tmp.txt > requirements.txt
+      - name: Verify requirements.txt
+        run: ls -l requirements.txt
+      - name: Print requirements.txt
+        run: |
+          echo "Contents of requirements.txt:"
+          cat requirements.txt
       - uses: pypa/gh-action-pip-audit@v1.1.0
         with:
           inputs: requirements.txt
+          ignore-vulns: |
+            PYSEC-2023-312
       - name: Upload pip-audit artifact
         uses: actions/upload-artifact@v4
         with:

.github/workflows/deploy-demo.yml

@@ -44,7 +44,7 @@ jobs:
       run: make bootstrap
 
     - name: Create requirements.txt
-      run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+      run: poetry export --output requirements.txt
 
     - name: Deploy to cloud.gov
       uses: cloud-gov/cg-cli-tools@main

.github/workflows/deploy-prod.yml

@@ -48,7 +48,7 @@ jobs:
       run: make bootstrap
 
     - name: Create requirements.txt
-      run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+      run: poetry export --output requirements.txt
 
     - name: Deploy to cloud.gov
       uses: cloud-gov/cg-cli-tools@main

.github/workflows/deploy.yml

@@ -50,7 +50,7 @@ jobs:
       run: make bootstrap
 
     - name: Create requirements.txt
-      run: poetry export --without-hashes --format=requirements.txt > requirements.txt
+      run: poetry export --output requirements.txt
 
     - name: Deploy to cloud.gov
       uses: cloud-gov/cg-cli-tools@main

.profile

@@ -6,4 +6,4 @@
 export http_proxy=$egress_proxy
 export https_proxy=$egress_proxy
 export NEW_RELIC_PROXY_HOST=$egress_proxy
-export no_proxy="apps.internal"
+export no_proxy="apps.internal,s3-fips.us-gov-west-1.amazonaws.com"

Makefile

@@ -16,8 +16,7 @@ GIT_HOOKS_PATH ?= $(shell git config --global core.hooksPath || echo "")
 .PHONY: bootstrap
 bootstrap: ## Set up everything to run the app
 	make generate-version-file
-	poetry lock --no-update
-	poetry install --sync --no-root
+	poetry sync --no-root
 	poetry run pre-commit install
 	createdb notification_api || true
 	createdb test_notification_api || true
@@ -26,8 +25,7 @@ bootstrap: ## Set up everything to run the app
 .PHONY: bootstrap-with-git-hooks
 bootstrap-with-git-hooks:  ## Sets everything up and accounts for pre-existing git hooks
 	make generate-version-file
-	poetry lock --no-update
-	poetry install --sync --no-root
+	poetry sync --no-root
 	git config --global --unset-all core.hooksPath
 	poetry run pre-commit install
 	git config --global core.hookspath "${GIT_HOOKS_PATH}"
@@ -112,19 +110,19 @@ test: ## Run tests and create coverage report
 
 .PHONY: py-lock
 py-lock: ## Syncs dependencies and updates lock file without performing recursive internal updates
-	poetry lock --no-update
-	poetry install --sync
+	poetry sync --no-root
+	poetry lock
 
 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
-	poetry export --without-hashes --format=requirements.txt > requirements.txt
+	poetry export --output > requirements.txt
 
 .PHONY: audit
 audit:
 	poetry requirements > requirements.txt
 	poetry requirements --dev > requirements_for_test.txt
-	poetry run pip-audit -r requirements.txt
-	poetry run pip-audit -r requirements_for_test.txt
+	poetry run pip-audit -r requirements.txt --skip-editable
+	poetry run pip-audit -r requirements_for_test.txt --skip-editable
 
 .PHONY: static-scan
 static-scan:

README.md

@@ -40,7 +40,7 @@ You will need the following items:
 This project currently works with these major versions of the following main
 components:
 
-- Python 3.12.x
+- Python 3.13.x
 - PostgreSQL 15.x (version 12.x is used in the hosted environments)
 
 These instructions will walk you through how to set your machine up with all of
@@ -173,12 +173,12 @@ session to make the changes take effect.
 Now we're ready to install the Python version we need with `pyenv`, like so:
 
 ```sh
-pyenv install 3.12
+pyenv install 3.13
 ```
 
-This will install the latest version of Python 3.12.
+This will install the latest version of Python 3.13.
 
-_NOTE: This project currently runs on Python 3.12.x._
+_NOTE: This project currently runs on Python 3.13.x._
 
 #### Python Dependency Installation
 
@@ -221,7 +221,7 @@ If you don't have a line for your `$PATH` environment variable, add it in like
 this, which will include the PostgreSQL binaries:
 
 ```
-export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH
+export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH"
 ```
 
 _NOTE: You don't want to overwrite your existing `$PATH` environment variable! Hence the reason why it is included on the end like this; paths are separated by a colon._
@@ -259,12 +259,12 @@ git clone git@github.com:GSA/notifications-api.git
 
 Now go into the project directory (`notifications-api` by default), create a
 virtual environment, and set the local Python version to point to the virtual
-environment (assumes version Python `3.12.2` is what is installed on your
+environment (assumes version Python `3.13.2` is what is installed on your
 machine):
 
 ```sh
 cd notifications-api
-pyenv virtualenv 3.12.2 notify-api
+pyenv virtualenv 3.13.2 notify-api
 pyenv local notify-api
 ```
 
@@ -311,10 +311,10 @@ If you're upgrading an existing project to a newer version of Python, you can
 follow these steps to get yourself up-to-date.
 
 First, use `pyenv` to install the newer version of Python you'd like to use;
-we'll use `3.12` in our example here since we recently upgraded to this version:
+we'll use `3.13` in our example here since we recently upgraded to this version:
 
 ```sh
-pyenv install 3.12
+pyenv install 3.13
 ```
 
 Next, delete the virtual environment you previously had set up.  If you followed
@@ -329,7 +329,7 @@ environment with the newer version of Python you just installed:
 
 ```sh
 cd notifications-api
-pyenv virtualenv 3.12.2 notify-api
+pyenv virtualenv 3.13.2 notify-api
 pyenv local notify-api
 ```
 
@@ -339,6 +339,21 @@ you'll be set with an upgraded version of Python.
 _If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._
 
 
+#### Poetry upgrades ####
+
+If you are doing a new project setup, then after you install poetry you need to install the export plugin
+
+```sh
+poetry self add poetry-plugin-export
+```
+
+If you are upgrading from poetry 1.8.5, you need to do this:
+
+```sh
+curl -sSL https://install.python-poetry.org | python3 - --version 2.1.3
+poetry self add poetry-export-plugin
+```
+
 ### Final environment setup
 
 There's one final thing to adjust in the newly created `.env` file.  This
@@ -462,6 +477,8 @@ instructions above for more details.
   - [Onboarding](./docs/all.md#onboarding)
   - [Setting up the infrastructure](./docs/all.md#setting-up-the-infrastructure)
 - [Using the logs](./docs/all.md#using-the-logs)
+- [`git` hooks](./docs/all.md#git-hooks)
+  - [detect-secrets pre-commit plugin](./docs/all.md#detect-secrets-pre-commit-plugin)
 - [Testing](./docs/all.md#testing)
   - [CI testing](./docs/all.md#ci-testing)
   - [Manual testing](./docs/all.md#manual-testing)

app/__init__.py

@@ -5,7 +5,7 @@ import string
 import time
 import uuid
 from contextlib import contextmanager
-from multiprocessing import Manager
+from threading import Lock
 from time import monotonic
 
 from celery import Celery, Task, current_task
@@ -32,6 +32,9 @@ from notifications_utils.clients.encryption.encryption_client import Encryption
 from notifications_utils.clients.redis.redis_client import RedisClient
 from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
 
+job_cache = {}
+job_cache_lock = Lock()
+
 
 class NotifyCelery(Celery):
     def init_app(self, app):
@@ -152,9 +155,6 @@ def create_app(application):
     redis_store.init_app(application)
     document_download_client.init_app(application)
 
-    manager = Manager()
-    application.config["job_cache"] = manager.dict()
-
     register_blueprint(application)
 
     # avoid circular imports by importing this file later

app/aws/s3.py

@@ -2,13 +2,14 @@ import csv
 import datetime
 import re
 import time
-from concurrent.futures import ThreadPoolExecutor
 from io import StringIO
 
 import botocore
+import eventlet
 from boto3 import Session
 from flask import current_app
 
+from app import job_cache, job_cache_lock
 from app.clients import AWS_CLIENT_CONFIG
 from notifications_utils import aware_utcnow
 
@@ -24,31 +25,33 @@ s3_client = None
 s3_resource = None
 
 
+def get_service_id_from_key(key):
+    key = key.replace("service-", "")
+    key = key.split("/")
+    key = key[0].replace("-notify", "")
+    return key
+
+
 def set_job_cache(key, value):
-    current_app.logger.debug(f"Setting {key} in the job_cache.")
-    job_cache = current_app.config["job_cache"]
-    job_cache[key] = (value, time.time() + 8 * 24 * 60 * 60)
+    # current_app.logger.debug(f"Setting {key} in the job_cache to {value}.")
+
+    with job_cache_lock:
+        job_cache[key] = (value, time.time() + 8 * 24 * 60 * 60)
 
 
 def get_job_cache(key):
-    job_cache = current_app.config["job_cache"]
+
     ret = job_cache.get(key)
-    if ret is None:
-        current_app.logger.warning(f"Could not find {key} in the job_cache.")
-    else:
-        current_app.logger.debug(f"Got {key} from job_cache.")
     return ret
 
 
 def len_job_cache():
-    job_cache = current_app.config["job_cache"]
     ret = len(job_cache)
     current_app.logger.debug(f"Length of job_cache is {ret}")
     return ret
 
 
 def clean_cache():
-    job_cache = current_app.config["job_cache"]
     current_time = time.time()
     keys_to_delete = []
     for key, (_, expiry_time) in job_cache.items():
@@ -58,8 +61,9 @@ def clean_cache():
     current_app.logger.debug(
         f"Deleting the following keys from the job_cache: {keys_to_delete}"
     )
-    for key in keys_to_delete:
-        del job_cache[key]
+    with job_cache_lock:
+        for key in keys_to_delete:
+            del job_cache[key]
 
 
 def get_s3_client():
@@ -73,7 +77,7 @@ def get_s3_client():
             aws_secret_access_key=secret_key,
             region_name=region,
         )
-        s3_client = session.client("s3")
+        s3_client = session.client("s3", config=AWS_CLIENT_CONFIG)
     return s3_client
 
 
@@ -116,9 +120,9 @@ def list_s3_objects():
                 )
             else:
                 break
-    except Exception:
+    except Exception as e:
         current_app.logger.exception(
-            "An error occurred while regenerating cache #notify-debug-admin-1200",
+            f"An error occurred while regenerating cache #notify-debug-admin-1200: {str(e)}",
         )
 
 
@@ -184,23 +188,24 @@ def read_s3_file(bucket_name, object_key, s3res):
     """
     try:
         job_id = get_job_id_from_s3_object_key(object_key)
+        service_id = get_service_id_from_key(object_key)
+
         if get_job_cache(job_id) is None:
-            object = (
+            job = (
                 s3res.Object(bucket_name, object_key)
                 .get()["Body"]
                 .read()
                 .decode("utf-8")
             )
-            set_job_cache(job_id, object)
-            set_job_cache(f"{job_id}_phones", extract_phones(object))
+            set_job_cache(job_id, job)
+            set_job_cache(f"{job_id}_phones", extract_phones(job, service_id, job_id))
             set_job_cache(
                 f"{job_id}_personalisation",
-                extract_personalisation(object),
+                extract_personalisation(job),
             )
 
-    except LookupError:
-        # perhaps our key is not formatted as we expected.  If so skip it.
-        current_app.logger.exception("LookupError #notify-debug-admin-1200")
+    except Exception as e:
+        current_app.logger.exception(str(e))
 
 
 def get_s3_files():
@@ -215,11 +220,21 @@ def get_s3_files():
     current_app.logger.info(
         f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200"
     )
+    count = 0
     try:
-        with ThreadPoolExecutor() as executor:
-            executor.map(lambda key: read_s3_file(bucket_name, key, s3res), object_keys)
+        for object_key in object_keys:
+            read_s3_file(bucket_name, object_key, s3res)
+            count = count + 1
+            eventlet.sleep(0.2)
     except Exception:
-        current_app.logger.exception("Connection pool issue")
+        current_app.logger.exception(
+            f"Trouble reading {object_key} which is # {count} during cache regeneration"
+        )
+    except OSError as e:
+        current_app.logger.exception(
+            f"Egress proxy issue reading {object_key} which is # {count}"
+        )
+        raise e
 
     current_app.logger.info(
         f"job_cache length after regen: {len_job_cache()} #notify-debug-admin-1200"
@@ -289,9 +304,7 @@ def file_exists(file_location):
 
 
 def get_job_location(service_id, job_id):
-    current_app.logger.debug(
-        f"#notify-debug-s3-partitioning NEW JOB_LOCATION: {NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id)}"
-    )
+
     return (
         current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
         NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id),
@@ -307,9 +320,7 @@ def get_old_job_location(service_id, job_id):
     but it will take a few days where we have to support both formats.
     Remove this when everything works with the NEW_FILE_LOCATION_STRUCTURE.
     """
-    current_app.logger.debug(
-        f"#notify-debug-s3-partitioning OLD JOB LOCATION: {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}"
-    )
+
     return (
         current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
         FILE_LOCATION_STRUCTURE.format(service_id, job_id),
@@ -375,7 +386,7 @@ def get_job_from_s3(service_id, job_id):
                 )
                 retries += 1
                 sleep_time = backoff_factor * (2**retries)  # Exponential backoff
-                time.sleep(sleep_time)
+                eventlet.sleep(sleep_time)
                 continue
             else:
                 # Typically this is "NoSuchKey"
@@ -448,7 +459,6 @@ def extract_personalisation(job):
 def get_phone_number_from_s3(service_id, job_id, job_row_number):
     job = get_job_cache(job_id)
     if job is None:
-        current_app.logger.debug(f"job {job_id} was not in the cache")
         job = get_job_from_s3(service_id, job_id)
         # Even if it is None, put it here to avoid KeyErrors
         set_job_cache(job_id, job)
@@ -462,8 +472,16 @@ def get_phone_number_from_s3(service_id, job_id, job_row_number):
         )
         return "Unavailable"
 
-    phones = extract_phones(job, service_id, job_id)
-    set_job_cache(f"{job_id}_phones", phones)
+    phones = get_job_cache(f"{job_id}_phones")
+    if phones is None:
+        current_app.logger.debug("HAVE TO REEXTRACT PHONES!")
+        phones = extract_phones(job, service_id, job_id)
+        set_job_cache(f"{job_id}_phones", phones)
+        current_app.logger.debug(f"SETTING PHONES TO {phones}")
+    else:
+        phones = phones[
+            0
+        ]  # we only want the phone numbers not the cache expiration time
 
     # If we can find the quick dictionary, use it
     phone_to_return = phones[job_row_number]
@@ -482,7 +500,6 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
     # So this is a little recycling mechanism to reduce the number of downloads.
     job = get_job_cache(job_id)
     if job is None:
-        current_app.logger.debug(f"job {job_id} was not in the cache")
         job = get_job_from_s3(service_id, job_id)
         # Even if it is None, put it here to avoid KeyErrors
         set_job_cache(job_id, job)
@@ -500,7 +517,9 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
         )
         return {}
 
-    set_job_cache(f"{job_id}_personalisation", extract_personalisation(job))
+    personalisation = get_job_cache(f"{job_id}_personalisation")
+    if personalisation is None:
+        set_job_cache(f"{job_id}_personalisation", extract_personalisation(job))
 
     return get_job_cache(f"{job_id}_personalisation")[0].get(job_row_number)
 

app/celery/tasks.py

@@ -1,6 +1,6 @@
 import json
-from time import sleep
 
+import eventlet
 from celery.signals import task_postrun
 from flask import current_app
 from requests import HTTPError, RequestException, request
@@ -83,7 +83,7 @@ def process_job(job_id, sender_id=None):
         process_row(row, template, job, service, sender_id=sender_id)
         count = count + 1
         if count % 3 == 0:
-            sleep(1)
+            eventlet.sleep(1)
 
     # End point/Exit point for message send flow.
     job_complete(job, start=start)

app/clients/cloudwatch/aws_cloudwatch.py

@@ -151,7 +151,7 @@ class AwsCloudwatchClient(Client):
     #         result = temp_client.get_query_results(queryId=query_id)
     #         if result['status'] == 'Complete':
     #             break
-    #         time.sleep(1)
+    #         eventlet.sleep(1)
 
     #     delivery_receipts = []
     #     for log in result['results']:

app/service/rest.py

@@ -1,4 +1,5 @@
 import itertools
+import time
 from datetime import datetime, timedelta
 from zoneinfo import ZoneInfo
 
@@ -504,6 +505,10 @@ def get_all_notifications_for_service(service_id):
         if "page_size" in data
         else current_app.config.get("PAGE_SIZE")
     )
+    # HARD CODE TO 100 for now.  1000 or 10000 causes reports to time out before they complete (if big)
+    # Tests are relying on the value in config (20), whereas the UI seems to pass 10000
+    if page_size > 100:
+        page_size = 100
     limit_days = data.get("limit_days")
     include_jobs = data.get("include_jobs", True)
     include_from_test_key = data.get("include_from_test_key", False)
@@ -517,6 +522,8 @@ def get_all_notifications_for_service(service_id):
         f"get pagination with {service_id} service_id filters {data} \
                              limit_days {limit_days} include_jobs {include_jobs} include_one_off {include_one_off}"
     )
+    start_time = time.time()
+    current_app.logger.debug(f"Start report generation  with page.size {page_size}")
     pagination = notifications_dao.get_notifications_for_service(
         service_id,
         filter_dict=data,
@@ -528,9 +535,13 @@ def get_all_notifications_for_service(service_id):
         include_from_test_key=include_from_test_key,
         include_one_off=include_one_off,
     )
+    current_app.logger.debug(f"Query complete at {int(time.time()-start_time)*1000}")
 
     for notification in pagination.items:
         if notification.job_id is not None:
+            current_app.logger.debug(
+                f"Processing job_id {notification.job_id} at {int(time.time()-start_time)*1000}"
+            )
             notification.personalisation = get_personalisation_from_s3(
                 notification.service_id,
                 notification.job_id,

app/socket_handlers.py

@@ -3,6 +3,17 @@ from flask_socketio import join_room, leave_room
 
 
 def register_socket_handlers(socketio):
+    @socketio.on("connect")
+    def on_connect():
+        current_app.logger.info(
+            f"Socket {request.sid} connected from {request.environ.get('HTTP_ORIGIN')}"
+        )
+        return True
+
+    @socketio.on("disconnect")
+    def on_disconnect():
+        current_app.logger.info(f"Socket {request.sid} disconnected")
+
     @socketio.on("join")
     def on_join(data):  # noqa: F401
         room = data.get("room")

application.py

@@ -4,7 +4,7 @@ from __future__ import print_function
 from flask import Flask
 from werkzeug.serving import WSGIRequestHandler
 
-from app import create_app
+from app import create_app, socketio  # noqa: F401
 
 WSGIRequestHandler.version_string = lambda self: "SecureServer"
 

docs/all.md

@@ -7,6 +7,7 @@
   - [Setting up the infrastructure](#setting-up-the-infrastructure)
 - [Using the logs](#using-the-logs)
 - [`git` hooks](#git-hooks)
+  - [detect-secrets pre-commit plugin](#detect-secrets-pre-commit-plugin)
 - [Testing](#testing)
   - [CI testing](#ci-testing)
   - [Manual testing](#manual-testing)
@@ -262,6 +263,17 @@ The configuration is stored in `.pre-commit-config.yaml`. In that config, there
 
 We do not maintain any hooks in this repository.
 
+## detect-secrets pre-commit plugin
+
+One of the pre-commit hooks we use is [`detect-secrets`](https://github.com/Yelp/detect-secrets), which checks for all sorts of things that might be committed accidently that should not be.  The project is already set up with a baseline file (`.ds.baseline`) and this should just work out of the box, but occasionally it will flag something new when you try and commit something; or, the file may need a refresh after a while.  In either case, to get things back on track and update the `.ds.baseline` file, run these two commands:
+
+```sh
+detect-secrets scan --baseline .ds.baseline
+detect-secrets audit .ds.baseline
+```
+
+The second command will walk you through all of the new detected secrets and ask you to validate if they actually are or if they're false positives.  Mark off each one as apppropriate (they should all be false positives - if they're not please stop and check in with the team!), then commit the updates to the `.ds.baseline` file and push them remotely so the project stays up-to-date.
+
 # Testing
 
 ```

docs/openapi.yml

@@ -570,7 +570,7 @@ paths:
                 reference:
                   type: string
             example:
-              phone_number: "2028675309"
+              phone_number: "800-555-0100"
               template_id: "85b58733-7ebf-494e-bee2-a21a4ce17d58"
               personalisation:
                 variable: "value"

pyproject.toml

@@ -1,5 +1,6 @@
 [tool.poetry]
 name = "notifications-api"
+package-mode = false
 version = "0.1.0"
 description = "Notify.gov backend"
 authors = ["Your Name <you@example.com>"]
@@ -7,26 +8,26 @@ readme = "README.md"
 
 
 [tool.poetry.dependencies]
-python = "^3.12.2"
-alembic = "==1.15.2"
+python = "^3.13.2"
+alembic = "==1.16.1"
 amqp = "==5.3.1"
 beautifulsoup4 = "==4.13.4"
 boto3 = "^1.34.150"
 botocore = "^1.34.159"
-cachetools = "==5.4.0"
-celery = {version = "==5.5.2", extras = ["redis"]}
+cachetools = "==6.0.0"
+celery = {version = "==5.5.3", extras = ["redis"]}
 certifi = ">=2022.12.7"
 cffi = "==1.17.1"
 charset-normalizer = "^3.4.2"
-click = "==8.1.8"
+click = "==8.2.1"
 click-datetime = "==0.4.0"
 click-didyoumean = "==0.3.1"
 click-plugins = "==1.1.1"
 click-repl = "==0.3.0"
 deprecated = "==1.2.14"
-eventlet = "==0.39.1"
+eventlet = "==0.40.0"
 expiringdict = "==1.2.2"
-flask = "~=3.0"
+flask = "~=3.1"
 flask-bcrypt = "==1.0.1"
 flask-marshmallow = "==1.2.1"
 flask-migrate = "==4.1.0"
@@ -34,7 +35,7 @@ flask-redis = "==0.4.0"
 flask-sqlalchemy = "==3.1.1"
 gunicorn = {version = "==23.0.0", extras = ["eventlet"]}
 iso8601 = "==2.1.0"
-jsonschema = {version = "==4.23.0", extras = ["format"]}
+jsonschema = {version = "==4.24.0", extras = ["format"]}
 lxml = "==5.4.0"
 marshmallow = "==3.26.1"
 marshmallow-sqlalchemy = "==1.0.0"
@@ -43,25 +44,25 @@ notifications-python-client = "==10.0.1"
 oscrypto = { git = "https://github.com/wbond/oscrypto.git", rev = "1547f53" }
 packaging = "==25.0"
 poetry-dotenv-plugin = "==0.2.0"
-psycopg2-binary = "==2.9.9"
+psycopg2-binary = "==2.9.10"
 pyjwt = "==2.10.1"
 python-dotenv = "==1.1.0"
-sqlalchemy = "==2.0.40"
+sqlalchemy = "==2.0.41"
 werkzeug = "^3.0.6"
-faker = "^37.1.0"
+faker = "^37.3.0"
 async-timeout = "^5.0.1"
 bleach = "^6.1.0"
 geojson = "^3.2.0"
-numpy = "^2.2.5"
+numpy = "^2.2.6"
 ordered-set = "^4.1.0"
-phonenumbers = "^9.0.4"
+phonenumbers = "^9.0.6"
 python-json-logger = "^3.3.0"
 regex = "^2024.11.6"
-shapely = "^2.0.5"
+shapely = "^2.1.1"
 smartypants = "^2.0.1"
 mistune = "^3.1.3"
 blinker = "^1.9.0"
-cryptography = "^44.0.3"
+cryptography = "^45.0.3"
 idna = "^3.7"
 jmespath = "^1.0.1"
 markupsafe = "^3.0.2"
@@ -85,24 +86,24 @@ awscli = "^1.33.32"
 bandit = "*"
 black = "^25.1.0"
 cloudfoundry-client = "*"
-exceptiongroup = "==1.2.2"
+exceptiongroup = "==1.3.0"
 flake8 = "^7.2.0"
 flake8-bugbear = "^24.12.12"
-freezegun = "^1.5.1"
+freezegun = "^1.5.2"
 honcho = "*"
 isort = "^6.0.1"
 jinja2-cli = {version = "==0.8.2", extras = ["yaml"]}
-moto = "==5.1.4"
+moto = "==5.1.5"
 pip-audit = "*"
 pre-commit = "^4.2.0"
-pytest = "^8.3.2"
+pytest = "^8.4.0"
 pytest-env = "^1.1.3"
-pytest-mock = "^3.14.0"
+pytest-mock = "^3.14.1"
 pytest-cov = "^6.1.1"
-pytest-xdist = "^3.5.0"
+pytest-xdist = "^3.7.0"
 radon = "^6.0.1"
 requests-mock = "^1.11.0"
-setuptools = "^80.3.1"
+setuptools = "^80.9.0"
 sqlalchemy-utils = "^0.41.2"
 vulture = "^2.10"
 detect-secrets = "^1.5.0"

runtime.txt

@@ -1 +1 @@
-python-3.12.x
+python-3.13.x

tests/app/aws/test_s3.py

@@ -1,7 +1,7 @@
 import os
 from datetime import timedelta
 from os import getenv
-from unittest.mock import MagicMock, Mock, call, patch
+from unittest.mock import ANY, MagicMock, Mock, call, patch
 
 import botocore
 import pytest
@@ -39,7 +39,7 @@ default_region = getenv("CSV_AWS_REGION")
 
 def single_s3_object_stub(key="foo", last_modified=None):
     return {
-        "ETag": '"d41d8cd98f00b204e9800998ecf8427e"',
+        "ETag": '"d"',
         "Key": key,
         "LastModified": last_modified or utc_now(),
     }
@@ -221,20 +221,6 @@ def test_get_s3_file_makes_correct_call(notify_api, mocker):
             2,
             "5555555552",
         ),
-        (
-            # simulate file saved with utf8withbom
-            "\\ufeffPHONE NUMBER\n",
-            "eee",
-            2,
-            "5555555552",
-        ),
-        (
-            # simulate file saved without utf8withbom
-            "\\PHONE NUMBER\n",
-            "eee",
-            2,
-            "5555555552",
-        ),
     ],
 )
 def test_get_phone_number_from_s3(
@@ -242,6 +228,7 @@ def test_get_phone_number_from_s3(
 ):
     get_job_mock = mocker.patch("app.aws.s3.get_job_from_s3")
     get_job_mock.return_value = job
+
     phone_number = get_phone_number_from_s3("service_id", job_id, job_row_number)
     assert phone_number == expected_phone_number
 
@@ -420,29 +407,17 @@ def test_get_s3_files_success(client, mocker):
         "CSV_UPLOAD_BUCKET": {"bucket": "test-bucket"},
         "job_cache": {},
     }
-    mock_thread_pool_executor = mocker.patch("app.aws.s3.ThreadPoolExecutor")
     mock_read_s3_file = mocker.patch("app.aws.s3.read_s3_file")
     mock_list_s3_objects = mocker.patch("app.aws.s3.list_s3_objects")
     mock_get_s3_resource = mocker.patch("app.aws.s3.get_s3_resource")
     mock_list_s3_objects.return_value = ["file1.csv", "file2.csv"]
     mock_s3_resource = MagicMock()
     mock_get_s3_resource.return_value = mock_s3_resource
-    mock_executor = MagicMock()
-
-    def mock_map(func, iterable):
-        for item in iterable:
-            func(item)
-
-    mock_executor.map.side_effect = mock_map
-    mock_thread_pool_executor.return_value.__enter__.return_value = mock_executor
 
     get_s3_files()
 
     # mock_current_app.config.__getitem__.assert_called_once_with("CSV_UPLOAD_BUCKET")
     mock_list_s3_objects.assert_called_once()
-    mock_thread_pool_executor.assert_called_once()
-
-    mock_executor.map.assert_called_once()
 
     calls = [
         (("test-bucket", "file1.csv", mock_s3_resource),),
@@ -473,7 +448,7 @@ def test_get_s3_client(mocker):
     mock_session.return_value.client.return_value = mock_s3_client
     result = get_s3_client()
 
-    mock_session.return_value.client.assert_called_once_with("s3")
+    mock_session.return_value.client.assert_called_once_with("s3", config=ANY)
     assert result == mock_s3_client
 
 
@@ -615,15 +590,6 @@ def test_get_s3_files_handles_exception(mocker):
     mock_read_s3_file = mocker.patch(
         "app.aws.s3.read_s3_file", side_effect=[None, Exception("exception here")]
     )
-
-    mock_thread_pool_executor = mocker.patch("app.aws.s3.ThreadPoolExecutor")
-    mock_executor = mock_thread_pool_executor.return_value.__enter__.return_value
-
-    def mock_map(func, iterable):
-        for item in iterable:
-            func(item)
-
-    mock_executor.map.side_effect = mock_map
     get_s3_files()
 
     calls = [
@@ -632,4 +598,6 @@ def test_get_s3_files_handles_exception(mocker):
     ]
     mock_read_s3_file.assert_has_calls(calls, any_order=True)
 
-    mock_current_app.logger.exception.assert_called_with("Connection pool issue")
+    mock_current_app.logger.exception.assert_called_with(
+        "Trouble reading file2.csv which is # 1 during cache regeneration"
+    )