darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-11 18:38:14 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into update-marshmallow-deps

Browse Source
This commit is contained in:
Carlo Costino
2025-06-02 10:19:17 -04:00
parent baa8cc4bfa 6d8e2221d0
commit 07195ba145
20 changed files with 985 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ds.baseline
+247 -1
View File
@@ -136,7 +136,253 @@
"line_number": 18, "line_number": 18,
"is_secret": false "is_secret": false
} }
],
".github/workflows/checks.yml": [
{
"type": "Secret Keyword",
"filename": ".github/workflows/checks.yml",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 28,
"is_secret": false
},
{
"type": "Basic Auth Credentials",
"filename": ".github/workflows/checks.yml",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 45,
"is_secret": false
}
],
".github/workflows/daily_checks.yml": [
{
"type": "Secret Keyword",
"filename": ".github/workflows/daily_checks.yml",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 71,
"is_secret": false
},
{
"type": "Basic Auth Credentials",
"filename": ".github/workflows/daily_checks.yml",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 87,
"is_secret": false
}
],
"app/enums.py": [
{
"type": "Secret Keyword",
"filename": "app/enums.py",
"hashed_secret": "12322e07b94ee3c7cd65a2952ece441538b53eb3",
"is_verified": false,
"line_number": 123,
"is_secret": false
}
],
"app/notifications/receive_notifications.py": [
{
"type": "Base64 High Entropy String",
"filename": "app/notifications/receive_notifications.py",
"hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
"is_verified": false,
"line_number": 29,
"is_secret": false
}
],
"deploy-config/sandbox.yml": [
{
"type": "Secret Keyword",
"filename": "deploy-config/sandbox.yml",
"hashed_secret": "113151dd10316fcb0d5507b6215d78e2f3fe9e54",
"is_verified": false,
"line_number": 11,
"is_secret": false
}
],
"sample.env": [
{
"type": "Basic Auth Credentials",
"filename": "sample.env",
"hashed_secret": "5b98cf4c3d794c8af1fcd7991e89cd4e52fb42a4",
"is_verified": false,
"line_number": 16,
"is_secret": false
}
],
"tests/app/clients/test_document_download.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/clients/test_document_download.py",
"hashed_secret": "3acfb2c2b433c0ea7ff107e33df91b18e52f960f",
"is_verified": false,
"line_number": 14,
"is_secret": false
}
],
"tests/app/clients/test_performance_platform.py": [
{
"type": "Base64 High Entropy String",
"filename": "tests/app/clients/test_performance_platform.py",
"hashed_secret": "76bb66c38ac4046bf73cd4a2c35a2b0af94aeb61",
"is_verified": false,
"line_number": 84,
"is_secret": false
}
],
"tests/app/dao/test_services_dao.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/dao/test_services_dao.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 289,
"is_secret": false
}
],
"tests/app/dao/test_users_dao.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/dao/test_users_dao.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 69,
"is_secret": false
},
{
"type": "Secret Keyword",
"filename": "tests/app/dao/test_users_dao.py",
"hashed_secret": "f2c57870308dc87f432e5912d4de6f8e322721ba",
"is_verified": false,
"line_number": 199,
"is_secret": false
}
],
"tests/app/db.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/db.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 90,
"is_secret": false
}
],
"tests/app/notifications/test_receive_notification.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/notifications/test_receive_notification.py",
"hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
"is_verified": false,
"line_number": 27,
"is_secret": false
},
{
"type": "Base64 High Entropy String",
"filename": "tests/app/notifications/test_receive_notification.py",
"hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
"is_verified": false,
"line_number": 57,
"is_secret": false
}
],
"tests/app/notifications/test_validators.py": [
{
"type": "Base64 High Entropy String",
"filename": "tests/app/notifications/test_validators.py",
"hashed_secret": "6c1a8443963d02d13ffe575a71abe19ea731fb66",
"is_verified": false,
"line_number": 672,
"is_secret": false
}
],
"tests/app/service/test_rest.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/service/test_rest.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 1285,
"is_secret": false
}
],
"tests/app/test_cloudfoundry_config.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/test_cloudfoundry_config.py",
"hashed_secret": "e5e178db7317356946d13e5d2da037d39ac61c71",
"is_verified": false,
"line_number": 12,
"is_secret": false
},
{
"type": "Basic Auth Credentials",
"filename": "tests/app/test_cloudfoundry_config.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 14,
"is_secret": false
},
{
"type": "Secret Keyword",
"filename": "tests/app/test_cloudfoundry_config.py",
"hashed_secret": "cfd48edeb81ba7d48cbddcf1eeede25ba67057e8",
"is_verified": false,
"line_number": 33,
"is_secret": false
}
],
"tests/app/user/test_rest.py": [
{
"type": "Secret Keyword",
"filename": "tests/app/user/test_rest.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 110,
"is_secret": false
},
{
"type": "Secret Keyword",
"filename": "tests/app/user/test_rest.py",
"hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
"is_verified": false,
"line_number": 864,
"is_secret": false
}
],
"tests/notifications_utils/clients/antivirus/test_antivirus_client.py": [
{
"type": "Secret Keyword",
"filename": "tests/notifications_utils/clients/antivirus/test_antivirus_client.py",
"hashed_secret": "932b25270abe1301c22c709a19082dff07d469ff",
"is_verified": false,
"line_number": 16,
"is_secret": false
}
],
"tests/notifications_utils/clients/encryption/test_encryption_client.py": [
{
"type": "Secret Keyword",
"filename": "tests/notifications_utils/clients/encryption/test_encryption_client.py",
"hashed_secret": "f1e923a9667de11be6a210849a8651c1bfd81605",
"is_verified": false,
"line_number": 13,
"is_secret": false
}
],
"tests/notifications_utils/clients/zendesk/test_zendesk_client.py": [
{
"type": "Secret Keyword",
"filename": "tests/notifications_utils/clients/zendesk/test_zendesk_client.py",
"hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
"is_verified": false,
"line_number": 16,
"is_secret": false
}
] ]
}, },
"generated_at": "2025-05-12T16:45:34Z" "generated_at": "2025-06-02T13:22:36Z"
} }
.github/actions/setup-project/action.yml
+4 -1
View File
@@ -15,7 +15,10 @@ runs:
python-version: "3.12.3" python-version: "3.12.3"
- name: Install poetry - name: Install poetry
shell: bash shell: bash
run: pip install poetry==1.8.5 run: pip install poetry==2.1.3
- name: Install poetry export
shell: bash
run: poetry self add poetry-plugin-export
- name: Downgrade virtualenv to compatible version - name: Downgrade virtualenv to compatible version
shell: bash shell: bash
run: pip install "virtualenv<20.30" run: pip install "virtualenv<20.30"
.github/dependabot.yml
+4
View File
@@ -11,3 +11,7 @@ updates:
interval: "daily" interval: "daily"
labels: labels:
- "dependabot" # Custom label to identify Dependabot PRs - "dependabot" # Custom label to identify Dependabot PRs
assignees:
- "alexjanousekGSA"
reviewers:
- "alexjanousekGSA"
.github/workflows/checks.yml
+11 -3
View File
@@ -87,12 +87,20 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: ./.github/actions/setup-project - uses: ./.github/actions/setup-project
- name: Create requirements.txt - name: Create requirements.txt
run: poetry export --without-hashes --format=requirements.txt > requirements.txt run: poetry export --output requirements_tmp.txt --without-hashes
- uses: pypa/gh-action-pip-audit@v1.0.8 - name: Filter requirements.txt
run: grep -v "oscrypto@ git" requirements_tmp.txt > requirements.txt
- name: Verify requirements.txt
run: ls -l requirements.txt
- name: Print requirements.txt
run: |
echo "Contents of requirements.txt:"
cat requirements.txt
- uses: pypa/gh-action-pip-audit@v1.1.0
with: with:
inputs: requirements.txt inputs: requirements.txt
ignore-vulns: | ignore-vulns: |
PYSEC-2022-43162 PYSEC-2023-312
static-scan: static-scan:
runs-on: ubuntu-latest runs-on: ubuntu-latest
.github/workflows/daily_checks.yml
+11 -1
View File
@@ -26,10 +26,20 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: ./.github/actions/setup-project - uses: ./.github/actions/setup-project
- name: Create requirements.txt - name: Create requirements.txt
run: poetry export --without-hashes --format=requirements.txt > requirements.txt run: poetry export --output requirements_tmp.txt --without-hashes
- name: Filter requirements.txt
run: grep -v "oscrypto@ git" requirements_tmp.txt > requirements.txt
- name: Verify requirements.txt
run: ls -l requirements.txt
- name: Print requirements.txt
run: |
echo "Contents of requirements.txt:"
cat requirements.txt
- uses: pypa/gh-action-pip-audit@v1.1.0 - uses: pypa/gh-action-pip-audit@v1.1.0
with: with:
inputs: requirements.txt inputs: requirements.txt
ignore-vulns: |
PYSEC-2023-312
- name: Upload pip-audit artifact - name: Upload pip-audit artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
.github/workflows/deploy-demo.yml
+1 -1
View File
@@ -44,7 +44,7 @@ jobs:
run: make bootstrap run: make bootstrap
- name: Create requirements.txt - name: Create requirements.txt
run: poetry export --without-hashes --format=requirements.txt > requirements.txt run: poetry export --output requirements.txt
- name: Deploy to cloud.gov - name: Deploy to cloud.gov
uses: cloud-gov/cg-cli-tools@main uses: cloud-gov/cg-cli-tools@main
.github/workflows/deploy-prod.yml
+1 -1
View File
@@ -48,7 +48,7 @@ jobs:
run: make bootstrap run: make bootstrap
- name: Create requirements.txt - name: Create requirements.txt
run: poetry export --without-hashes --format=requirements.txt > requirements.txt run: poetry export --output requirements.txt
- name: Deploy to cloud.gov - name: Deploy to cloud.gov
uses: cloud-gov/cg-cli-tools@main uses: cloud-gov/cg-cli-tools@main
.github/workflows/deploy.yml
+1 -1
View File
@@ -50,7 +50,7 @@ jobs:
run: make bootstrap run: make bootstrap
- name: Create requirements.txt - name: Create requirements.txt
run: poetry export --without-hashes --format=requirements.txt > requirements.txt run: poetry export --output requirements.txt
- name: Deploy to cloud.gov - name: Deploy to cloud.gov
uses: cloud-gov/cg-cli-tools@main uses: cloud-gov/cg-cli-tools@main
Makefile
+7 -9
View File
@@ -16,8 +16,7 @@ GIT_HOOKS_PATH ?= $(shell git config --global core.hooksPath || echo "")
.PHONY: bootstrap .PHONY: bootstrap
bootstrap: ## Set up everything to run the app bootstrap: ## Set up everything to run the app
make generate-version-file make generate-version-file
poetry lock --no-update poetry sync --no-root
poetry install --sync --no-root
poetry run pre-commit install poetry run pre-commit install
createdb notification_api || true createdb notification_api || true
createdb test_notification_api || true createdb test_notification_api || true
@@ -26,8 +25,7 @@ bootstrap: ## Set up everything to run the app
.PHONY: bootstrap-with-git-hooks .PHONY: bootstrap-with-git-hooks
bootstrap-with-git-hooks: ## Sets everything up and accounts for pre-existing git hooks bootstrap-with-git-hooks: ## Sets everything up and accounts for pre-existing git hooks
make generate-version-file make generate-version-file
poetry lock --no-update poetry sync --no-root
poetry install --sync --no-root
git config --global --unset-all core.hooksPath git config --global --unset-all core.hooksPath
poetry run pre-commit install poetry run pre-commit install
git config --global core.hookspath "${GIT_HOOKS_PATH}" git config --global core.hookspath "${GIT_HOOKS_PATH}"
@@ -116,19 +114,19 @@ test-debug:
.PHONY: py-lock .PHONY: py-lock
py-lock: ## Syncs dependencies and updates lock file without performing recursive internal updates py-lock: ## Syncs dependencies and updates lock file without performing recursive internal updates
poetry lock --no-update poetry sync --no-root
poetry install --sync poetry lock
.PHONY: freeze-requirements .PHONY: freeze-requirements
freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
poetry export --without-hashes --format=requirements.txt > requirements.txt poetry export --output > requirements.txt
.PHONY: audit .PHONY: audit
audit: audit:
poetry requirements > requirements.txt poetry requirements > requirements.txt
poetry requirements --dev > requirements_for_test.txt poetry requirements --dev > requirements_for_test.txt
poetry run pip-audit -r requirements.txt poetry run pip-audit -r requirements.txt --skip-editable
poetry run pip-audit -r requirements_for_test.txt poetry run pip-audit -r requirements_for_test.txt --skip-editable
.PHONY: static-scan .PHONY: static-scan
static-scan: static-scan:
README.md
+18 -1
View File
@@ -221,7 +221,7 @@ If you don't have a line for your `$PATH` environment variable, add it in like
this, which will include the PostgreSQL binaries: this, which will include the PostgreSQL binaries:
``` ```
export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH"
``` ```
_NOTE: You don't want to overwrite your existing `$PATH` environment variable! Hence the reason why it is included on the end like this; paths are separated by a colon._ _NOTE: You don't want to overwrite your existing `$PATH` environment variable! Hence the reason why it is included on the end like this; paths are separated by a colon._
@@ -339,6 +339,21 @@ you'll be set with an upgraded version of Python.
_If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._ _If you're not sure about the details of your current virtual environment, you can run `poetry env info` to get more information. If you've been using `pyenv` for everything, you can also see all available virtual environments with `pyenv virtualenvs`._
#### Poetry upgrades ####
If you are doing a new project setup, then after you install poetry you need to install the export plugin
```sh
poetry self add poetry-plugin-export
```
If you are upgrading from poetry 1.8.5, you need to do this:
```sh
curl -sSL https://install.python-poetry.org | python3 - --version 2.1.3
poetry self add poetry-export-plugin
```
### Final environment setup ### Final environment setup
There's one final thing to adjust in the newly created `.env` file. This There's one final thing to adjust in the newly created `.env` file. This
@@ -462,6 +477,8 @@ instructions above for more details.
- [Onboarding](./docs/all.md#onboarding) - [Onboarding](./docs/all.md#onboarding)
- [Setting up the infrastructure](./docs/all.md#setting-up-the-infrastructure) - [Setting up the infrastructure](./docs/all.md#setting-up-the-infrastructure)
- [Using the logs](./docs/all.md#using-the-logs) - [Using the logs](./docs/all.md#using-the-logs)
- [`git` hooks](./docs/all.md#git-hooks)
- [detect-secrets pre-commit plugin](./docs/all.md#detect-secrets-pre-commit-plugin)
- [Testing](./docs/all.md#testing) - [Testing](./docs/all.md#testing)
- [CI testing](./docs/all.md#ci-testing) - [CI testing](./docs/all.md#ci-testing)
- [Manual testing](./docs/all.md#manual-testing) - [Manual testing](./docs/all.md#manual-testing)
app/__init__.py
+4 -4
View File
@@ -5,7 +5,7 @@ import string
import time import time
import uuid import uuid
from contextlib import contextmanager from contextlib import contextmanager
from multiprocessing import Manager from threading import Lock
from time import monotonic from time import monotonic
from celery import Celery, Task, current_task from celery import Celery, Task, current_task
@@ -31,6 +31,9 @@ from notifications_utils.clients.encryption.encryption_client import Encryption
from notifications_utils.clients.redis.redis_client import RedisClient from notifications_utils.clients.redis.redis_client import RedisClient
from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
job_cache = {}
job_cache_lock = Lock()
class NotifyCelery(Celery): class NotifyCelery(Celery):
def init_app(self, app): def init_app(self, app):
@@ -149,9 +152,6 @@ def create_app(application):
redis_store.init_app(application) redis_store.init_app(application)
document_download_client.init_app(application) document_download_client.init_app(application)
manager = Manager()
application.config["job_cache"] = manager.dict()
register_blueprint(application) register_blueprint(application)
# avoid circular imports by importing this file later # avoid circular imports by importing this file later
app/aws/s3.py
+38 -28
View File
@@ -9,6 +9,7 @@ import eventlet
from boto3 import Session from boto3 import Session
from flask import current_app from flask import current_app
from app import job_cache, job_cache_lock
from app.clients import AWS_CLIENT_CONFIG from app.clients import AWS_CLIENT_CONFIG
from notifications_utils import aware_utcnow from notifications_utils import aware_utcnow
@@ -32,30 +33,25 @@ def get_service_id_from_key(key):
def set_job_cache(key, value): def set_job_cache(key, value):
current_app.logger.debug(f"Setting {key} in the job_cache to {value}.") # current_app.logger.debug(f"Setting {key} in the job_cache to {value}.")
job_cache = current_app.config["job_cache"]
job_cache[key] = (value, time.time() + 8 * 24 * 60 * 60) with job_cache_lock:
job_cache[key] = (value, time.time() + 8 * 24 * 60 * 60)
def get_job_cache(key): def get_job_cache(key):
job_cache = current_app.config["job_cache"]
ret = job_cache.get(key) ret = job_cache.get(key)
if ret is None:
current_app.logger.warning(f"Could not find {key} in the job_cache.")
else:
current_app.logger.debug(f"Got {key} from job_cache with value {ret}.")
return ret return ret
def len_job_cache(): def len_job_cache():
job_cache = current_app.config["job_cache"]
ret = len(job_cache) ret = len(job_cache)
current_app.logger.debug(f"Length of job_cache is {ret}") current_app.logger.debug(f"Length of job_cache is {ret}")
return ret return ret
def clean_cache(): def clean_cache():
job_cache = current_app.config["job_cache"]
current_time = time.time() current_time = time.time()
keys_to_delete = [] keys_to_delete = []
for key, (_, expiry_time) in job_cache.items(): for key, (_, expiry_time) in job_cache.items():
@@ -65,8 +61,9 @@ def clean_cache():
current_app.logger.debug( current_app.logger.debug(
f"Deleting the following keys from the job_cache: {keys_to_delete}" f"Deleting the following keys from the job_cache: {keys_to_delete}"
) )
for key in keys_to_delete: with job_cache_lock:
del job_cache[key] for key in keys_to_delete:
del job_cache[key]
def get_s3_client(): def get_s3_client():
@@ -80,7 +77,7 @@ def get_s3_client():
aws_secret_access_key=secret_key, aws_secret_access_key=secret_key,
region_name=region, region_name=region,
) )
s3_client = session.client("s3") s3_client = session.client("s3", config=AWS_CLIENT_CONFIG)
return s3_client return s3_client
@@ -207,9 +204,8 @@ def read_s3_file(bucket_name, object_key, s3res):
extract_personalisation(job), extract_personalisation(job),
) )
except LookupError: except Exception as e:
# perhaps our key is not formatted as we expected. If so skip it. current_app.logger.exception(str(e))
current_app.logger.exception("LookupError #notify-debug-admin-1200")
def get_s3_files(): def get_s3_files():
@@ -224,11 +220,21 @@ def get_s3_files():
current_app.logger.info( current_app.logger.info(
f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200" f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200"
) )
count = 0
try: try:
for object_key in object_keys: for object_key in object_keys:
read_s3_file(bucket_name, object_key, s3res) read_s3_file(bucket_name, object_key, s3res)
count = count + 1
eventlet.sleep(0.2)
except Exception: except Exception:
current_app.logger.exception("Connection pool issue") current_app.logger.exception(
f"Trouble reading {object_key} which is # {count} during cache regeneration"
)
except OSError as e:
current_app.logger.exception(
f"Egress proxy issue reading {object_key} which is # {count}"
)
raise e
current_app.logger.info( current_app.logger.info(
f"job_cache length after regen: {len_job_cache()} #notify-debug-admin-1200" f"job_cache length after regen: {len_job_cache()} #notify-debug-admin-1200"
@@ -298,9 +304,7 @@ def file_exists(file_location):
def get_job_location(service_id, job_id): def get_job_location(service_id, job_id):
current_app.logger.debug(
f"#notify-debug-s3-partitioning NEW JOB_LOCATION: {NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id)}"
)
return ( return (
current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id), NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id),
@@ -316,9 +320,7 @@ def get_old_job_location(service_id, job_id):
but it will take a few days where we have to support both formats. but it will take a few days where we have to support both formats.
Remove this when everything works with the NEW_FILE_LOCATION_STRUCTURE. Remove this when everything works with the NEW_FILE_LOCATION_STRUCTURE.
""" """
current_app.logger.debug(
f"#notify-debug-s3-partitioning OLD JOB LOCATION: {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}"
)
return ( return (
current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
FILE_LOCATION_STRUCTURE.format(service_id, job_id), FILE_LOCATION_STRUCTURE.format(service_id, job_id),
@@ -457,7 +459,6 @@ def extract_personalisation(job):
def get_phone_number_from_s3(service_id, job_id, job_row_number): def get_phone_number_from_s3(service_id, job_id, job_row_number):
job = get_job_cache(job_id) job = get_job_cache(job_id)
if job is None: if job is None:
current_app.logger.debug(f"job {job_id} was not in the cache")
job = get_job_from_s3(service_id, job_id) job = get_job_from_s3(service_id, job_id)
# Even if it is None, put it here to avoid KeyErrors # Even if it is None, put it here to avoid KeyErrors
set_job_cache(job_id, job) set_job_cache(job_id, job)
@@ -471,8 +472,16 @@ def get_phone_number_from_s3(service_id, job_id, job_row_number):
) )
return "Unavailable" return "Unavailable"
phones = extract_phones(job, service_id, job_id) phones = get_job_cache(f"{job_id}_phones")
set_job_cache(f"{job_id}_phones", phones) if phones is None:
current_app.logger.debug("HAVE TO REEXTRACT PHONES!")
phones = extract_phones(job, service_id, job_id)
set_job_cache(f"{job_id}_phones", phones)
current_app.logger.debug(f"SETTING PHONES TO {phones}")
else:
phones = phones[
0
] # we only want the phone numbers not the cache expiration time
# If we can find the quick dictionary, use it # If we can find the quick dictionary, use it
phone_to_return = phones[job_row_number] phone_to_return = phones[job_row_number]
@@ -491,7 +500,6 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
# So this is a little recycling mechanism to reduce the number of downloads. # So this is a little recycling mechanism to reduce the number of downloads.
job = get_job_cache(job_id) job = get_job_cache(job_id)
if job is None: if job is None:
current_app.logger.debug(f"job {job_id} was not in the cache")
job = get_job_from_s3(service_id, job_id) job = get_job_from_s3(service_id, job_id)
# Even if it is None, put it here to avoid KeyErrors # Even if it is None, put it here to avoid KeyErrors
set_job_cache(job_id, job) set_job_cache(job_id, job)
@@ -509,7 +517,9 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
) )
return {} return {}
set_job_cache(f"{job_id}_personalisation", extract_personalisation(job)) personalisation = get_job_cache(f"{job_id}_personalisation")
if personalisation is None:
set_job_cache(f"{job_id}_personalisation", extract_personalisation(job))
return get_job_cache(f"{job_id}_personalisation")[0].get(job_row_number) return get_job_cache(f"{job_id}_personalisation")[0].get(job_row_number)
app/service/rest.py
+11
View File
@@ -1,4 +1,5 @@
import itertools import itertools
import time
from datetime import datetime, timedelta from datetime import datetime, timedelta
from zoneinfo import ZoneInfo from zoneinfo import ZoneInfo
@@ -513,6 +514,10 @@ def get_all_notifications_for_service(service_id):
if "page_size" in data if "page_size" in data
else current_app.config.get("PAGE_SIZE") else current_app.config.get("PAGE_SIZE")
) )
# HARD CODE TO 100 for now. 1000 or 10000 causes reports to time out before they complete (if big)
# Tests are relying on the value in config (20), whereas the UI seems to pass 10000
if page_size > 100:
page_size = 100
limit_days = data.get("limit_days") limit_days = data.get("limit_days")
include_jobs = data.get("include_jobs", True) include_jobs = data.get("include_jobs", True)
include_from_test_key = data.get("include_from_test_key", False) include_from_test_key = data.get("include_from_test_key", False)
@@ -526,6 +531,8 @@ def get_all_notifications_for_service(service_id):
f"get pagination with {service_id} service_id filters {data} \ f"get pagination with {service_id} service_id filters {data} \
limit_days {limit_days} include_jobs {include_jobs} include_one_off {include_one_off}" limit_days {limit_days} include_jobs {include_jobs} include_one_off {include_one_off}"
) )
start_time = time.time()
current_app.logger.debug(f"Start report generation with page.size {page_size}")
pagination = notifications_dao.get_notifications_for_service( pagination = notifications_dao.get_notifications_for_service(
service_id, service_id,
filter_dict=data, filter_dict=data,
@@ -537,9 +544,13 @@ def get_all_notifications_for_service(service_id):
include_from_test_key=include_from_test_key, include_from_test_key=include_from_test_key,
include_one_off=include_one_off, include_one_off=include_one_off,
) )
current_app.logger.debug(f"Query complete at {int(time.time()-start_time)*1000}")
for notification in pagination.items: for notification in pagination.items:
if notification.job_id is not None: if notification.job_id is not None:
current_app.logger.debug(
f"Processing job_id {notification.job_id} at {int(time.time()-start_time)*1000}"
)
notification.personalisation = get_personalisation_from_s3( notification.personalisation = get_personalisation_from_s3(
notification.service_id, notification.service_id,
notification.job_id, notification.job_id,
app/socket_handlers.py
+11
View File
@@ -3,6 +3,17 @@ from flask_socketio import join_room, leave_room
def register_socket_handlers(socketio): def register_socket_handlers(socketio):
@socketio.on("connect")
def on_connect():
current_app.logger.info(
f"Socket {request.sid} connected from {request.environ.get('HTTP_ORIGIN')}"
)
return True
@socketio.on("disconnect")
def on_disconnect():
current_app.logger.info(f"Socket {request.sid} disconnected")
@socketio.on("join") @socketio.on("join")
def on_join(data): # noqa: F401 def on_join(data): # noqa: F401
room = data.get("room") room = data.get("room")
application.py
+1 -1
View File
@@ -4,7 +4,7 @@ from __future__ import print_function
from flask import Flask from flask import Flask
from werkzeug.serving import WSGIRequestHandler from werkzeug.serving import WSGIRequestHandler
from app import create_app from app import create_app, socketio # noqa: F401
WSGIRequestHandler.version_string = lambda self: "SecureServer" WSGIRequestHandler.version_string = lambda self: "SecureServer"
docs/all.md
+12
View File
@@ -7,6 +7,7 @@
- [Setting up the infrastructure](#setting-up-the-infrastructure) - [Setting up the infrastructure](#setting-up-the-infrastructure)
- [Using the logs](#using-the-logs) - [Using the logs](#using-the-logs)
- [`git` hooks](#git-hooks) - [`git` hooks](#git-hooks)
- [detect-secrets pre-commit plugin](#detect-secrets-pre-commit-plugin)
- [Testing](#testing) - [Testing](#testing)
- [CI testing](#ci-testing) - [CI testing](#ci-testing)
- [Manual testing](#manual-testing) - [Manual testing](#manual-testing)
@@ -262,6 +263,17 @@ The configuration is stored in `.pre-commit-config.yaml`. In that config, there
We do not maintain any hooks in this repository. We do not maintain any hooks in this repository.
## detect-secrets pre-commit plugin
One of the pre-commit hooks we use is [`detect-secrets`](https://github.com/Yelp/detect-secrets), which checks for all sorts of things that might be committed accidently that should not be. The project is already set up with a baseline file (`.ds.baseline`) and this should just work out of the box, but occasionally it will flag something new when you try and commit something; or, the file may need a refresh after a while. In either case, to get things back on track and update the `.ds.baseline` file, run these two commands:
```sh
detect-secrets scan --baseline .ds.baseline
detect-secrets audit .ds.baseline
```
The second command will walk you through all of the new detected secrets and ask you to validate if they actually are or if they're false positives. Mark off each one as apppropriate (they should all be false positives - if they're not please stop and check in with the team!), then commit the updates to the `.ds.baseline` file and push them remotely so the project stays up-to-date.
# Testing # Testing
``` ```
docs/openapi.yml
+1 -1
View File
@@ -570,7 +570,7 @@ paths:
reference: reference:
type: string type: string
example: example:
phone_number: "2028675309" phone_number: "800-555-0100"
template_id: "85b58733-7ebf-494e-bee2-a21a4ce17d58" template_id: "85b58733-7ebf-494e-bee2-a21a4ce17d58"
personalisation: personalisation:
variable: "value" variable: "value"
poetry.lock
Generated
+582 -571
View File
File diff suppressed because it is too large Load Diff
pyproject.toml
+14 -13
View File
@@ -1,5 +1,6 @@
[tool.poetry] [tool.poetry]
name = "notifications-api" name = "notifications-api"
package-mode = false
version = "0.1.0" version = "0.1.0"
description = "Notify.gov backend" description = "Notify.gov backend"
authors = ["Your Name <you@example.com>"] authors = ["Your Name <you@example.com>"]
@@ -8,17 +9,17 @@ readme = "README.md"
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.12.2" python = "^3.12.2"
alembic = "==1.15.2" alembic = "==1.16.1"
amqp = "==5.3.1" amqp = "==5.3.1"
beautifulsoup4 = "==4.13.4" beautifulsoup4 = "==4.13.4"
boto3 = "^1.34.150" boto3 = "^1.34.150"
botocore = "^1.34.159" botocore = "^1.34.159"
cachetools = "==5.4.0" cachetools = "==6.0.0"
celery = {version = "==5.5.2", extras = ["redis"]} celery = {version = "==5.5.2", extras = ["redis"]}
certifi = ">=2022.12.7" certifi = ">=2022.12.7"
cffi = "==1.17.1" cffi = "==1.17.1"
charset-normalizer = "^3.4.2" charset-normalizer = "^3.4.2"
click = "==8.2.0" click = "==8.2.1"
click-datetime = "==0.4.0" click-datetime = "==0.4.0"
click-didyoumean = "==0.3.1" click-didyoumean = "==0.3.1"
click-plugins = "==1.1.1" click-plugins = "==1.1.1"
@@ -33,7 +34,7 @@ flask-redis = "==0.4.0"
flask-sqlalchemy = "^3.1.1" flask-sqlalchemy = "^3.1.1"
gunicorn = {version = "==23.0.0", extras = ["eventlet"]} gunicorn = {version = "==23.0.0", extras = ["eventlet"]}
iso8601 = "==2.1.0" iso8601 = "==2.1.0"
jsonschema = {version = "==4.23.0", extras = ["format"]} jsonschema = {version = "==4.24.0", extras = ["format"]}
lxml = "==5.4.0" lxml = "==5.4.0"
marshmallow = "^4.0.0" marshmallow = "^4.0.0"
marshmallow-sqlalchemy = "^1.4.2" marshmallow-sqlalchemy = "^1.4.2"
@@ -51,16 +52,16 @@ faker = "^37.3.0"
async-timeout = "^5.0.1" async-timeout = "^5.0.1"
bleach = "^6.1.0" bleach = "^6.1.0"
geojson = "^3.2.0" geojson = "^3.2.0"
numpy = "^2.2.5" numpy = "^2.2.6"
ordered-set = "^4.1.0" ordered-set = "^4.1.0"
phonenumbers = "^9.0.5" phonenumbers = "^9.0.6"
python-json-logger = "^3.3.0" python-json-logger = "^3.3.0"
regex = "^2024.11.6" regex = "^2024.11.6"
shapely = "^2.0.5" shapely = "^2.1.1"
smartypants = "^2.0.1" smartypants = "^2.0.1"
mistune = "^3.1.3" mistune = "^3.1.3"
blinker = "^1.9.0" blinker = "^1.9.0"
cryptography = "^44.0.3" cryptography = "^45.0.3"
idna = "^3.7" idna = "^3.7"
jmespath = "^1.0.1" jmespath = "^1.0.1"
markupsafe = "^3.0.2" markupsafe = "^3.0.2"
@@ -88,21 +89,21 @@ cloudfoundry-client = "*"
exceptiongroup = "==1.3.0" exceptiongroup = "==1.3.0"
flake8 = "^7.2.0" flake8 = "^7.2.0"
flake8-bugbear = "^24.12.12" flake8-bugbear = "^24.12.12"
freezegun = "^1.5.1" freezegun = "^1.5.2"
honcho = "*" honcho = "*"
isort = "^6.0.1" isort = "^6.0.1"
jinja2-cli = {version = "==0.8.2", extras = ["yaml"]} jinja2-cli = {version = "==0.8.2", extras = ["yaml"]}
moto = "==5.1.4" moto = "==5.1.5"
pip-audit = "*" pip-audit = "*"
pre-commit = "^4.2.0" pre-commit = "^4.2.0"
pytest = "^8.3.2" pytest = "^8.3.2"
pytest-env = "^1.1.3" pytest-env = "^1.1.3"
pytest-mock = "^3.14.0" pytest-mock = "^3.14.1"
pytest-cov = "^6.1.1" pytest-cov = "^6.1.1"
pytest-xdist = "^3.5.0" pytest-xdist = "^3.7.0"
radon = "^6.0.1" radon = "^6.0.1"
requests-mock = "^1.11.0" requests-mock = "^1.11.0"
setuptools = "^80.7.1" setuptools = "^80.9.0"
sqlalchemy-utils = "^0.41.2" sqlalchemy-utils = "^0.41.2"
vulture = "^2.10" vulture = "^2.10"
detect-secrets = "^1.5.0" detect-secrets = "^1.5.0"
tests/app/aws/test_s3.py
+6 -17
View File
@@ -1,7 +1,7 @@
import os import os
from datetime import timedelta from datetime import timedelta
from os import getenv from os import getenv
from unittest.mock import MagicMock, Mock, call, patch from unittest.mock import ANY, MagicMock, Mock, call, patch
import botocore import botocore
import pytest import pytest
@@ -221,20 +221,6 @@ def test_get_s3_file_makes_correct_call(notify_api, mocker):
2, 2,
"5555555552", "5555555552",
), ),
(
# simulate file saved with utf8withbom
"\\ufeffPHONE NUMBER\n",
"eee",
2,
"5555555552",
),
(
# simulate file saved without utf8withbom
"\\PHONE NUMBER\n",
"eee",
2,
"5555555552",
),
], ],
) )
def test_get_phone_number_from_s3( def test_get_phone_number_from_s3(
@@ -242,6 +228,7 @@ def test_get_phone_number_from_s3(
): ):
get_job_mock = mocker.patch("app.aws.s3.get_job_from_s3") get_job_mock = mocker.patch("app.aws.s3.get_job_from_s3")
get_job_mock.return_value = job get_job_mock.return_value = job
phone_number = get_phone_number_from_s3("service_id", job_id, job_row_number) phone_number = get_phone_number_from_s3("service_id", job_id, job_row_number)
assert phone_number == expected_phone_number assert phone_number == expected_phone_number
@@ -461,7 +448,7 @@ def test_get_s3_client(mocker):
mock_session.return_value.client.return_value = mock_s3_client mock_session.return_value.client.return_value = mock_s3_client
result = get_s3_client() result = get_s3_client()
mock_session.return_value.client.assert_called_once_with("s3") mock_session.return_value.client.assert_called_once_with("s3", config=ANY)
assert result == mock_s3_client assert result == mock_s3_client
@@ -611,4 +598,6 @@ def test_get_s3_files_handles_exception(mocker):
] ]
mock_read_s3_file.assert_has_calls(calls, any_order=True) mock_read_s3_file.assert_has_calls(calls, any_order=True)
mock_current_app.logger.exception.assert_called_with("Connection pool issue") mock_current_app.logger.exception.assert_called_with(
"Trouble reading file2.csv which is # 1 during cache regeneration"
)