merge from main

.ds.baseline

@@ -133,7 +133,7 @@
         "filename": ".github/workflows/checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 27,
+        "line_number": 28,
         "is_secret": false
       },
       {
@@ -141,7 +141,7 @@
         "filename": ".github/workflows/checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 44,
+        "line_number": 45,
         "is_secret": false
       }
     ],
@@ -384,5 +384,5 @@
       }
     ]
   },
-  "generated_at": "2024-08-13T22:32:28Z"
+  "generated_at": "2024-08-22T18:00:24Z"
 }

.github/workflows/checks.yml

@@ -16,6 +16,7 @@ env:
   AWS_US_TOLL_FREE_NUMBER: "+18556438890"
 
 jobs:
+
   build:
     runs-on: ubuntu-latest
 

.github/workflows/deploy.yml

@@ -11,6 +11,7 @@ permissions:
   contents: read
 
 jobs:
+
   deploy:
     runs-on: ubuntu-latest
     if: ${{ github.event.workflow_run.conclusion == 'success' }}

app/aws/s3.py

@@ -1,3 +1,4 @@
+import datetime
 import re
 import time
 
@@ -8,6 +9,7 @@ from flask import current_app
 
 from app import redis_store
 from app.clients import AWS_CLIENT_CONFIG
+from notifications_utils import aware_utcnow
 
 FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
 
@@ -58,11 +60,15 @@ def list_s3_objects():
 
     bucket_name = current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
     s3_client = get_s3_client()
+    # Our reports only support 7 days, but pull 8 days to avoid
+    # any edge cases
+    time_limit = aware_utcnow() - datetime.timedelta(days=8)
     try:
         response = s3_client.list_objects_v2(Bucket=bucket_name)
         while True:
             for obj in response.get("Contents", []):
-                yield obj["Key"]
+                if obj["LastModified"] >= time_limit:
+                    yield obj["Key"]
             if "NextContinuationToken" in response:
                 response = s3_client.list_objects_v2(
                     Bucket=bucket_name,

app/commands.py

@@ -1070,4 +1070,14 @@ def add_test_users_to_db(generate, state, admin):
             state=state,
             platform_admin=admin,
         )
-        current_app.logger.info(f"{num} {user.email_address} created")
+        current_app.logger.info("User created")
+
+
+# generate a new salt value
+@notify_command(name="generate-salt")
+def generate_salt():
+    if getenv("NOTIFY_ENVIRONMENT", "") not in ["development", "test"]:
+        current_app.logger.error("Can only be run in development")
+        return
+    salt = secrets.token_hex(16)
+    print(salt)  # noqa

app/dao/jobs_dao.py

@@ -188,7 +188,8 @@ def dao_get_jobs_older_than_data_retention(notification_types):
             .all()
         )
 
-    end_date = today - timedelta(days=7)
+    # notify-api-1287, make default data retention 7 days, 23 hours
+    end_date = today - timedelta(days=7, hours=23)
     for notification_type in notification_types:
         services_with_data_retention = [
             x.service_id

docs/all.md

@@ -1425,3 +1425,12 @@ For application 'notify-api-sandbox': Service instance 'notify-api-rds-sandbox'
 ```
 
 Run `cf target -o gsa-tts-benefits-studio -s notify-sandbox` before pushing to the Sandbox
+
+### Searchable tags for 'hot' items
+
+Note: better to search on space 'notify-production' rather than specifically for admin or api
+
+#notify-admin-1200 (job cache regeneration)
+#notify-admin-1505 (general login issues)
+#notify-admin-1701 (wrong sender phone number)
+#notify-admin-1859 (job is created with created_at being the wrong time)

notifications_utils/logging.py

@@ -160,8 +160,12 @@ class JSONFormatter(BaseJSONFormatter):
         log_record["logType"] = "application"
         try:
             log_record["message"] = log_record["message"].format(**log_record)
-        except (KeyError, IndexError) as e:
+        except KeyError as e:
-            logger.exception(
+            # We get occasional log messages that are nested dictionaries,
-                "failed to format log message: {} not found".format(e), exc_info=True
+            # for example, delivery receipts, where the formatting fails
-            )
+            # This is not a huge problem, don't dump stack traces into the logs
+            # for it.
+            logger.warning(f"failed to format log message: {e}")
+        except IndexError as e:
+            logger.exception(f"failed to format log message: {e}")
         return log_record

poetry.lock

@@ -2330,13 +2330,13 @@ files = [
 
 [[package]]
 name = "marshmallow"
-version = "3.21.3"
+version = "3.22.0"
 description = "A lightweight library for converting complex datatypes to and from native Python datatypes."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "marshmallow-3.21.3-py3-none-any.whl", hash = "sha256:86ce7fb914aa865001a4b2092c4c2872d13bc347f3d42673272cabfdbad386f1"},
+    {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"},
-    {file = "marshmallow-3.21.3.tar.gz", hash = "sha256:4f57c5e050a54d66361e826f94fba213eb10b67b2fdb02c3e0343ce207ba1662"},
+    {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"},
 ]
 
 [package.dependencies]
@@ -2344,7 +2344,7 @@ packaging = ">=17.0"
 
 [package.extras]
 dev = ["marshmallow[tests]", "pre-commit (>=3.5,<4.0)", "tox"]
-docs = ["alabaster (==0.7.16)", "autodocsumm (==0.2.12)", "sphinx (==7.3.7)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"]
+docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.13)", "sphinx (==8.0.2)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"]
 tests = ["pytest", "pytz", "simplejson"]
 
 [[package]]
@@ -4800,4 +4800,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12.2"
-content-hash = "ef3b001044f519c6954e86f2288605b7debecbf0af4603ac858d547a56d761d6"
+content-hash = "213689af42ea6eb91a6a4baf3c3f41a8a69e75a022827ef18fe564645dd90762"

pyproject.toml

@@ -36,7 +36,7 @@ gunicorn = {version = "==22.0.0", extras = ["eventlet"]}
 iso8601 = "==2.1.0"
 jsonschema = {version = "==4.23.0", extras = ["format"]}
 lxml = "==5.2.2"
-marshmallow = "==3.21.3"
+marshmallow = "==3.22.0"
 marshmallow-sqlalchemy = "==1.0.0"
 newrelic = "*"
 notifications-python-client = "==10.0.0"