merge from main

.profile

@@ -3,5 +3,6 @@
 # https://docs.cloudfoundry.org/devguide/deploy-apps/deploy-app.html#profile
 ##
 
+export http_proxy=$egress_proxy
 export https_proxy=$egress_proxy
 export NEW_RELIC_PROXY_HOST=$egress_proxy

app/aws/s3.py

@@ -11,7 +11,8 @@ from app.clients import AWS_CLIENT_CONFIG
 FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
 
 
-JOBS = ExpiringDict(max_len=100, max_age_seconds=3600 * 4)
+JOBS = ExpiringDict(max_len=1000, max_age_seconds=3600 * 4)
+
 
 JOBS_CACHE_HITS = "JOBS_CACHE_HITS"
 JOBS_CACHE_MISSES = "JOBS_CACHE_MISSES"
@@ -93,7 +94,33 @@ def incr_jobs_cache_hits():
         redis_store.incr(JOBS_CACHE_HITS)
     hits = redis_store.get(JOBS_CACHE_HITS).decode("utf-8")
     misses = redis_store.get(JOBS_CACHE_MISSES).decode("utf-8")
-    current_app.logger.info(f"JOBS CACHE MISS hits {hits} misses {misses}")
+    current_app.logger.info(f"JOBS CACHE HIT hits {hits} misses {misses}")
+
+
+def extract_phones(job):
+    job = job.split("\r\n")
+    first_row = job[0]
+    job.pop(0)
+    first_row = first_row.split(",")
+    phone_index = 0
+    for item in first_row:
+        if item.lower() == "phone number":
+            break
+        phone_index = phone_index + 1
+    phones = {}
+    job_row = 0
+    for row in job:
+        row = row.split(",")
+        phone_index = 0
+        for item in first_row:
+            if item.lower() == "phone number":
+                break
+            phone_index = phone_index + 1
+        my_phone = row[phone_index]
+        my_phone = re.sub(r"[\+\s\(\)\-\.]*", "", my_phone)
+        phones[job_row] = my_phone
+        job_row = job_row + 1
+    return phones
 
 
 def get_phone_number_from_s3(service_id, job_id, job_row_number):
@@ -108,25 +135,37 @@ def get_phone_number_from_s3(service_id, job_id, job_row_number):
     else:
         incr_jobs_cache_hits()
 
-    job = job.split("\r\n")
-    first_row = job[0]
-    job.pop(0)
-    first_row = first_row.split(",")
-    phone_index = 0
-    for item in first_row:
-        if item.lower() == "phone number":
-            break
-        phone_index = phone_index + 1
-
-    correct_row = job[job_row_number]
-    correct_row = correct_row.split(",")
-
-    # This could happen if an old job cannot be retrieved from s3
-    if len(correct_row) <= phone_index:
+    # If the job is None after our attempt to retrieve it from s3, it
+    # probably means the job is old and has been deleted from s3, in
+    # which case there is nothing we can do.  It's unlikely to run into
+    # this, but it could theoretically happen, especially if we ever
+    # change the task schedules
+    if job is None:
+        current_app.logger.warning(
+            "Couldnt find phone for job_id {job_id} row number {job_row_number} because job is missing"
+        )
+        return "Unknown Phone"
+
+    # If we look in the JOBS cache for the quick lookup dictionary of phones for a given job
+    # and that dictionary is not there, create it
+    if JOBS.get(f"{job_id}_phones") is None:
+        JOBS[f"{job_id}_phones"] = extract_phones(job)
+
+    # If we can find the quick dictionary, use it
+    if JOBS.get(f"{job_id}_phones") is not None:
+        phone_to_return = JOBS.get(f"{job_id}_phones").get(job_row_number)
+        if phone_to_return:
+            return phone_to_return
+        else:
+            current_app.logger.warning(
+                f"Was unable to retrieve phone number from lookup dictionary for job {job_id}"
+            )
+            return "Unknown Phone"
+    else:
+        current_app.logger.error(
+            f"Was unable to construct lookup dictionary for job {job_id}"
+        )
         return "Unknown Phone"
-    my_phone = correct_row[phone_index]
-    my_phone = re.sub(r"[\+\s\(\)\-\.]*", "", my_phone)
-    return my_phone
 
 
 def get_personalisation_from_s3(service_id, job_id, job_row_number):

app/service/rest.py

@@ -440,6 +440,9 @@ def get_all_notifications_for_service(service_id):
             )
             notification.to = recipient
             notification.normalised_to = recipient
+        else:
+            notification.to = "1"
+            notification.normalised_to = "1"
 
     kwargs = request.args.to_dict()
     kwargs["service_id"] = service_id
@@ -479,7 +482,7 @@ def get_all_notifications_for_service(service_id):
                 page,
                 len(next_page_of_pagination.items),
                 ".get_all_notifications_for_service",
-                **kwargs
+                **kwargs,
             )
             if count_pages
             else {},

deploy-config/egress_proxy/notify-api-demo.allow.acl

@@ -8,3 +8,5 @@ s3-fips.us-west-2.amazonaws.com
 sns-fips.us-east-1.amazonaws.com
 gov-collector.newrelic.com
 egress-proxy-notify-api-demo.apps.internal
+idp.int.identitysandbox.gov
+secure.login.gov

deploy-config/egress_proxy/notify-api-production.allow.acl

@@ -7,3 +7,5 @@ s3-fips.us-gov-west-1.amazonaws.com
 sns.us-gov-west-1.amazonaws.com
 gov-collector.newrelic.com
 egress-proxy-notify-api-production.apps.internal
+idp.int.identitysandbox.gov
+secure.login.gov

deploy-config/egress_proxy/notify-api-staging.allow.acl

@@ -8,3 +8,5 @@ s3-fips.us-west-2.amazonaws.com
 sns-fips.us-west-2.amazonaws.com
 gov-collector.newrelic.com
 egress-proxy-notify-api-staging.apps.internal
+idp.int.identitysandbox.gov
+secure.login.gov

poetry.lock

@@ -280,19 +280,22 @@ typecheck = ["mypy"]
 
 [[package]]
 name = "beautifulsoup4"
-version = "4.12.2"
+version = "4.12.3"
 description = "Screen-scraping library"
 optional = false
 python-versions = ">=3.6.0"
 files = [
-    {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"},
-    {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"},
+    {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"},
+    {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"},
 ]
 
 [package.dependencies]
 soupsieve = ">1.2"
 
 [package.extras]
+cchardet = ["cchardet"]
+chardet = ["chardet"]
+charset-normalizer = ["charset-normalizer"]
 html5lib = ["html5lib"]
 lxml = ["lxml"]
 
@@ -4720,4 +4723,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.12"
-content-hash = "3389aa4ce5477dd99ab467168569e9920b942777f37e671bceb8e362ca362f76"
+content-hash = "a6d1d609f32455bcd4a1b2934512bba368d3bad792a032bb0e8e2b49f6f5f99a"

pyproject.toml

@@ -10,7 +10,7 @@ readme = "README.md"
 python = ">=3.9,<3.12"
 alembic = "==1.13.1"
 amqp = "==5.2.0"
-beautifulsoup4 = "==4.12.2"
+beautifulsoup4 = "==4.12.3"
 boto3 = "^1.29.6"
 botocore = "^1.32.6"
 cachetools = "==5.3.2"