darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-09 17:45:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix regenerate_job_cache

Browse Source
This commit is contained in:
Kenneth Kehl
2025-05-14 07:49:30 -07:00
parent 575f1b4c6b
commit af107ccd27
1 changed files with 17 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
app/aws/s3.py
View File

@@ -2,7 +2,6 @@ import csv
import datetime import datetime
import re import re
import time import time
from concurrent.futures import ThreadPoolExecutor
from io import StringIO from io import StringIO
import botocore import botocore
@@ -25,8 +24,15 @@ s3_client = None
s3_resource = None s3_resource = None
def get_service_id_from_key(key):
key = key.replace("service-", "")
key = key.split("/")
key = key[0].replace("-notify", "")
return key
def set_job_cache(key, value): def set_job_cache(key, value):
current_app.logger.debug(f"Setting {key} in the job_cache.") current_app.logger.debug(f"Setting {key} in the job_cache to {value}.")
job_cache = current_app.config["job_cache"] job_cache = current_app.config["job_cache"]
job_cache[key] = (value, time.time() + 8 * 24 * 60 * 60) job_cache[key] = (value, time.time() + 8 * 24 * 60 * 60)
@@ -37,7 +43,7 @@ def get_job_cache(key):
if ret is None: if ret is None:
current_app.logger.warning(f"Could not find {key} in the job_cache.") current_app.logger.warning(f"Could not find {key} in the job_cache.")
else: else:
current_app.logger.debug(f"Got {key} from job_cache.") current_app.logger.debug(f"Got {key} from job_cache with value {ret}.")
return ret return ret
@@ -185,18 +191,20 @@ def read_s3_file(bucket_name, object_key, s3res):
""" """
try: try:
job_id = get_job_id_from_s3_object_key(object_key) job_id = get_job_id_from_s3_object_key(object_key)
service_id = get_service_id_from_key(object_key)
if get_job_cache(job_id) is None: if get_job_cache(job_id) is None:
object = ( job = (
s3res.Object(bucket_name, object_key) s3res.Object(bucket_name, object_key)
.get()["Body"] .get()["Body"]
.read() .read()
.decode("utf-8") .decode("utf-8")
) )
set_job_cache(job_id, object) set_job_cache(job_id, job)
set_job_cache(f"{job_id}_phones", extract_phones(object)) set_job_cache(f"{job_id}_phones", extract_phones(job, service_id, job_id))
set_job_cache( set_job_cache(
f"{job_id}_personalisation", f"{job_id}_personalisation",
extract_personalisation(object), extract_personalisation(job),
) )
except LookupError: except LookupError:
@@ -217,8 +225,8 @@ def get_s3_files():
f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200" f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200"
) )
try: try:
with ThreadPoolExecutor() as executor: for object_key in object_keys:
executor.map(lambda key: read_s3_file(bucket_name, key, s3res), object_keys) read_s3_file(bucket_name, object_key, s3res)
except Exception: except Exception:
current_app.logger.exception("Connection pool issue") current_app.logger.exception("Connection pool issue")