darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-05 10:42:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

optimize S3 partitioning

Browse Source
This commit is contained in:
Kenneth Kehl
2024-09-27 08:38:11 -07:00
parent 0736c982f0
commit 26ffb931c9
1 changed files with 28 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
app/aws/s3.py
View File

@@ -12,6 +12,7 @@ from app.clients import AWS_CLIENT_CONFIG
from notifications_utils import aware_utcnow
FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
NEW_FILE_LOCATION_STRUCTURE = "{}-service-notify/{}.csv"
# Temporarily extend cache to 7 days
ttl = 60 * 60 * 24 * 7
@@ -211,6 +212,21 @@ def file_exists(file_location):
def get_job_location(service_id, job_id):
return (
current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id),
current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"],
current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"],
current_app.config["CSV_UPLOAD_BUCKET"]["region"],
)
def get_old_job_location(service_id, job_id):
"""
This is deprecated. We are transitioning to NEW_FILE_LOCATION_STRUCTURE,
but it will take a few days where we have to support both formats.
Remove this when everything works with the NEW_FILE_LOCATION_STRUCTURE.
"""
return (
current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
FILE_LOCATION_STRUCTURE.format(service_id, job_id),
@@ -239,9 +255,11 @@ def get_job_from_s3(service_id, job_id):
max_retries = 4
backoff_factor = 0.2
if not file_exists(FILE_LOCATION_STRUCTURE.format(service_id, job_id)):
if not file_exists(
FILE_LOCATION_STRUCTURE.format(service_id, job_id)
) and not file_exists(NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id)):
current_app.logger.error(
f"This file does not exist {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}"
f"This file with service_id {service_id} and job_id {job_id} does not exist"
)
return None
@@ -249,6 +267,9 @@ def get_job_from_s3(service_id, job_id):
try:
obj = get_s3_object(*get_job_location(service_id, job_id))
# TODO remove this when we have fully transitioned to the NEW_FILE_LOCATION_STRUCTURE
if obj is None:
obj = get_s3_object(*get_old_job_location(service_id, job_id))
return obj.get()["Body"].read().decode("utf-8")
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] in [
@@ -257,7 +278,7 @@ def get_job_from_s3(service_id, job_id):
"SlowDown",
]:
current_app.logger.exception(
f"Retrying job fetch {FILE_LOCATION_STRUCTURE.format(service_id, job_id)} retry_count={retries}",
f"Retrying job fetch retry_count={retries}",
)
retries += 1
sleep_time = backoff_factor * (2**retries) # Exponential backoff
@@ -266,18 +287,18 @@ def get_job_from_s3(service_id, job_id):
else:
# Typically this is "NoSuchKey"
current_app.logger.exception(
f"Failed to get job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}",
f"Failed to get job with service_id {service_id} job_id {job_id}",
)
return None
except Exception:
current_app.logger.exception(
f"Failed to get job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)} retry_count={retries}",
f"Failed to get job with service_id {service_id} job_id {job_id}retry_count={retries}",
)
return None
current_app.logger.error(
f"Never retrieved job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}",
f"Never retrieved job with service_id {service_id} job_id {job_id}",
)
return None
@@ -358,7 +379,7 @@ def get_phone_number_from_s3(service_id, job_id, job_row_number):
if job is None:
current_app.logger.error(
f"Couldnt find phone for job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)} because job is missing"
f"Couldnt find phone for job with service_id {service_id} job_id {job_id} because job is missing"
)
return "Unavailable"