darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-23 07:46:06 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1356 from GSA/debug_s3_partitioning

Browse Source 
debug s3 partitioning
This commit is contained in:
Carlo Costino
2024-10-03 11:06:11 -04:00
committed by GitHub
parent ae5c0d2d91 4abd54094d
commit f87e71060b
3 changed files with 10 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
app/aws/s3.py
View File

@@ -264,6 +264,9 @@ def file_exists(file_location):
def get_job_location(service_id, job_id): def get_job_location(service_id, job_id):
current_app.logger.info(
f"#s3-partitioning NEW JOB_LOCATION: {NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id)}"
)
return ( return (
current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id), NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id),
@@ -279,6 +282,9 @@ def get_old_job_location(service_id, job_id):
but it will take a few days where we have to support both formats. but it will take a few days where we have to support both formats.
Remove this when everything works with the NEW_FILE_LOCATION_STRUCTURE. Remove this when everything works with the NEW_FILE_LOCATION_STRUCTURE.
""" """
current_app.logger.info(
f"#s3-partitioning OLD JOB LOCATION: {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}"
)
return ( return (
current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], current_app.config["CSV_UPLOAD_BUCKET"]["bucket"],
FILE_LOCATION_STRUCTURE.format(service_id, job_id), FILE_LOCATION_STRUCTURE.format(service_id, job_id),
@@ -486,6 +492,9 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
def get_job_metadata_from_s3(service_id, job_id): def get_job_metadata_from_s3(service_id, job_id):
current_app.logger.info(
f"#s3-partitioning CALLING GET_JOB_METADATA with {service_id}, {job_id}"
)
obj = get_s3_object(*get_job_location(service_id, job_id)) obj = get_s3_object(*get_job_location(service_id, job_id))
return obj.get()["Metadata"] return obj.get()["Metadata"]

1
app/job/rest.py
View File

@@ -175,6 +175,7 @@ def create_job(service_id):
original_file_name = data.get("original_file_name") original_file_name = data.get("original_file_name")
data.update({"service": service_id}) data.update({"service": service_id})
try: try:
current_app.logger.info(f"#s3-partitioning DATA IN CREATE_JOB: {data}")
data.update(**get_job_metadata_from_s3(service_id, data["id"])) data.update(**get_job_metadata_from_s3(service_id, data["id"]))
except KeyError: except KeyError:
raise InvalidRequest( raise InvalidRequest(

1
app/user/rest.py
View File

@@ -599,7 +599,6 @@ def fetch_user_by_email():
fetched_user = get_user_by_email(email["email"]) fetched_user = get_user_by_email(email["email"])
debug_not_production(hilite(f"fetched user is {fetched_user}")) debug_not_production(hilite(f"fetched user is {fetched_user}"))
result = fetched_user.serialize() result = fetched_user.serialize()
debug_not_production(hilite(f"result is serialized to {result}"))
return jsonify(data=result) return jsonify(data=result)
except Exception as e: except Exception as e:
debug_not_production(hilite(f"Failed with {e}!!")) debug_not_production(hilite(f"Failed with {e}!!"))