From 438ab92fd6d92a5e3e18519ee451b34f8a4206df Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 15 Aug 2024 17:11:31 -0700 Subject: [PATCH 1/4] fix job retrieval --- app/aws/s3.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index 2b7feaf15..290203ec1 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -173,7 +173,7 @@ def get_job_and_metadata_from_s3(service_id, job_id): def get_job_from_s3(service_id, job_id): retries = 0 - max_retries = 5 + max_retries = 3 backoff_factor = 1 while retries < max_retries: @@ -190,11 +190,14 @@ def get_job_from_s3(service_id, job_id): sleep_time = backoff_factor * (2**retries) # Exponential backoff time.sleep(sleep_time) continue + else: + current_app.logger.error("Failed to get job from bucket", exc_info=True) + return None except Exception as e: - current_app.logger.error(f"Failed to get object from bucket {e}") - raise + current_app.logger.error(f"Failed to get job from bucket", exc_info=True) + return None - raise Exception("Failed to get object after 5 attempts") + raise Exception("Failed to get object after 3 attempts") def incr_jobs_cache_misses(): From f332e9f907eef683672cee480f42f2690fcfd119 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 15 Aug 2024 17:14:24 -0700 Subject: [PATCH 2/4] fix job retrieval --- app/aws/s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index 290203ec1..e04d7ae98 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -193,8 +193,8 @@ def get_job_from_s3(service_id, job_id): else: current_app.logger.error("Failed to get job from bucket", exc_info=True) return None - except Exception as e: - current_app.logger.error(f"Failed to get job from bucket", exc_info=True) + except Exception: + current_app.logger.error("Failed to get job from bucket", exc_info=True) return None raise Exception("Failed to get object after 3 attempts") From 0c3b382562e14235699457604ed70a71f28e2f4c Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 15 Aug 2024 17:25:03 -0700 Subject: [PATCH 3/4] fix test --- tests/app/aws/test_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index d625f6b06..a148855ac 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -102,7 +102,7 @@ def test_get_job_from_s3_exponential_backoff(mocker): mocker.patch("app.aws.s3.get_s3_object", side_effect=mock_s3_get_object_slowdown) with pytest.raises(Exception) as exc_info: get_job_from_s3("service_id", "job_id") - assert "Failed to get object after 5 attempts" in str(exc_info) + assert "Failed to get object after 3 attempts" in str(exc_info) @pytest.mark.parametrize( From e2e3501aa4e3a053cf741633c849939dbe5c4472 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 15 Aug 2024 17:55:59 -0700 Subject: [PATCH 4/4] improve debug --- app/aws/s3.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/aws/s3.py b/app/aws/s3.py index e04d7ae98..a907254bb 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -191,10 +191,16 @@ def get_job_from_s3(service_id, job_id): time.sleep(sleep_time) continue else: - current_app.logger.error("Failed to get job from bucket", exc_info=True) + current_app.logger.error( + f"Failed to get job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)} from bucket", + exc_info=True, + ) return None except Exception: - current_app.logger.error("Failed to get job from bucket", exc_info=True) + current_app.logger.error( + f"Failed to get job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)} from bucket", + exc_info=True, + ) return None raise Exception("Failed to get object after 3 attempts")