diff --git a/app/aws/s3.py b/app/aws/s3.py index e226f88c2..cbc3d9f2e 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -224,11 +224,21 @@ def get_s3_files(): current_app.logger.info( f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200" ) + count = 0 try: for object_key in object_keys: read_s3_file(bucket_name, object_key, s3res) + count = count + 1 + eventlet.sleep(0.2) except Exception: - current_app.logger.exception("Connection pool issue") + current_app.logger.exception( + f"Trouble reading {object_key} which is # {count} during cache regeneration" + ) + except OSError as e: + current_app.logger.exception( + f"Egress proxy issue reading {object_key} which is # {count}" + ) + raise e current_app.logger.info( f"job_cache length after regen: {len_job_cache()} #notify-debug-admin-1200" diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index cd6b76962..467ce6829 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -611,4 +611,6 @@ def test_get_s3_files_handles_exception(mocker): ] mock_read_s3_file.assert_has_calls(calls, any_order=True) - mock_current_app.logger.exception.assert_called_with("Connection pool issue") + mock_current_app.logger.exception.assert_called_with( + "Trouble reading file2.csv which is # 1 during cache regeneration" + )