Merge pull request #1719 from GSA/blocking

add sleep, more error handling, better debug message

app/aws/s3.py

@@ -224,11 +224,21 @@ def get_s3_files():
     current_app.logger.info(
         f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200"
     )
+    count = 0
     try:
         for object_key in object_keys:
             read_s3_file(bucket_name, object_key, s3res)
+            count = count + 1
+            eventlet.sleep(0.2)
     except Exception:
-        current_app.logger.exception("Connection pool issue")
+        current_app.logger.exception(
+            f"Trouble reading {object_key} which is # {count} during cache regeneration"
+        )
+    except OSError as e:
+        current_app.logger.exception(
+            f"Egress proxy issue reading {object_key} which is # {count}"
+        )
+        raise e
 
     current_app.logger.info(
         f"job_cache length after regen: {len_job_cache()} #notify-debug-admin-1200"

tests/app/aws/test_s3.py

@@ -611,4 +611,6 @@ def test_get_s3_files_handles_exception(mocker):
     ]
     mock_read_s3_file.assert_has_calls(calls, any_order=True)
 
-    mock_current_app.logger.exception.assert_called_with("Connection pool issue")
+    mock_current_app.logger.exception.assert_called_with(
+        "Trouble reading file2.csv which is # 1 during cache regeneration"
+    )