code review feedback and merge from main

app/aws/s3.py

@@ -77,9 +77,42 @@ def list_s3_objects():
             else:
                 break
     except Exception:
-        current_app.logger.error(
+        current_app.logger.exception(
             "An error occurred while regenerating cache #notify-admin-1200",
-            exc_info=True,
+        )
+
+
+def get_bucket_name():
+    return current_app.config["CSV_UPLOAD_BUCKET"]["bucket"]
+
+
+def cleanup_old_s3_objects():
+
+    bucket_name = get_bucket_name()
+
+    s3_client = get_s3_client()
+    # Our reports only support 7 days, but can be scheduled 3 days in advance
+    # Use 14 day for the v1.0 version of this behavior
+    time_limit = aware_utcnow() - datetime.timedelta(days=14)
+    try:
+        response = s3_client.list_objects_v2(Bucket=bucket_name)
+        print(f"RESPONSE = {response}")
+        while True:
+            for obj in response.get("Contents", []):
+                if obj["LastModified"] <= time_limit:
+                    current_app.logger.info(
+                        f"#delete-old-s3-objects Wanting to delete: {obj['LastModified']} {obj['Key']}"
+                    )
+            if "NextContinuationToken" in response:
+                response = s3_client.list_objects_v2(
+                    Bucket=bucket_name,
+                    ContinuationToken=response["NextContinuationToken"],
+                )
+            else:
+                break
+    except Exception:
+        current_app.logger.exception(
+            "#delete-old-s3-objects An error occurred while cleaning up old s3 objects",
         )
 
 
@@ -109,7 +142,7 @@ def get_s3_files():
                     JOBS[job_id] = object
         except LookupError:
             # perhaps our key is not formatted as we expected.  If so skip it.
-            current_app.logger.error("LookupError #notify-admin-1200", exc_info=True)
+            current_app.logger.exception("LookupError #notify-admin-1200")
 
     current_app.logger.info(
         f"JOBS cache length after regen: {len(JOBS)} #notify-admin-1200"
@@ -131,13 +164,13 @@ def download_from_s3(
         result = s3.download_file(bucket_name, s3_key, local_filename)
         current_app.logger.info(f"File downloaded successfully to {local_filename}")
     except botocore.exceptions.NoCredentialsError as nce:
-        current_app.logger.error("Credentials not found", exc_info=True)
+        current_app.logger.exception("Credentials not found")
         raise Exception(nce)
     except botocore.exceptions.PartialCredentialsError as pce:
-        current_app.logger.error("Incomplete credentials provided", exc_info=True)
+        current_app.logger.exception("Incomplete credentials provided")
         raise Exception(pce)
     except Exception:
-        current_app.logger.error("An error occurred", exc_info=True)
+        current_app.logger.exception("An error occurred")
         text = f"EXCEPTION local_filename {local_filename}"
         raise Exception(text)
     return result
@@ -149,8 +182,8 @@ def get_s3_object(bucket_name, file_location, access_key, secret_key, region):
     try:
         return s3.Object(bucket_name, file_location)
     except botocore.exceptions.ClientError:
-        current_app.logger.error(
-            f"Can't retrieve S3 Object from {file_location}", exc_info=True
+        current_app.logger.exception(
+            f"Can't retrieve S3 Object from {file_location}",
         )
 
 
@@ -224,9 +257,8 @@ def get_job_from_s3(service_id, job_id):
                 "RequestTimeout",
                 "SlowDown",
             ]:
-                current_app.logger.error(
+                current_app.logger.exception(
                     f"Retrying job fetch {FILE_LOCATION_STRUCTURE.format(service_id, job_id)} retry_count={retries}",
-                    exc_info=True,
                 )
                 retries += 1
                 sleep_time = backoff_factor * (2**retries)  # Exponential backoff
@@ -234,22 +266,19 @@ def get_job_from_s3(service_id, job_id):
                 continue
             else:
                 # Typically this is "NoSuchKey"
-                current_app.logger.error(
+                current_app.logger.exception(
                     f"Failed to get job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}",
-                    exc_info=True,
                 )
                 return None
 
         except Exception:
-            current_app.logger.error(
+            current_app.logger.exception(
                 f"Failed to get job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)} retry_count={retries}",
-                exc_info=True,
             )
             return None
 
     current_app.logger.error(
         f"Never retrieved job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}",
-        exc_info=True,
     )
     return None
 
@@ -289,7 +318,6 @@ def extract_phones(job):
             phones[job_row] = "Unavailable"
             current_app.logger.error(
                 "Corrupt csv file, missing columns or possibly a byte order mark in the file",
-                exc_info=True,
             )
 
         else:
@@ -352,7 +380,7 @@ def get_phone_number_from_s3(service_id, job_id, job_row_number):
             return "Unavailable"
     else:
         current_app.logger.error(
-            f"Was unable to construct lookup dictionary for job {job_id}", exc_info=True
+            f"Was unable to construct lookup dictionary for job {job_id}"
         )
         return "Unavailable"
 
@@ -399,7 +427,7 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
             return {}
     else:
         current_app.logger.error(
-            f"Was unable to construct lookup dictionary for job {job_id}", exc_info=True
+            f"Was unable to construct lookup dictionary for job {job_id}"
         )
         return {}