diff --git a/app/aws/s3.py b/app/aws/s3.py index 9c397fc1a..8ca10676e 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -123,10 +123,48 @@ def list_s3_objects(): ) +def get_notification_reports(service_id): + + bucket_name = _get_bucket_name() + s3_client = get_s3_client() + # Our reports only support 7 days, but pull 8 days to avoid + # any edge cases + time_limit = aware_utcnow() - datetime.timedelta(days=8) + reports = [] + try: + response = s3_client.list_objects_v2(Bucket=bucket_name) + while True: + for obj in response.get("Contents", []): + if obj["LastModified"] >= time_limit: + if service_id in obj["Key"] and "report" in obj["Key"]: + reports.append(obj) + if "NextContinuationToken" in response: + response = s3_client.list_objects_v2( + Bucket=bucket_name, + ContinuationToken=response["NextContinuationToken"], + ) + else: + break + except Exception as e: + current_app.logger.exception( + f"An error occurred while regenerating cache #notify-debug-admin-1200: {str(e)}", + ) + return reports + + def get_bucket_name(): return current_app.config["CSV_UPLOAD_BUCKET"]["bucket"] +def delete_s3_object(key): + + try: + remove_csv_object(key) + current_app.logger.debug(f"#delete-s3-object Deleted: {key}") + except botocore.exceptions.ClientError: + current_app.logger.exception(f"Couldn't delete {key}") + + def cleanup_old_s3_objects(): bucket_name = get_bucket_name() diff --git a/app/celery/tasks.py b/app/celery/tasks.py index b665f2d09..f11a13e54 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -2,7 +2,6 @@ import csv import io import json import os -import uuid import gevent from celery.signals import task_postrun @@ -555,7 +554,8 @@ def process_incomplete_job(job_id): def _generate_notifications_report(service_id, report_id, limit_days): - current_app.logger.debug(hilite("ENTER _generate_notifications_report()")) + + # Hard code these values for now page = 1 page_size = 20000 include_jobs = True @@ -608,7 +608,9 @@ def _generate_notifications_report(service_id, report_id, limit_days): notifications = [ notification.serialize_for_csv() for notification in pagination.items ] - current_app.logger.debug(hilite(f"NUMBER OF NOTFICIATIONS IS {len(notifications)}")) + current_app.logger.debug( + hilite(f"Number of notifications in report: {len(notifications)}") + ) # We try and get the next page of results to work out if we need provide a pagination link to the next page # in our response if it exists. Note, this could be done instead by changing `count_pages` in the previous @@ -632,10 +634,12 @@ def _generate_notifications_report(service_id, report_id, limit_days): exp_region = current_app.config["CSV_UPLOAD_BUCKET"]["region"] tier = os.getenv("NOTIFY_ENVIRONMENT") raise Exception( - f"NO BUCKET NAME SHOULD BE: {exp_bucket} WITH REGION {exp_region} TIER {tier}" + f"No bucket name should be: {exp_bucket} with region {exp_region} and tier {tier}" ) - current_app.logger.debug(hilite(f"UPLOADING THIS {file_location}")) + # Delete yesterday's version of this report + s3.delete_s3_object(file_location) + s3upload( filedata=csv_bytes, region=region, @@ -644,12 +648,11 @@ def _generate_notifications_report(service_id, report_id, limit_days): access_key=access_key, secret_key=secret_key, ) - current_app.logger.debug(hilite("FINITO")) + current_app.logger.info(f"generate-notifications-report uploaded {file_location}") @notify_celery.task(name="generate-notifications-reports") def generate_notification_reports_task(): - current_app.logger.debug(hilite("ENTER GET ALL NOTIFICATIONS FOR SERVICE@")) services = dao_fetch_all_services(only_active=True) for service in services: @@ -657,7 +660,7 @@ def generate_notification_reports_task(): limit_days = [1, 3, 5, 7] for limit_day in limit_days: - report_id = f"{str(uuid.uuid4())}-{limit_day}-day" + report_id = f"{limit_day}-day-report" _generate_notifications_report(service.id, report_id, limit_day) current_app.logger.info("Notifications report generation complete")