Refactored reports to use pregenerated docs instead (#2831)

* Refactored reports to use pregenerated docs instead

* Fixed e2e test

* Fixed anothr bug

* Cleanup

* Fixed timezone conversion

* Updated ref files

* Updated reference files, refreshed ui/ux for report generation. Buttons toggle on and off based on if report exists

* Fixed linting errors, removed pytz

* Fixed test failure

* e2e test fix

* Speeding up unit tests

* Removed python time library that was causing performance issues with unit tests

* Updated poetry lock

* Unit test improvements

* Made change that ken reccomended

app/main/forms.py

@@ -3,7 +3,6 @@ from datetime import datetime, timedelta
 from itertools import chain
 from numbers import Number
 
-import pytz
 from flask import render_template, request
 from flask_login import current_user
 from flask_wtf import FlaskForm as Form
@@ -62,7 +61,7 @@ from app.main.validators import (
 )
 from app.models.organization import Organization
 from app.utils import merge_jsonlike
-from app.utils.csv import get_user_preferred_timezone
+from app.utils.csv import get_user_preferred_timezone, get_user_preferred_timezone_obj
 from app.utils.user_permissions import all_ui_permissions, permission_options
 from notifications_utils.formatters import strip_all_whitespace
 from notifications_utils.insensitive_dict import InsensitiveDict
@@ -70,7 +69,7 @@ from notifications_utils.recipients import InvalidPhoneError, validate_phone_num
 
 
 def get_time_value_and_label(future_time):
-    preferred_tz = pytz.timezone(get_user_preferred_timezone())
+    preferred_tz = get_user_preferred_timezone_obj()
     return (
         future_time.astimezone(preferred_tz).isoformat(),
         "{} at {} {}".format(
@@ -90,7 +89,7 @@ def get_human_time(time):
 def get_human_day(time, prefix_today_with="T"):
     #  Add 1 hour to get ‘midnight today’ instead of ‘midnight tomorrow’
 
-    preferred_tz = pytz.timezone(get_user_preferred_timezone())
+    preferred_tz = get_user_preferred_timezone_obj()
     time = (time - timedelta(hours=1)).strftime("%A")
     if time == datetime.now(preferred_tz).strftime("%A"):
         return "{}oday".format(prefix_today_with)
@@ -101,12 +100,12 @@ def get_human_day(time, prefix_today_with="T"):
 
 def get_furthest_possible_scheduled_time():
     # We want local time to find date boundaries
-    preferred_tz = pytz.timezone(get_user_preferred_timezone())
+    preferred_tz = get_user_preferred_timezone_obj()
     return (datetime.now(preferred_tz) + timedelta(days=4)).replace(hour=0)
 
 
 def get_next_hours_until(until):
-    preferred_tz = pytz.timezone(get_user_preferred_timezone())
+    preferred_tz = get_user_preferred_timezone_obj()
     now = datetime.now(preferred_tz)
     hours = int((until - now).total_seconds() / (60 * 60))
     return [
@@ -116,7 +115,7 @@ def get_next_hours_until(until):
 
 
 def get_next_days_until(until):
-    preferred_tz = pytz.timezone(get_user_preferred_timezone())
+    preferred_tz = get_user_preferred_timezone_obj()
     now = datetime.now(preferred_tz)
     days = int((until - now).total_seconds() / (60 * 60 * 24))
     return [

app/main/views/activity.py

@@ -1,9 +1,12 @@
+import gevent
 from flask import abort, render_template, request, url_for
 
 from app import current_service, job_api_client
 from app.enums import NotificationStatus, ServicePermission
 from app.formatters import get_time_left
 from app.main import main
+from app.s3_client import check_s3_file_exists
+from app.s3_client.s3_csv_client import get_csv_upload
 from app.utils.pagination import (
     generate_next_dict,
     generate_pagination_pages,
@@ -13,26 +16,37 @@ from app.utils.pagination import (
 from app.utils.user import user_has_permissions
 
 
-def get_download_availability(service_id):
-    jobs_1_day = job_api_client.get_page_of_jobs(service_id, page=1, limit_days=1)
-    jobs_3_days = job_api_client.get_page_of_jobs(service_id, page=1, limit_days=3)
-    jobs_5_days = job_api_client.get_page_of_jobs(service_id, page=1, limit_days=5)
-    jobs_7_days = job_api_client.get_immediate_jobs(service_id)
+def get_report_info(service_id, report_name):
+    try:
+        obj = get_csv_upload(service_id, report_name)
+        if check_s3_file_exists(obj):
+            size_bytes = obj.content_length
+            if size_bytes < 1024:
+                size_str = f"{size_bytes} B"
+            elif size_bytes < 1024 * 1024:
+                size_str = f"{size_bytes / 1024:.1f} KB"
+            else:
+                size_str = f"{size_bytes / (1024 * 1024):.1f} MB"
+            return {"available": True, "size": size_str}
+    except Exception:
+        return {"available": False, "size": None}
+    return {"available": False, "size": None}
 
-    has_1_day_data = len(generate_job_dict(jobs_1_day)) > 0
-    has_3_day_data = len(generate_job_dict(jobs_3_days)) > 0
-    has_5_day_data = len(generate_job_dict(jobs_5_days)) > 0
-    has_7_day_data = len(jobs_7_days) > 0
+
+def get_download_availability(service_id):
+    report_names = ["1-day-report", "3-day-report", "5-day-report", "7-day-report"]
+
+    greenlets = [
+        gevent.spawn(get_report_info, service_id, name) for name in report_names
+    ]
+    gevent.joinall(greenlets)
+    results = [g.value for g in greenlets]
 
     return {
-        "has_1_day_data": has_1_day_data,
-        "has_3_day_data": has_3_day_data,
-        "has_5_day_data": has_5_day_data,
-        "has_7_day_data": has_7_day_data,
-        "has_any_download_data": has_1_day_data
-        or has_3_day_data
-        or has_5_day_data
-        or has_7_day_data,
+        "report_1_day": results[0],
+        "report_3_day": results[1],
+        "report_5_day": results[2],
+        "report_7_day": results[3],
     }
 
 

app/main/views/notifications.py

@@ -1,10 +1,13 @@
 # -*- coding: utf-8 -*-
 from datetime import datetime
+from zoneinfo import ZoneInfo
 
 from flask import (
     Response,
+    current_app,
     flash,
     jsonify,
+    redirect,
     render_template,
     request,
     stream_with_context,
@@ -15,6 +18,7 @@ from app import current_service, job_api_client, notification_api_client
 from app.enums import ServicePermission
 from app.main import main
 from app.notify_client.api_key_api_client import KEY_TYPE_TEST
+from app.s3_client.s3_csv_client import s3download
 from app.utils import (
     DELIVERED_STATUSES,
     FAILURE_STATUSES,
@@ -23,8 +27,10 @@ from app.utils import (
     set_status_filters,
 )
 from app.utils.csv import generate_notifications_csv, get_user_preferred_timezone
+from app.utils.s3_csv import convert_s3_csv_timestamps
 from app.utils.templates import get_template
 from app.utils.user import user_has_permissions
+from notifications_utils.s3 import S3ObjectNotFound
 
 
 @main.route("/services/<uuid:service_id>/notification/<uuid:notification_id>")
@@ -135,6 +141,14 @@ def get_all_personalisation_from_notification(notification):
     return notification["personalisation"]
 
 
+PERIOD_TO_S3_FILENAME = {
+    "one_day": "1-day-report",
+    "three_day": "3-day-report",
+    "five_day": "5-day-report",
+    "seven_day": "7-day-report",
+}
+
+
 @main.route("/services/<uuid:service_id>/download-notifications.csv")
 @user_has_permissions(ServicePermission.VIEW_ACTIVITY)
 def download_notifications_csv(service_id):
@@ -144,14 +158,52 @@ def download_notifications_csv(service_id):
     service_data_retention_days = current_service.get_days_of_retention(
         filter_args.get("message_type")[0], number_of_days
     )
-    file_time = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
-    file_time = f"{file_time} {get_user_preferred_timezone()}"
+    user_tz_name = get_user_preferred_timezone()
+    user_tz = ZoneInfo(user_tz_name)
+    file_time = datetime.now(user_tz).strftime("%Y-%m-%d %I:%M:%S %p")
+    file_time = f"{file_time} {user_tz_name}"
 
+    job_id = request.args.get("job_id")
+    if not job_id and number_of_days in PERIOD_TO_S3_FILENAME:
+        try:
+            s3_report_id = PERIOD_TO_S3_FILENAME[number_of_days]
+            current_app.logger.info(
+                f"User is attempting to download {s3_report_id} for service {service_id}"
+            )
+            s3_file_content = s3download(service_id, s3_report_id)
+            return Response(
+                stream_with_context(convert_s3_csv_timestamps(s3_file_content)),
+                mimetype="text/csv",
+                headers={
+                    "Content-Disposition": 'inline; filename="{} - {} - {} report.csv"'.format(
+                        file_time,
+                        filter_args["message_type"][0],
+                        current_service.name,
+                    )
+                },
+            )
+        except S3ObjectNotFound:
+            # Edge case: File was deleted between page load and download attempt
+            current_app.logger.warning(
+                f"File {s3_report_id} was expected but not found for service {service_id}. "
+                "It may have been deleted after page load."
+            )
+            flash(
+                "The report is no longer available. Please refresh the page.", "default"
+            )
+            return redirect(
+                url_for(
+                    "main.view_notifications",
+                    service_id=service_id,
+                    message_type=filter_args["message_type"][0],
+                    status="sending,delivered,failed",
+                )
+            )
     return Response(
         stream_with_context(
             generate_notifications_csv(
                 service_id=service_id,
-                job_id=None,
+                job_id=job_id,
                 status=filter_args.get("status"),
                 page=request.args.get("page", 1),
                 page_size=10000,

app/main/views/performance.py

@@ -3,17 +3,16 @@ from itertools import groupby
 from operator import itemgetter
 from statistics import mean
 
-import pytz
 from flask import render_template
 
 from app import performance_dashboard_api_client, status_api_client
 from app.main import main
-from app.utils.csv import get_user_preferred_timezone
+from app.utils.csv import get_user_preferred_timezone_obj
 
 
 @main.route("/performance")
 def performance():
-    preferred_tz = pytz.timezone(get_user_preferred_timezone())
+    preferred_tz = get_user_preferred_timezone_obj()
     stats = performance_dashboard_api_client.get_performance_dashboard_stats(
         start_date=(datetime.now(preferred_tz) - timedelta(days=7)).date(),
         end_date=datetime.now(preferred_tz).date(),