Refactored reports to use pregenerated docs instead (#2831)

* Refactored reports to use pregenerated docs instead

* Fixed e2e test

* Fixed anothr bug

* Cleanup

* Fixed timezone conversion

* Updated ref files

* Updated reference files, refreshed ui/ux for report generation. Buttons toggle on and off based on if report exists

* Fixed linting errors, removed pytz

* Fixed test failure

* e2e test fix

* Speeding up unit tests

* Removed python time library that was causing performance issues with unit tests

* Updated poetry lock

* Unit test improvements

* Made change that ken reccomended

tests/app/main/views/test_accept_invite.py

@@ -892,7 +892,6 @@ def test_existing_email_auth_user_with_phone_can_set_sms_auth(
     api_user_active,
     service_one,
     sample_invite,
-    mock_get_existing_user_by_email,
     mock_check_invite_token,
     mock_accept_invite,
     mock_update_user_attribute,
@@ -903,6 +902,11 @@ def test_existing_email_auth_user_with_phone_can_set_sms_auth(
     service_one["permissions"].append(ServicePermission.EMAIL_AUTH)
     sample_invite["auth_type"] = "sms_auth"
 
+    # Mock get_user_by_email explicitly to avoid hanging
+    mock_get_existing_user_by_email = mocker.patch(
+        "app.user_api_client.get_user_by_email", return_value=api_user_active
+    )
+
     client_request.get(
         "main.accept_invite",
         token="thisisnotarealtoken",

tests/app/main/views/test_activity.py

@@ -302,6 +302,10 @@ def test_download_links_show_when_data_available(
         "app.job_api_client.get_page_of_jobs", return_value=mock_jobs_with_data
     )
     mocker.patch("app.job_api_client.get_immediate_jobs", return_value=[{"id": "job1"}])
+    mocker.patch("app.s3_client.check_s3_file_exists", return_value=True)
+    mock_obj = mocker.Mock()
+    mock_obj.content_length = 1024
+    mocker.patch("app.s3_client.s3_csv_client.get_csv_upload", return_value=mock_obj)
 
     page = client_request.get(
         "main.all_jobs_activity",
@@ -309,11 +313,10 @@ def test_download_links_show_when_data_available(
     )
 
     assert "Download recent reports" in page.text
-    assert "Download all data last 24 hours" in page.text
-    assert "Download all data last 3 days" in page.text
-    assert "Download all data last 5 days" in page.text
-    assert "Download all data last 7 days" in page.text
-    assert "No recent activity to download" not in page.text
+    assert "Yesterday" in page.text
+    assert "Last 3 days" in page.text
+    assert "Last 5 days" in page.text
+    assert "Last 7 days" in page.text
 
 
 def test_download_links_partial_data_available(
@@ -338,6 +341,10 @@ def test_download_links_partial_data_available(
         "app.job_api_client.get_page_of_jobs", side_effect=mock_get_page_of_jobs
     )
     mocker.patch("app.job_api_client.get_immediate_jobs", return_value=[])
+    mocker.patch("app.s3_client.check_s3_file_exists", return_value=True)
+    mock_obj = mocker.Mock()
+    mock_obj.content_length = 2048
+    mocker.patch("app.s3_client.s3_csv_client.get_csv_upload", return_value=mock_obj)
 
     page = client_request.get(
         "main.all_jobs_activity",
@@ -345,10 +352,10 @@ def test_download_links_partial_data_available(
     )
 
     assert "Download recent reports" in page.text
-    assert "Download all data last 24 hours" in page.text
-    assert "Download all data last 3 days" not in page.text
-    assert "Download all data last 5 days" in page.text
-    assert "Download all data last 7 days" not in page.text
+    assert "Yesterday" in page.text
+    assert "Last 3 days" in page.text
+    assert "Last 5 days" in page.text
+    assert "Last 7 days" in page.text
     assert "No recent activity to download" not in page.text
 
 
@@ -362,6 +369,7 @@ def test_download_links_no_data_available(
 
     mocker.patch("app.job_api_client.get_page_of_jobs", return_value=mock_jobs_empty)
     mocker.patch("app.job_api_client.get_immediate_jobs", return_value=[])
+    mocker.patch("app.s3_client.check_s3_file_exists", return_value=False)
 
     page = client_request.get(
         "main.all_jobs_activity",
@@ -369,14 +377,11 @@ def test_download_links_no_data_available(
     )
 
     assert "Download recent reports" in page.text
-    assert "Download all data last 24 hours" not in page.text
-    assert "Download all data last 3 days" not in page.text
-    assert "Download all data last 5 days" not in page.text
-    assert "Download all data last 7 days" not in page.text
-    assert (
-        "No recent activity to download. Download links will appear when jobs are available."
-        in page.text
-    )
+    assert "Yesterday" in page.text
+    assert "No messages sent" in page.text
+    assert "Last 3 days - No messages sent" in page.text
+    assert "Last 5 days - No messages sent" in page.text
+    assert "Last 7 days - No messages sent" in page.text
 
 
 def test_download_not_available_to_users_without_dashboard(

tests/app/main/views/test_download_notifications_csv_s3.py

@@ -0,0 +1,111 @@
+from unittest.mock import patch
+
+from app.main.views.notifications import PERIOD_TO_S3_FILENAME
+from tests.conftest import SERVICE_ONE_ID
+
+
+def test_period_to_s3_filename_mapping():
+    assert PERIOD_TO_S3_FILENAME["one_day"] == "1-day-report"
+    assert PERIOD_TO_S3_FILENAME["three_day"] == "3-day-report"
+    assert PERIOD_TO_S3_FILENAME["five_day"] == "5-day-report"
+    assert PERIOD_TO_S3_FILENAME["seven_day"] == "7-day-report"
+
+
+@patch("app.main.views.notifications.s3download")
+@patch("app.main.views.notifications.generate_notifications_csv")
+def test_job_based_reports_dont_use_s3(
+    mock_generate_csv,
+    mock_s3download,
+    client_request,
+    service_one,
+    mock_get_service_data_retention,
+):
+    mock_generate_csv.return_value = iter(["test,data\n"])
+
+    response = client_request.get_response(
+        "main.download_notifications_csv",
+        service_id=SERVICE_ONE_ID,
+        number_of_days="one_day",
+        message_type="sms",
+        job_id="test-job-123",
+        _test_page_title=False,
+    )
+
+    mock_s3download.assert_not_called()
+    mock_generate_csv.assert_called_once()
+    assert response.status_code == 200
+
+
+@patch("app.main.views.notifications.s3download")
+@patch("app.main.views.notifications.generate_notifications_csv")
+def test_general_reports_use_s3(
+    mock_generate_csv,
+    mock_s3download,
+    client_request,
+    service_one,
+    mock_get_service_data_retention,
+):
+    mock_s3download.return_value = b"s3,csv,content\n"
+
+    response = client_request.get_response(
+        "main.download_notifications_csv",
+        service_id=SERVICE_ONE_ID,
+        number_of_days="three_day",
+        message_type="sms",
+        _test_page_title=False,
+    )
+
+    mock_s3download.assert_called_once_with(SERVICE_ONE_ID, "3-day-report")
+    mock_generate_csv.assert_not_called()
+    assert response.status_code == 200
+
+
+@patch("app.main.views.notifications.s3download")
+def test_missing_s3_file_redirects_gracefully(
+    mock_s3download,
+    client_request,
+    service_one,
+    mock_get_service_data_retention,
+):
+    from notifications_utils.s3 import S3ObjectNotFound
+
+    mock_s3download.side_effect = S3ObjectNotFound(
+        {"Error": {"Code": "NoSuchKey"}}, "GetObject"
+    )
+
+    # Verify that when an S3 file is missing, we redirect gracefully
+    # instead of showing a 500 error
+    client_request.get(
+        "main.download_notifications_csv",
+        service_id=SERVICE_ONE_ID,
+        number_of_days="five_day",
+        message_type="sms",
+        _expected_redirect=f"/services/{SERVICE_ONE_ID}/notifications/sms?status=sending,delivered,failed",
+    )
+
+    # The redirect happens, which means no 500 error occurred
+    mock_s3download.assert_called_once_with(SERVICE_ONE_ID, "5-day-report")
+
+
+@patch("app.main.views.notifications.convert_s3_csv_timestamps")
+@patch("app.main.views.notifications.s3download")
+def test_s3_csv_gets_timezone_converted(
+    mock_s3download,
+    mock_convert,
+    client_request,
+    service_one,
+    mock_get_service_data_retention,
+):
+    mock_s3download.return_value = b"csv,data"
+    mock_convert.return_value = iter(["converted,csv,data\n"])
+
+    response = client_request.get_response(
+        "main.download_notifications_csv",
+        service_id=SERVICE_ONE_ID,
+        number_of_days="three_day",
+        message_type="sms",
+        _test_page_title=False,
+    )
+
+    mock_convert.assert_called_once_with(b"csv,data")
+    assert response.status_code == 200

tests/app/main/views/test_jobs_activity.py

@@ -52,6 +52,8 @@ def test_all_activity(
         "app.job_api_client.get_page_of_jobs", return_value=MOCK_JOBS
     )
     mocker.patch("app.job_api_client.get_immediate_jobs", return_value=[])
+    mocker.patch("app.s3_client.check_s3_file_exists", return_value=False)
+    mocker.patch("app.s3_client.s3_csv_client.get_csv_upload", return_value=None)
 
     response = client_request.get_response(
         "main.all_jobs_activity",
@@ -138,6 +140,8 @@ def test_all_activity_no_jobs(client_request, mocker):
         },
     )
     mocker.patch("app.job_api_client.get_immediate_jobs", return_value=[])
+    mocker.patch("app.s3_client.check_s3_file_exists", return_value=False)
+    mocker.patch("app.s3_client.s3_csv_client.get_csv_upload", return_value=None)
     response = client_request.get_response(
         "main.all_jobs_activity",
         service_id=SERVICE_ONE_ID,
@@ -191,6 +195,8 @@ def test_all_activity_pagination(client_request, mocker):
         },
     )
     mocker.patch("app.job_api_client.get_immediate_jobs", return_value=[])
+    mocker.patch("app.s3_client.check_s3_file_exists", return_value=False)
+    mocker.patch("app.s3_client.s3_csv_client.get_csv_upload", return_value=None)
 
     response = client_request.get_response(
         "main.all_jobs_activity",
@@ -228,6 +234,8 @@ def test_all_activity_filters(client_request, mocker, filter_type, expected_limi
         "app.job_api_client.get_page_of_jobs", return_value=MOCK_JOBS
     )
     mocker.patch("app.job_api_client.get_immediate_jobs", return_value=[])
+    mocker.patch("app.s3_client.check_s3_file_exists", return_value=False)
+    mocker.patch("app.s3_client.s3_csv_client.get_csv_upload", return_value=None)
 
     kwargs = {"filter": filter_type} if filter_type else {}
     response = client_request.get_response(
@@ -239,7 +247,10 @@ def test_all_activity_filters(client_request, mocker, filter_type, expected_limi
 
     if expected_limit_days:
         mock_get_page_of_jobs.assert_any_call(
-            SERVICE_ONE_ID, page=current_page, limit_days=expected_limit_days, use_processing_time=True
+            SERVICE_ONE_ID,
+            page=current_page,
+            limit_days=expected_limit_days,
+            use_processing_time=True,
         )
     else:
         mock_get_page_of_jobs.assert_any_call(SERVICE_ONE_ID, page=current_page)

tests/app/main/views/test_tour.py

@@ -34,7 +34,12 @@ def test_should_200_for_tour_start(
         "service one: ((one)) ((two)) ((three))"
     )
 
-    assert page.select("a.usa-button")[0]["href"] == url_for(
+    # Find the tour step button specifically, not just any usa-button
+    tour_buttons = [
+        btn for btn in page.select("a.usa-button") if "tour" in btn.get("href", "")
+    ]
+    assert len(tour_buttons) > 0, "No tour button found"
+    assert tour_buttons[0]["href"] == url_for(
         ".tour_step", service_id=SERVICE_ONE_ID, template_id=fake_uuid, step_index=1
     )