Merge branch 'main' into 1543-review-and-update-total-messages-chart-on-dashboard-and-usagehtml-with-new-data-from-backend

app/main/views/dashboard.py

@@ -51,8 +51,12 @@ def service_dashboard(service_id):
     job_response = job_api_client.get_jobs(service_id)["data"]
     service_data_retention_days = 7
 
-    filtered_jobs = [job for job in job_response if job["job_status"] != "cancelled"]
-    sorted_jobs = sorted(filtered_jobs, key=lambda job: job["created_at"], reverse=True)
+    active_jobs = [job for job in job_response if job["job_status"] != "cancelled"]
+    sorted_jobs = sorted(active_jobs, key=lambda job: job["created_at"], reverse=True)
+    job_lists = [
+        {**job_dict, "finished_processing": job_is_finished(job_dict)}
+        for job_dict in sorted_jobs
+    ]
 
     total_messages = service_api_client.get_service_message_ratio(service_id)
     messages_remaining = total_messages.get('messages_remaining', 0)
@@ -61,32 +65,49 @@ def service_dashboard(service_id):
         "views/dashboard/dashboard.html",
         updates_url=url_for(".service_dashboard_updates", service_id=service_id),
         partials=get_dashboard_partials(service_id),
-        jobs=sorted_jobs,
+        jobs=job_lists,
         service_data_retention_days=service_data_retention_days,
         messages_remaining=messages_remaining,
         messages_sent=messages_sent
     )
 
 
-@main.route("/daily_stats.json")
-def get_daily_stats():
-    service_id = session.get("service_id")
-    date_range = get_stats_date_range()
+def job_is_finished(job_dict):
+    done_statuses = [
+        "delivered",
+        "sent",
+        "failed",
+        "technical-failure",
+        "temporary-failure",
+        "permanent-failure",
+        "cancelled",
+    ]
 
+    processed_count = sum(
+        stat["count"]
+        for stat in job_dict["statistics"]
+        if stat["status"] in done_statuses
+    )
+    return job_dict["notification_count"] == processed_count
+
+
+@main.route("/services/<uuid:service_id>/daily-stats.json")
+@user_has_permissions()
+def get_daily_stats(service_id):
+    date_range = get_stats_date_range()
     stats = service_api_client.get_service_notification_statistics_by_day(
         service_id, start_date=date_range["start_date"], days=date_range["days"]
     )
     return jsonify(stats)
 
 
-@main.route("/daily_stats_by_user.json")
-def get_daily_stats_by_user():
-    service_id = session.get("service_id")
+@main.route("/services/<uuid:service_id>/daily-stats-by-user.json")
+@user_has_permissions()
+def get_daily_stats_by_user(service_id):
     date_range = get_stats_date_range()
-    user_id = current_user.id
     stats = service_api_client.get_user_service_notification_statistics_by_day(
         service_id,
-        user_id,
+        user_id=current_user.id,
         start_date=date_range["start_date"],
         days=date_range["days"],
     )

app/main/views/jobs.py

@@ -401,7 +401,9 @@ def get_job_partials(job):
     )
 
     if request.referrer is not None:
-        session["arrived_from_preview_page"] = ("check" in request.referrer) or ("help=0" in request.referrer)
+        session["arrived_from_preview_page"] = ("check" in request.referrer) or (
+            "help=0" in request.referrer
+        )
     else:
         session["arrived_from_preview_page"] = False
 

app/main/views/platform_admin.py

@@ -116,6 +116,70 @@ def download_all_users():
     return response
 
 
+@main.route("/platform-admin/get-redis-report")
+@user_is_platform_admin
+def get_redis_report():
+
+    memory_info = redis_client.info("memory")
+    memory_used = memory_info.get("used_memory_human", "N/A")
+    max_memory = memory_info.get("maxmemory_human", "N/A")
+    if max_memory == "0B":
+        max_memory = "No set limit"
+    mem_fragmentation = memory_info.get("mem_fragmentation_ratio", "N/A")
+    frag_quality = "Swapping (bad)"
+    if mem_fragmentation >= 1.0:
+        frag_quality = "Healthy"
+    if mem_fragmentation > 1.5:
+        frag_quality = "Problematic"
+    if mem_fragmentation > 2.0:
+        frag_quality = "Severe fragmentation"
+
+    frag_note = ""
+    if mem_fragmentation > 2.0:
+        frag_note = "Use MEMORY PURGE.\nReplace multiple small keys with hashes.\nAvoid long keys.\nSet max_memory."
+    elif mem_fragmentation < 1.0:
+        frag_note = "Allocate more RAM.\nSet max_memory."
+
+    keys = redis_client.keys("*")
+    key_details = []
+
+    for key in keys:
+        key_type = redis_client.type(key).decode("utf-8")
+        ttl = redis_client.ttl(key)
+        ttl_str = "No Expiry" if ttl == -1 else f"{ttl} seconds"
+        key_details.append(
+            {"Key": key.decode("utf-8"), "Type": key_type, "TTL": ttl_str}
+        )
+    output = StringIO()
+    writer = csv.writer(
+        output,
+    )
+    writer.writerow(["Redis Report"])
+    writer.writerow([])
+
+    writer.writerow(["Memory"])
+    writer.writerow(["", "Memory Used", memory_used])
+    writer.writerow(["", "Max Memory", max_memory])
+    writer.writerow(["", "Memory Fragmentation Ratio", mem_fragmentation])
+    writer.writerow(["", "Memory Fragmentation Quality", frag_quality, frag_note])
+    writer.writerow([])
+
+    writer.writerow(["Keys Overview"])
+    writer.writerow(["", "TTL", "Type", "Key"])
+    for key_detail in key_details:
+        writer.writerow(
+            ["", key_detail["TTL"], key_detail["Type"], key_detail["Key"][0:50]]
+        )
+
+    csv_data = output.getvalue()
+
+    # Create a direct download response with the CSV data and appropriate headers
+    response = Response(csv_data, content_type="text/csv; charset=utf-8")
+    response.headers["Content-Disposition"] = "attachment; filename=redis.csv"
+
+    return response
+
+
 def is_over_threshold(number, total, threshold):
     percentage = number / total * 100 if total else 0
     return percentage > threshold

app/main/views/sign_in.py

@@ -68,11 +68,12 @@ def _get_access_token(code):  # pragma: no cover
     id_token = get_id_token(response_json)
     nonce = id_token["nonce"]
     nonce_key = f"login-nonce-{unquote(nonce)}"
-    stored_nonce = redis_client.get(nonce_key).decode("utf8")
+    if not os.getenv("NOTIFY_ENVIRONMENT") == "development":
+        stored_nonce = redis_client.get(nonce_key).decode("utf8")
 
-    if nonce != stored_nonce:
-        current_app.logger.error(f"Nonce Error: {nonce} != {stored_nonce}")
-        abort(403)
+        if nonce != stored_nonce:
+            current_app.logger.error(f"Nonce Error: {nonce} != {stored_nonce}")
+            abort(403)
 
     try:
         access_token = response_json["access_token"]
@@ -112,7 +113,7 @@ def _do_login_dot_gov():  # $ pragma: no cover
         verify_key = f"login-verify_email-{unquote(state)}"
         verify_path = bool(redis_client.get(verify_key))
 
-        if not verify_path:
+        if not verify_path and not os.getenv("NOTIFY_ENVIRONMENT") == "development":
             state_key = f"login-state-{unquote(state)}"
             stored_state = unquote(redis_client.get(state_key).decode("utf8"))
             if state != stored_state: