Feat: Adds dashboard application to parallel modal evals (#847)
<!-- .github/pull_request_template.md --> ## Description Adds dashboard application to parallel modal evals to enable fast retriever development/evaluation ## DCO Affirmation I affirm that all code in every commit of this pull request conforms to the terms of the Topoteretes Developer Certificate of Origin. --------- Co-authored-by: lxobr <122801072+lxobr@users.noreply.github.com>
This commit is contained in:
parent
4c52ef62aa
commit
7eee769251
1 changed files with 94 additions and 0 deletions
94
cognee/eval_framework/modal_eval_dashboard.py
Normal file
94
cognee/eval_framework/modal_eval_dashboard.py
Normal file
|
|
@ -0,0 +1,94 @@
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import pandas as pd
|
||||||
|
import subprocess
|
||||||
|
import modal
|
||||||
|
import streamlit as st
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------------
|
||||||
|
# Volume and Image Setup
|
||||||
|
# ----------------------------------------------------------------------------
|
||||||
|
metrics_volume = modal.Volume.from_name("evaluation_dashboard_results", create_if_missing=True)
|
||||||
|
|
||||||
|
image = (
|
||||||
|
modal.Image.debian_slim(python_version="3.11")
|
||||||
|
.pip_install("streamlit", "pandas", "plotly")
|
||||||
|
.add_local_file(__file__, "/root/serve_dashboard.py")
|
||||||
|
)
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------------
|
||||||
|
# Define and Deploy the App
|
||||||
|
# ----------------------------------------------------------------------------
|
||||||
|
app = modal.App(
|
||||||
|
name="metrics-dashboard",
|
||||||
|
image=image,
|
||||||
|
volumes={"/data": metrics_volume},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.function()
|
||||||
|
@modal.web_server(port=8000)
|
||||||
|
def run():
|
||||||
|
"""
|
||||||
|
Launch Streamlit server on port 8000 inside the container.
|
||||||
|
"""
|
||||||
|
cmd = (
|
||||||
|
"streamlit run /root/serve_dashboard.py "
|
||||||
|
"--server.port 8000 "
|
||||||
|
"--server.enableCORS=false "
|
||||||
|
"--server.enableXsrfProtection=false"
|
||||||
|
)
|
||||||
|
subprocess.Popen(cmd, shell=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------------
|
||||||
|
# Streamlit Dashboard Application Logic
|
||||||
|
# ----------------------------------------------------------------------------
|
||||||
|
def main():
|
||||||
|
st.set_page_config(page_title="Metrics Dashboard", layout="wide")
|
||||||
|
st.title("📊 Cognee Evaluations Dashboard")
|
||||||
|
|
||||||
|
data_path = "/data"
|
||||||
|
records = []
|
||||||
|
|
||||||
|
for filename in sorted(os.listdir(data_path)):
|
||||||
|
if not filename.endswith(".json"):
|
||||||
|
continue
|
||||||
|
base = filename.rsplit(".", 1)[0]
|
||||||
|
parts = base.split("_")
|
||||||
|
benchmark = parts[1] if len(parts) >= 3 else ""
|
||||||
|
|
||||||
|
full_path = os.path.join(data_path, filename)
|
||||||
|
with open(full_path, "r") as f:
|
||||||
|
items = json.load(f)
|
||||||
|
num_q = len(items)
|
||||||
|
total_em = sum(q["metrics"]["EM"]["score"] for q in items)
|
||||||
|
total_f1 = sum(q["metrics"]["f1"]["score"] for q in items)
|
||||||
|
total_corr = sum(q["metrics"]["correctness"]["score"] for q in items)
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"file": parts[0].upper() + "_____" + parts[2],
|
||||||
|
"benchmark": benchmark,
|
||||||
|
"questions": num_q,
|
||||||
|
"avg_EM": round(total_em / num_q, 4),
|
||||||
|
"avg_F1": round(total_f1 / num_q, 4),
|
||||||
|
"avg_correctness": round(total_corr / num_q, 4),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
df = pd.DataFrame(records)
|
||||||
|
if df.empty:
|
||||||
|
st.warning("No JSON files found in the volume.")
|
||||||
|
return
|
||||||
|
|
||||||
|
st.subheader("Results by benchmark")
|
||||||
|
for bm, group in df.groupby("benchmark"):
|
||||||
|
st.markdown(f"### {bm}")
|
||||||
|
st.dataframe(
|
||||||
|
group[["file", "questions", "avg_EM", "avg_F1", "avg_correctness"]],
|
||||||
|
use_container_width=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Loading…
Add table
Reference in a new issue