gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

eval on random samples instead of first couple

Browse source
This commit is contained in:
Rita Aleksziev 2025-01-20 17:47:52 +01:00
parent 1a73779353
commit 015f0084c8
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
evals/eval_on_hotpot.py
View file

@ -10,8 +10,10 @@ from cognee.infrastructure.llm.prompts import read_query_prompt, render_prompt
from evals.qa_dataset_utils import load_qa_dataset
from evals.qa_metrics_utils import get_metrics
from evals.qa_context_provider_utils import qa_context_providers, valid_pipeline_slices
import random
logger = logging.getLogger(__name__)
random.seed(42)
async def answer_qa_instance(instance, context_provider):
@ -77,7 +79,7 @@ async def eval_on_QA_dataset(
dataset = load_qa_dataset(dataset_name_or_filename)
context_provider = qa_context_providers[context_provider_name]
eval_metrics = get_metrics(metric_name_list)
instances = dataset if not num_samples else dataset[:num_samples]
instances = dataset if not num_samples else random.sample(dataset, num_samples)
if "promptfoo_metrics" in eval_metrics:
promptfoo_results = await eval_metrics["promptfoo_metrics"].measure(