6a0c0e3ef8
<!-- .github/pull_request_template.md --> This PR contains the evaluation framework development for cognee ## DCO Affirmation I affirm that all code in every commit of this pull request conforms to the terms of the Topoteretes Developer Certificate of Origin <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Expanded evaluation framework now integrates asynchronous corpus building, question answering, and performance evaluation with adaptive benchmarks for improved metrics (correctness, exact match, and F1 score). - **Infrastructure** - Added database integration for persistent storage of questions, answers, and metrics. - Launched an interactive metrics dashboard featuring advanced visualizations. - Introduced an automated testing workflow for continuous quality assurance. - **Documentation** - Updated guidelines for generating concise, clear answers. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
22 lines
791 B
Python
22 lines
791 B
Python
from typing import Optional, Union, Any, LiteralString
|
|
|
|
from evals.eval_framework.benchmark_adapters.base_benchmark_adapter import BaseBenchmarkAdapter
|
|
|
|
|
|
class DummyAdapter(BaseBenchmarkAdapter):
|
|
def load_corpus(
|
|
self, limit: Optional[int] = None, seed: int = 42
|
|
) -> tuple[list[Union[LiteralString, str]], list[dict[str, str]]]:
|
|
corpus_list = [
|
|
"The cognee is an AI memory engine that supports different vector and graph databases",
|
|
"Neo4j is a graph database supported by cognee",
|
|
]
|
|
question_answer_pairs = [
|
|
{
|
|
"answer": "Yes",
|
|
"question": "Is Neo4j supported by cognee?",
|
|
"type": "dummy",
|
|
}
|
|
]
|
|
|
|
return corpus_list, question_answer_pairs
|