508165e883
<!-- .github/pull_request_template.md --> This PR introduces wide vector and graph structure filtering capabilities. With these changes, the graph completion retriever and all retrievers that inherit from it will now filter relevant vector elements and subgraphs based on the query. This improvement significantly increases search speed for large graphs while maintaining—and in some cases slightly improving—accuracy. Changes in This PR: -Introduced new wide_search_top_k parameter: Controls the initial search space size -Added graph adapter level filtering method: Enables relevant subgraph filtering while maintaining backward compatibility. For community or custom graph adapters that don't implement this method, the system gracefully falls back to the original search behavior. -Updated modal dashboard and evaluation framework: Fixed compatibility issues. Added comprehensive unit tests: Introduced unit tests for brute_force_triplet_search (previously untested) and expanded the CogneeGraph test suite. Integration tests: Existing integration tests verify end-to-end search functionality (no changes required). Acceptance Criteria and Testing To verify the new search behavior, run search queries with different wide_search_top_k parameters while logging is enabled: None: Triggers a full graph search (default behavior) 1: Projects a minimal subgraph (demonstrates maximum filtering) Custom values: Test intermediate levels of filtering Internal Testing and results: Performance and accuracy benchmarks are available upon request. The implementation demonstrates measurable improvements in query latency for large graphs without sacrificing result quality. ## Type of Change <!-- Please check the relevant option --> - [ ] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [x] Code refactoring - [x] Performance improvement - [ ] Other (please specify): ## Screenshots/Videos (if applicable) None ## Pre-submission Checklist <!-- Please check all boxes that apply before submitting your PR --> - [x] **I have tested my changes thoroughly before submitting this PR** - [x] **This PR contains minimal changes necessary to address the issue/feature** - [x] My code follows the project's coding standards and style guidelines - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have added necessary documentation (if applicable) - [x] All new and existing tests pass - [x] I have searched existing PRs to ensure this change hasn't been submitted already - [x] I have linked any relevant issues in the description - [x] My commits have clear and descriptive messages ## DCO Affirmation I affirm that all code in every commit of this pull request conforms to the terms of the Topoteretes Developer Certificate of Origin. --------- Co-authored-by: Pavel Zorin <pazonec@yandex.ru>
76 lines
3 KiB
Python
76 lines
3 KiB
Python
from functools import lru_cache
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
from typing import List, Optional
|
|
|
|
|
|
class EvalConfig(BaseSettings):
|
|
# Corpus builder params
|
|
building_corpus_from_scratch: bool = True
|
|
number_of_samples_in_corpus: int = 1
|
|
benchmark: str = "Dummy" # Options: 'HotPotQA', 'Dummy', 'TwoWikiMultiHop'
|
|
task_getter_type: str = (
|
|
"Default" # Options: 'Default', 'CascadeGraph', 'NoSummaries', 'JustChunks'
|
|
)
|
|
|
|
# Question answering params
|
|
answering_questions: bool = True
|
|
qa_engine: str = "cognee_graph_completion" # Options: 'cognee_completion' or 'cognee_graph_completion' or 'cognee_graph_completion_cot' or 'cognee_graph_completion_context_extension'
|
|
|
|
# Evaluation params
|
|
evaluating_answers: bool = True
|
|
evaluating_contexts: bool = True
|
|
evaluation_engine: str = "DeepEval" # Options: 'DeepEval' (uses deepeval_model), 'DirectLLM' (uses default llm from .env)
|
|
evaluation_metrics: List[str] = [
|
|
"correctness",
|
|
"EM",
|
|
"f1",
|
|
] # Use only 'correctness' for DirectLLM
|
|
deepeval_model: str = "gpt-4o-mini"
|
|
|
|
# Metrics params
|
|
calculate_metrics: bool = True
|
|
|
|
# Visualization
|
|
dashboard: bool = True
|
|
|
|
# file paths
|
|
questions_path: str = "questions_output.json"
|
|
answers_path: str = "answers_output.json"
|
|
metrics_path: str = "metrics_output.json"
|
|
aggregate_metrics_path: str = "aggregate_metrics.json"
|
|
dashboard_path: str = "dashboard.html"
|
|
direct_llm_system_prompt: str = "direct_llm_eval_system.txt"
|
|
direct_llm_eval_prompt: str = "direct_llm_eval_prompt.txt"
|
|
instance_filter: Optional[List[str]] = None
|
|
|
|
model_config = SettingsConfigDict(env_file=".env", extra="allow")
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"building_corpus_from_scratch": self.building_corpus_from_scratch,
|
|
"number_of_samples_in_corpus": self.number_of_samples_in_corpus,
|
|
"benchmark": self.benchmark,
|
|
"answering_questions": self.answering_questions,
|
|
"qa_engine": self.qa_engine,
|
|
"evaluating_answers": self.evaluating_answers,
|
|
"evaluating_contexts": self.evaluating_contexts, # Controls whether context evaluation should be performed
|
|
"evaluation_engine": self.evaluation_engine,
|
|
"evaluation_metrics": self.evaluation_metrics,
|
|
"calculate_metrics": self.calculate_metrics,
|
|
"dashboard": self.dashboard,
|
|
"questions_path": self.questions_path,
|
|
"answers_path": self.answers_path,
|
|
"metrics_path": self.metrics_path,
|
|
"aggregate_metrics_path": self.aggregate_metrics_path,
|
|
"dashboard_path": self.dashboard_path,
|
|
"deepeval_model": self.deepeval_model,
|
|
"task_getter_type": self.task_getter_type,
|
|
"direct_llm_system_prompt": self.direct_llm_system_prompt,
|
|
"direct_llm_eval_prompt": self.direct_llm_eval_prompt,
|
|
"instance_filter": self.instance_filter,
|
|
}
|
|
|
|
|
|
@lru_cache
|
|
def get_llm_config():
|
|
return EvalConfig()
|