diff --git a/evals/eval_framework/benchmark_adapters/benchmark_adapters.py b/evals/eval_framework/benchmark_adapters/benchmark_adapters.py index 444a47890..f040818e0 100644 --- a/evals/eval_framework/benchmark_adapters/benchmark_adapters.py +++ b/evals/eval_framework/benchmark_adapters/benchmark_adapters.py @@ -10,7 +10,7 @@ from evals.eval_framework.benchmark_adapters.twowikimultihop_adapter import TwoW class BenchmarkAdapter(Enum): DUMMY = ("Dummy", DummyAdapter) HOTPOTQA = ("HotPotQA", HotpotQAAdapter) - MUSIQUE = ('Musique', MusiqueQAAdapter) + MUSIQUE = ("Musique", MusiqueQAAdapter) TWOWIKIMULTIHOP = ("TwoWikiMultiHop", TwoWikiMultihopAdapter) def __new__(cls, adapter_name: str, adapter_class: Type): diff --git a/evals/eval_framework/benchmark_adapters/musique_adapter.py b/evals/eval_framework/benchmark_adapters/musique_adapter.py index 8c1abfdce..982e8e8e8 100644 --- a/evals/eval_framework/benchmark_adapters/musique_adapter.py +++ b/evals/eval_framework/benchmark_adapters/musique_adapter.py @@ -18,10 +18,8 @@ class MusiqueQAAdapter(BaseBenchmarkAdapter): dataset_info = { # Name of the final file we want to load "filename": "musique_ans_v1.0_dev.jsonl", - # A Google Drive URL (or share link) to the ZIP containing this file "download_url": "https://drive.google.com/file/d/1tGdADlNjWFaHLeZZGShh2IRcpO6Lv24h/view?usp=sharing", - # The name of the ZIP archive we expect after downloading "zip_filename": "musique_v1.0.zip", } @@ -69,9 +67,7 @@ class MusiqueQAAdapter(BaseBenchmarkAdapter): for item in data: # Each 'paragraphs' is a list of dicts; we can concatenate their 'paragraph_text' paragraphs = item.get("paragraphs", []) - combined_paragraphs = " ".join( - paragraph["paragraph_text"] for paragraph in paragraphs - ) + combined_paragraphs = " ".join(paragraph["paragraph_text"] for paragraph in paragraphs) corpus_list.append(combined_paragraphs) # Example question & answer diff --git a/evals/eval_framework/corpus_builder/corpus_builder_executor.py b/evals/eval_framework/corpus_builder/corpus_builder_executor.py index 49abc9489..0f5da89ca 100644 --- a/evals/eval_framework/corpus_builder/corpus_builder_executor.py +++ b/evals/eval_framework/corpus_builder/corpus_builder_executor.py @@ -3,11 +3,15 @@ import logging from typing import Optional, Tuple, List, Dict, Union, Any from evals.eval_framework.benchmark_adapters.benchmark_adapters import BenchmarkAdapter +from evals.eval_framework.corpus_builder.task_getters.task_getters import TaskGetters +from evals.eval_framework.corpus_builder.task_getters.base_task_getter import BaseTaskGetter from cognee.shared.utils import setup_logging class CorpusBuilderExecutor: - def __init__(self, benchmark: Union[str, Any] = "Dummy") -> None: + def __init__( + self, benchmark: Union[str, Any] = "Dummy", task_getter_type: str = "DEFAULT" + ) -> None: if isinstance(benchmark, str): try: adapter_enum = BenchmarkAdapter(benchmark) @@ -20,6 +24,13 @@ class CorpusBuilderExecutor: self.raw_corpus = None self.questions = None + try: + task_enum = TaskGetters(task_getter_type) + except KeyError: + raise ValueError(f"Invalid task getter type: {task_getter_type}") + + self.task_getter: BaseTaskGetter = task_enum.getter_class() + def load_corpus(self, limit: Optional[int] = None) -> Tuple[List[Dict], List[str]]: self.raw_corpus, self.questions = self.adapter.load_corpus(limit=limit) return self.raw_corpus, self.questions @@ -32,12 +43,10 @@ class CorpusBuilderExecutor: async def run_cognee(self) -> None: setup_logging(logging.ERROR) - # Pruning system and databases. await cognee.prune.prune_data() await cognee.prune.prune_system(metadata=True) - # Adding corpus elements to the cognee metastore. await cognee.add(self.raw_corpus) - # Running cognify to build the knowledge graph. - await cognee.cognify() + tasks = await self.task_getter.get_tasks() + await cognee.cognify(tasks=tasks) diff --git a/evals/eval_framework/corpus_builder/run_corpus_builder.py b/evals/eval_framework/corpus_builder/run_corpus_builder.py index 01ce58537..599cc5c4b 100644 --- a/evals/eval_framework/corpus_builder/run_corpus_builder.py +++ b/evals/eval_framework/corpus_builder/run_corpus_builder.py @@ -30,7 +30,10 @@ async def create_and_insert_questions_table(questions_payload): async def run_corpus_builder(params: dict) -> None: if params.get("building_corpus_from_scratch"): logging.info("Corpus Builder started...") - corpus_builder = CorpusBuilderExecutor(benchmark=params["benchmark"]) + corpus_builder = CorpusBuilderExecutor( + benchmark=params["benchmark"], + task_getter_type=params.get("task_getter_type", "Default"), + ) questions = await corpus_builder.build_corpus( limit=params.get("number_of_samples_in_corpus") ) diff --git a/evals/eval_framework/corpus_builder/task_getters/__init__.py b/evals/eval_framework/corpus_builder/task_getters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/evals/eval_framework/corpus_builder/task_getters/base_task_getter.py b/evals/eval_framework/corpus_builder/task_getters/base_task_getter.py new file mode 100644 index 000000000..42ad80edf --- /dev/null +++ b/evals/eval_framework/corpus_builder/task_getters/base_task_getter.py @@ -0,0 +1,12 @@ +from abc import ABC, abstractmethod +from typing import List +from cognee.modules.pipelines.tasks.Task import Task + + +class BaseTaskGetter(ABC): + """Abstract base class for asynchronous task retrieval implementations.""" + + @abstractmethod + async def get_tasks(self) -> List[Task]: + """Asynchronously retrieve a list of tasks. Must be implemented by subclasses.""" + pass diff --git a/evals/eval_framework/corpus_builder/task_getters/default_task_getter.py b/evals/eval_framework/corpus_builder/task_getters/default_task_getter.py new file mode 100644 index 000000000..9bbb45d41 --- /dev/null +++ b/evals/eval_framework/corpus_builder/task_getters/default_task_getter.py @@ -0,0 +1,12 @@ +from cognee.api.v1.cognify.cognify_v2 import get_default_tasks +from typing import List +from evals.eval_framework.corpus_builder.task_getters.base_task_getter import BaseTaskGetter +from cognee.modules.pipelines.tasks.Task import Task + + +class DefaultTaskGetter(BaseTaskGetter): + """Default task getter that retrieves tasks using the standard get_default_tasks function.""" + + async def get_tasks(self) -> List[Task]: + """Retrieve default tasks asynchronously.""" + return await get_default_tasks() diff --git a/evals/eval_framework/corpus_builder/task_getters/task_getters.py b/evals/eval_framework/corpus_builder/task_getters/task_getters.py new file mode 100644 index 000000000..4ba35f5d4 --- /dev/null +++ b/evals/eval_framework/corpus_builder/task_getters/task_getters.py @@ -0,0 +1,19 @@ +from enum import Enum +from typing import Type +from evals.eval_framework.corpus_builder.task_getters.default_task_getter import DefaultTaskGetter + + +class TaskGetters(Enum): + """Enum mapping task getter types to their respective classes.""" + + DEFAULT = ("Default", DefaultTaskGetter) + # CUSTOM = ("Custom", CustomTaskGetter) + + def __new__(cls, getter_name: str, getter_class: Type): + obj = object.__new__(cls) + obj._value_ = getter_name + obj.getter_class = getter_class + return obj + + def __str__(self): + return self.value diff --git a/evals/eval_framework/eval_config.py b/evals/eval_framework/eval_config.py index 5651a3e47..e0a4c93c8 100644 --- a/evals/eval_framework/eval_config.py +++ b/evals/eval_framework/eval_config.py @@ -8,6 +8,7 @@ class EvalConfig(BaseSettings): building_corpus_from_scratch: bool = True number_of_samples_in_corpus: int = 1 benchmark: str = "Dummy" # Options: 'HotPotQA', 'Dummy', 'TwoWikiMultiHop' + task_getter_type: str = "Default" # Question answering params answering_questions: bool = True @@ -48,6 +49,7 @@ class EvalConfig(BaseSettings): "metrics_path": self.metrics_path, "dashboard_path": self.dashboard_path, "deepeval_model": self.deepeval_model, + "task_getter_type": self.task_getter_type, } diff --git a/poetry.lock b/poetry.lock index 0c6d89eaa..0d3c5a724 100644 --- a/poetry.lock +++ b/poetry.lock @@ -550,7 +550,7 @@ typecheck = ["mypy"] name = "beautifulsoup4" version = "4.13.3" description = "Screen-scraping library" -optional = true +optional = false python-versions = ">=3.7.0" files = [ {file = "beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16"}, @@ -1067,7 +1067,6 @@ files = [ {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb"}, {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b"}, {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543"}, - {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:60eb32934076fa07e4316b7b2742fa52cbb190b42c2df2863dbc4230a0a9b385"}, {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e"}, {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e"}, {file = "cryptography-44.0.0-cp37-abi3-win32.whl", hash = "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053"}, @@ -1078,7 +1077,6 @@ files = [ {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289"}, {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7"}, {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c"}, - {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9abcc2e083cbe8dde89124a47e5e53ec38751f0d7dfd36801008f316a127d7ba"}, {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64"}, {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285"}, {file = "cryptography-44.0.0-cp39-abi3-win32.whl", hash = "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417"}, @@ -1945,6 +1943,26 @@ test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe, test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] tqdm = ["tqdm"] +[[package]] +name = "gdown" +version = "5.2.0" +description = "Google Drive Public File/Folder Downloader" +optional = false +python-versions = ">=3.8" +files = [ + {file = "gdown-5.2.0-py3-none-any.whl", hash = "sha256:33083832d82b1101bdd0e9df3edd0fbc0e1c5f14c9d8c38d2a35bf1683b526d6"}, + {file = "gdown-5.2.0.tar.gz", hash = "sha256:2145165062d85520a3cd98b356c9ed522c5e7984d408535409fd46f94defc787"}, +] + +[package.dependencies] +beautifulsoup4 = "*" +filelock = "*" +requests = {version = "*", extras = ["socks"]} +tqdm = "*" + +[package.extras] +test = ["build", "mypy", "pytest", "pytest-xdist", "ruff", "twine", "types-requests", "types-setuptools"] + [[package]] name = "ghp-import" version = "2.1.0" @@ -5204,8 +5222,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -6193,8 +6211,8 @@ astroid = ">=3.3.8,<=3.4.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.2", markers = "python_version < \"3.11\""}, - {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, + {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, ] isort = ">=4.2.5,<5.13.0 || >5.13.0,<7" mccabe = ">=0.6,<0.8" @@ -6295,6 +6313,18 @@ docs = ["myst_parser", "sphinx", "sphinx_rtd_theme"] full = ["Pillow (>=8.0.0)", "PyCryptodome", "cryptography"] image = ["Pillow (>=8.0.0)"] +[[package]] +name = "pysocks" +version = "1.7.1" +description = "A Python SOCKS client module. See https://github.com/Anorov/PySocks for more information." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "PySocks-1.7.1-py27-none-any.whl", hash = "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299"}, + {file = "PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5"}, + {file = "PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"}, +] + [[package]] name = "pytest" version = "7.4.4" @@ -7025,6 +7055,7 @@ files = [ certifi = ">=2017.4.17" charset-normalizer = ">=2,<4" idna = ">=2.5,<4" +PySocks = {version = ">=1.5.6,<1.5.7 || >1.5.7", optional = true, markers = "extra == \"socks\""} urllib3 = ">=1.21.1,<3" [package.extras] @@ -7786,7 +7817,7 @@ files = [ name = "soupsieve" version = "2.6" description = "A modern CSS selector implementation for Beautiful Soup." -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, @@ -9185,4 +9216,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.10.0,<3.13" -content-hash = "a213bb86ac98ac7b327e40eafae1ef038ff74071dea144b8c005eb86b252612f" +content-hash = "fa5e6192627dc994e4d4b8c3471f836326afbe045b44407bc5e29ec11bff5f08" diff --git a/pyproject.toml b/pyproject.toml index 7d41c8daa..a51c3f53a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ parso = {version = "^0.8.4", optional = true} jedi = {version = "^0.19.2", optional = true} plotly = "^6.0.0" mistral-common = {version = "^1.5.2", optional = true} +gdown = "^5.2.0" [tool.poetry.extras]