Merge branch 'dev' of github.com:topoteretes/cognee into dev

.github/workflows/test_memgraph.yml

@@ -0,0 +1,57 @@
+name: test | memgraph
+
+# on:
+#  workflow_dispatch:
+#  pull_request:
+#    types: [labeled, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUNTIME__LOG_LEVEL: ERROR
+
+jobs:
+  run_memgraph_integration_test:
+    name: test
+    runs-on: ubuntu-22.04
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@master
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10.x'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1.4.1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Install dependencies
+        run: poetry install -E memgraph --no-interaction
+
+      - name: Run default Memgraph
+        env:
+          ENV: 'dev'
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
+          LLM_ENDPOINT: ${{ secrets.LLM_ENDPOINT }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_API_VERSION: ${{ secrets.LLM_API_VERSION }}
+          EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }}
+          EMBEDDING_ENDPOINT: ${{ secrets.EMBEDDING_ENDPOINT }}
+          EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
+          EMBEDDING_API_VERSION: ${{ secrets.EMBEDDING_API_VERSION }}
+          GRAPH_DATABASE_URL: ${{ secrets.MEMGRAPH_API_URL }}
+          GRAPH_DATABASE_PASSWORD: ${{ secrets.MEMGRAPH_API_KEY }}
+          GRAPH_DATABASE_USERNAME: " "
+        run: poetry run python ./cognee/tests/test_memgraph.py

cognee-mcp/pyproject.toml

@@ -1,12 +1,14 @@
 [project]
 name = "cognee-mcp"
-version = "0.2.3"
+version = "0.3.0"
 description = "A MCP server project"
 readme = "README.md"
 requires-python = ">=3.10"
 
 dependencies = [
-    "cognee[postgres,codegraph,gemini,huggingface]==0.1.39",
+    # For local cognee repo usage remove comment bellow and add absolute path to cognee
+    #"cognee[postgres,codegraph,gemini,huggingface] @ file:/Users/<username>/Desktop/cognee",
+    "cognee[postgres,codegraph,gemini,huggingface]==0.1.40",
     "fastmcp>=1.0",
     "mcp==1.5.0",
     "uv>=0.6.3",
@@ -28,5 +30,8 @@ dev = [
     "debugpy>=1.8.12",
 ]
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [project.scripts]
 cognee = "src:main"

cognee-mcp/src/server.py

@@ -9,6 +9,9 @@ import importlib.util
 from contextlib import redirect_stdout
 import mcp.types as types
 from mcp.server import FastMCP
+from cognee.modules.pipelines.operations.get_pipeline_status import get_pipeline_status
+from cognee.modules.data.methods.get_unique_dataset_id import get_unique_dataset_id
+from cognee.modules.users.methods import get_default_user
 from cognee.api.v1.cognify.code_graph_pipeline import run_code_graph_pipeline
 from cognee.modules.search.types import SearchType
 from cognee.shared.data_models import KnowledgeGraph
@@ -28,7 +31,6 @@ async def cognify(text: str, graph_model_file: str = None, graph_model_name: str
         """Build knowledge graph from the input text"""
         # NOTE: MCP uses stdout to communicate, we must redirect all output
         #       going to stdout ( like the print function ) to stderr.
-        #       As cognify is an async background job the output had to be redirected again.
         with redirect_stdout(sys.stderr):
             logger.info("Cognify process starting.")
             if graph_model_file and graph_model_name:
@@ -55,8 +57,8 @@ async def cognify(text: str, graph_model_file: str = None, graph_model_name: str
 
     text = (
         f"Background process launched due to MCP timeout limitations.\n"
-        f"Average completion time is around 4 minutes.\n"
-        f"For current cognify status you can check the log file at: {log_file}"
+        f"To check current cognify status use the cognify_status tool\n"
+        f"or check the log file at: {log_file}"
     )
 
     return [
@@ -72,7 +74,6 @@ async def codify(repo_path: str) -> list:
     async def codify_task(repo_path: str):
         # NOTE: MCP uses stdout to communicate, we must redirect all output
         #       going to stdout ( like the print function ) to stderr.
-        #       As codify is an async background job the output had to be redirected again.
         with redirect_stdout(sys.stderr):
             logger.info("Codify process starting.")
             results = []
@@ -88,8 +89,8 @@ async def codify(repo_path: str) -> list:
 
     text = (
         f"Background process launched due to MCP timeout limitations.\n"
-        f"Average completion time is around 4 minutes.\n"
-        f"For current codify status you can check the log file at: {log_file}"
+        f"To check current codify status use the codify_status tool\n"
+        f"or you can check the log file at: {log_file}"
     )
 
     return [
@@ -138,6 +139,24 @@ async def prune():
         return [types.TextContent(type="text", text="Pruned")]
 
 
+@mcp.tool()
+async def cognify_status():
+    """Get status of cognify pipeline"""
+    with redirect_stdout(sys.stderr):
+        user = await get_default_user()
+        status = await get_pipeline_status([await get_unique_dataset_id("main_dataset", user)])
+        return [types.TextContent(type="text", text=str(status))]
+
+
+@mcp.tool()
+async def codify_status():
+    """Get status of codify pipeline"""
+    with redirect_stdout(sys.stderr):
+        user = await get_default_user()
+        status = await get_pipeline_status([await get_unique_dataset_id("codebase", user)])
+        return [types.TextContent(type="text", text=str(status))]
+
+
 def node_to_string(node):
     node_data = ", ".join(
         [f'{key}: "{value}"' for key, value in node.items() if key in ["id", "name"]]

cognee/api/v1/cognify/code_graph_pipeline.py

@@ -13,6 +13,7 @@ from cognee.modules.pipelines.tasks.task import Task
 from cognee.modules.users.methods import get_default_user
 from cognee.shared.data_models import KnowledgeGraph
 from cognee.tasks.documents import classify_documents, extract_chunks_from_documents
+from cognee.modules.data.methods.get_unique_dataset_id import get_unique_dataset_id
 from cognee.tasks.graph import extract_graph_from_data
 from cognee.tasks.ingestion import ingest_data
 from cognee.tasks.repo_processor import get_non_py_files, get_repo_file_dependencies
@@ -64,7 +65,7 @@ async def run_code_graph_pipeline(repo_path, include_docs=False):
             ),
         ]
 
-    dataset_id = uuid5(NAMESPACE_OID, "codebase")
+    dataset_id = await get_unique_dataset_id("codebase", user)
 
     if include_docs:
         non_code_pipeline_run = run_tasks(

cognee/infrastructure/databases/graph/get_graph_engine.py

@@ -67,6 +67,18 @@ def create_graph_engine(
 
         return KuzuAdapter(db_path=graph_file_path)
 
+    elif graph_database_provider == "memgraph":
+        if not (graph_database_url and graph_database_username and graph_database_password):
+            raise EnvironmentError("Missing required Memgraph credentials.")
+
+        from .memgraph.memgraph_adapter import MemgraphAdapter
+
+        return MemgraphAdapter(
+            graph_database_url=graph_database_url,
+            graph_database_username=graph_database_username,
+            graph_database_password=graph_database_password,
+        )
+
     from .networkx.adapter import NetworkXAdapter
 
     graph_client = NetworkXAdapter(filename=graph_file_path)

cognee/infrastructure/databases/graph/memgraph/memgraph_adapter.py

@@ -0,0 +1,690 @@
+"""Memgraph Adapter for Graph Database"""
+
+import json
+from cognee.shared.logging_utils import get_logger, ERROR
+import asyncio
+from textwrap import dedent
+from typing import Optional, Any, List, Dict
+from contextlib import asynccontextmanager
+from uuid import UUID
+from neo4j import AsyncSession
+from neo4j import AsyncGraphDatabase
+from neo4j.exceptions import Neo4jError
+from cognee.infrastructure.engine import DataPoint
+from cognee.infrastructure.databases.graph.graph_db_interface import GraphDBInterface
+from cognee.modules.storage.utils import JSONEncoder
+
+logger = get_logger("MemgraphAdapter", level=ERROR)
+
+
+class MemgraphAdapter(GraphDBInterface):
+    def __init__(
+        self,
+        graph_database_url: str,
+        graph_database_username: str,
+        graph_database_password: str,
+        driver: Optional[Any] = None,
+    ):
+        self.driver = driver or AsyncGraphDatabase.driver(
+            graph_database_url,
+            auth=(graph_database_username, graph_database_password),
+            max_connection_lifetime=120,
+        )
+
+    @asynccontextmanager
+    async def get_session(self) -> AsyncSession:
+        async with self.driver.session() as session:
+            yield session
+
+    async def query(
+        self,
+        query: str,
+        params: Optional[Dict[str, Any]] = None,
+    ) -> List[Dict[str, Any]]:
+        try:
+            async with self.get_session() as session:
+                result = await session.run(query, params)
+                data = await result.data()
+                return data
+        except Neo4jError as error:
+            logger.error("Memgraph query error: %s", error, exc_info=True)
+            raise error
+
+    async def has_node(self, node_id: str) -> bool:
+        results = await self.query(
+            """
+                MATCH (n)
+                WHERE n.id = $node_id
+                RETURN COUNT(n) > 0 AS node_exists
+            """,
+            {"node_id": node_id},
+        )
+        return results[0]["node_exists"] if len(results) > 0 else False
+
+    async def add_node(self, node: DataPoint):
+        serialized_properties = self.serialize_properties(node.model_dump())
+
+        query = """
+        MERGE (node {id: $node_id})
+        ON CREATE SET node:$node_label, node += $properties, node.updated_at = timestamp()
+        ON MATCH SET node:$node_label, node += $properties, node.updated_at = timestamp()
+        RETURN ID(node) AS internal_id,node.id AS nodeId
+        """
+
+        params = {
+            "node_id": str(node.id),
+            "node_label": type(node).__name__,
+            "properties": serialized_properties,
+        }
+        return await self.query(query, params)
+
+    async def add_nodes(self, nodes: list[DataPoint]) -> None:
+        query = """
+        UNWIND $nodes AS node
+        MERGE (n {id: node.node_id})
+        ON CREATE SET n:node.label, n += node.properties, n.updated_at = timestamp()
+        ON MATCH SET n:node.label, n += node.properties, n.updated_at = timestamp()
+        RETURN ID(n) AS internal_id, n.id AS nodeId
+        """
+
+        nodes = [
+            {
+                "node_id": str(node.id),
+                "label": type(node).__name__,
+                "properties": self.serialize_properties(node.model_dump()),
+            }
+            for node in nodes
+        ]
+
+        results = await self.query(query, dict(nodes=nodes))
+        return results
+
+    async def extract_node(self, node_id: str):
+        results = await self.extract_nodes([node_id])
+
+        return results[0] if len(results) > 0 else None
+
+    async def extract_nodes(self, node_ids: List[str]):
+        query = """
+        UNWIND $node_ids AS id
+        MATCH (node {id: id})
+        RETURN node"""
+
+        params = {"node_ids": node_ids}
+
+        results = await self.query(query, params)
+
+        return [result["node"] for result in results]
+
+    async def delete_node(self, node_id: str):
+        sanitized_id = node_id.replace(":", "_")
+
+        query = "MATCH (node: {{id: $node_id}}) DETACH DELETE node"
+        params = {"node_id": sanitized_id}
+
+        return await self.query(query, params)
+
+    async def delete_nodes(self, node_ids: list[str]) -> None:
+        query = """
+        UNWIND $node_ids AS id
+        MATCH (node {id: id})
+        DETACH DELETE node"""
+
+        params = {"node_ids": node_ids}
+
+        return await self.query(query, params)
+
+    async def has_edge(self, from_node: UUID, to_node: UUID, edge_label: str) -> bool:
+        query = """
+            MATCH (from_node)-[relationship]->(to_node)
+            WHERE from_node.id = $from_node_id AND to_node.id = $to_node_id AND type(relationship) = $edge_label
+            RETURN COUNT(relationship) > 0 AS edge_exists
+        """
+
+        params = {
+            "from_node_id": str(from_node),
+            "to_node_id": str(to_node),
+            "edge_label": edge_label,
+        }
+
+        records = await self.query(query, params)
+        return records[0]["edge_exists"] if records else False
+
+    async def has_edges(self, edges):
+        query = """
+            UNWIND $edges AS edge
+            MATCH (a)-[r]->(b)
+            WHERE id(a) = edge.from_node AND id(b) = edge.to_node AND type(r) = edge.relationship_name
+            RETURN edge.from_node AS from_node, edge.to_node AS to_node, edge.relationship_name AS relationship_name, count(r) > 0 AS edge_exists
+        """
+
+        try:
+            params = {
+                "edges": [
+                    {
+                        "from_node": str(edge[0]),
+                        "to_node": str(edge[1]),
+                        "relationship_name": edge[2],
+                    }
+                    for edge in edges
+                ],
+            }
+
+            results = await self.query(query, params)
+            return [result["edge_exists"] for result in results]
+        except Neo4jError as error:
+            logger.error("Memgraph query error: %s", error, exc_info=True)
+            raise error
+
+    async def add_edge(
+        self,
+        from_node: UUID,
+        to_node: UUID,
+        relationship_name: str,
+        edge_properties: Optional[Dict[str, Any]] = None,
+    ):
+        serialized_properties = self.serialize_properties(edge_properties or {})
+
+        query = dedent(
+            f"""\
+            MATCH (from_node {{id: $from_node}}),
+                  (to_node {{id: $to_node}})
+            MERGE (from_node)-[r:{relationship_name}]->(to_node)
+            ON CREATE SET r += $properties, r.updated_at = timestamp()
+            ON MATCH SET r += $properties, r.updated_at = timestamp()
+            RETURN r
+            """
+        )
+
+        params = {
+            "from_node": str(from_node),
+            "to_node": str(to_node),
+            "relationship_name": relationship_name,
+            "properties": serialized_properties,
+        }
+
+        return await self.query(query, params)
+
+    async def add_edges(self, edges: list[tuple[str, str, str, dict[str, Any]]]) -> None:
+        query = """
+            UNWIND $edges AS edge
+            MATCH (from_node {id: edge.from_node})
+            MATCH (to_node {id: edge.to_node})
+            CALL merge.relationship(
+                from_node,
+                edge.relationship_name,
+                {
+                    source_node_id: edge.from_node,
+                    target_node_id: edge.to_node
+                },
+                edge.properties,
+                to_node,
+                {}
+            ) YIELD rel
+            RETURN rel"""
+
+        edges = [
+            {
+                "from_node": str(edge[0]),
+                "to_node": str(edge[1]),
+                "relationship_name": edge[2],
+                "properties": {
+                    **(edge[3] if edge[3] else {}),
+                    "source_node_id": str(edge[0]),
+                    "target_node_id": str(edge[1]),
+                },
+            }
+            for edge in edges
+        ]
+
+        try:
+            results = await self.query(query, dict(edges=edges))
+            return results
+        except Neo4jError as error:
+            logger.error("Memgraph query error: %s", error, exc_info=True)
+            raise error
+
+    async def get_edges(self, node_id: str):
+        query = """
+        MATCH (n {id: $node_id})-[r]-(m)
+        RETURN n, r, m
+        """
+
+        results = await self.query(query, dict(node_id=node_id))
+
+        return [
+            (result["n"]["id"], result["m"]["id"], {"relationship_name": result["r"][1]})
+            for result in results
+        ]
+
+    async def get_disconnected_nodes(self) -> list[str]:
+        query = """
+        // Step 1: Collect all nodes
+        MATCH (n)
+        WITH COLLECT(n) AS nodes
+
+        // Step 2: Find all connected components
+        WITH nodes
+        CALL {
+          WITH nodes
+          UNWIND nodes AS startNode
+          MATCH path = (startNode)-[*]-(connectedNode)
+          WITH startNode, COLLECT(DISTINCT connectedNode) AS component
+          RETURN component
+        }
+
+        // Step 3: Aggregate components
+        WITH COLLECT(component) AS components
+
+        // Step 4: Identify the largest connected component
+        UNWIND components AS component
+        WITH component
+        ORDER BY SIZE(component) DESC
+        LIMIT 1
+        WITH component AS largestComponent
+
+        // Step 5: Find nodes not in the largest connected component
+        MATCH (n)
+        WHERE NOT n IN largestComponent
+        RETURN COLLECT(ID(n)) AS ids
+        """
+
+        results = await self.query(query)
+        return results[0]["ids"] if len(results) > 0 else []
+
+    async def get_predecessors(self, node_id: str, edge_label: str = None) -> list[str]:
+        if edge_label is not None:
+            query = """
+            MATCH (node)<-[r]-(predecessor)
+            WHERE node.id = $node_id AND type(r) = $edge_label
+            RETURN predecessor
+            """
+
+            results = await self.query(
+                query,
+                dict(
+                    node_id=node_id,
+                    edge_label=edge_label,
+                ),
+            )
+
+            return [result["predecessor"] for result in results]
+        else:
+            query = """
+            MATCH (node)<-[r]-(predecessor)
+            WHERE node.id = $node_id
+            RETURN predecessor
+            """
+
+            results = await self.query(
+                query,
+                dict(
+                    node_id=node_id,
+                ),
+            )
+
+            return [result["predecessor"] for result in results]
+
+    async def get_successors(self, node_id: str, edge_label: str = None) -> list[str]:
+        if edge_label is not None:
+            query = """
+            MATCH (node)-[r]->(successor)
+            WHERE node.id = $node_id AND type(r) = $edge_label
+            RETURN successor
+            """
+
+            results = await self.query(
+                query,
+                dict(
+                    node_id=node_id,
+                    edge_label=edge_label,
+                ),
+            )
+
+            return [result["successor"] for result in results]
+        else:
+            query = """
+            MATCH (node)-[r]->(successor)
+            WHERE node.id = $node_id
+            RETURN successor
+            """
+
+            results = await self.query(
+                query,
+                dict(
+                    node_id=node_id,
+                ),
+            )
+
+            return [result["successor"] for result in results]
+
+    async def get_neighbours(self, node_id: str) -> List[Dict[str, Any]]:
+        predecessors, successors = await asyncio.gather(
+            self.get_predecessors(node_id), self.get_successors(node_id)
+        )
+
+        return predecessors + successors
+
+    async def get_connections(self, node_id: UUID) -> list:
+        predecessors_query = """
+        MATCH (node)<-[relation]-(neighbour)
+        WHERE node.id = $node_id
+        RETURN neighbour, relation, node
+        """
+        successors_query = """
+        MATCH (node)-[relation]->(neighbour)
+        WHERE node.id = $node_id
+        RETURN node, relation, neighbour
+        """
+
+        predecessors, successors = await asyncio.gather(
+            self.query(predecessors_query, dict(node_id=str(node_id))),
+            self.query(successors_query, dict(node_id=str(node_id))),
+        )
+
+        connections = []
+
+        for neighbour in predecessors:
+            neighbour = neighbour["relation"]
+            connections.append((neighbour[0], {"relationship_name": neighbour[1]}, neighbour[2]))
+
+        for neighbour in successors:
+            neighbour = neighbour["relation"]
+            connections.append((neighbour[0], {"relationship_name": neighbour[1]}, neighbour[2]))
+
+        return connections
+
+    async def remove_connection_to_predecessors_of(
+        self, node_ids: list[str], edge_label: str
+    ) -> None:
+        query = f"""
+        UNWIND $node_ids AS nid
+        MATCH (node {id: nid})-[r]->(predecessor)
+        WHERE type(r) = $edge_label
+        DELETE r;
+        """
+
+        params = {"node_ids": node_ids, "edge_label": edge_label}
+
+        return await self.query(query, params)
+
+    async def remove_connection_to_successors_of(
+        self, node_ids: list[str], edge_label: str
+    ) -> None:
+        query = f"""
+        UNWIND $node_ids AS id
+        MATCH (node:`{id}`)<-[r:{edge_label}]-(successor)
+        DELETE r;
+        """
+
+        params = {"node_ids": node_ids}
+
+        return await self.query(query, params)
+
+    async def delete_graph(self):
+        query = """MATCH (node)
+                DETACH DELETE node;"""
+
+        return await self.query(query)
+
+    def serialize_properties(self, properties=dict()):
+        serialized_properties = {}
+
+        for property_key, property_value in properties.items():
+            if isinstance(property_value, UUID):
+                serialized_properties[property_key] = str(property_value)
+                continue
+
+            if isinstance(property_value, dict):
+                serialized_properties[property_key] = json.dumps(property_value, cls=JSONEncoder)
+                continue
+
+            serialized_properties[property_key] = property_value
+
+        return serialized_properties
+
+    async def get_model_independent_graph_data(self):
+        query_nodes = "MATCH (n) RETURN collect(n) AS nodes"
+        nodes = await self.query(query_nodes)
+
+        query_edges = "MATCH (n)-[r]->(m) RETURN collect([n, r, m]) AS elements"
+        edges = await self.query(query_edges)
+
+        return (nodes, edges)
+
+    async def get_graph_data(self):
+        query = "MATCH (n) RETURN ID(n) AS id, labels(n) AS labels, properties(n) AS properties"
+
+        result = await self.query(query)
+
+        nodes = [
+            (
+                record["id"],
+                record["properties"],
+            )
+            for record in result
+        ]
+
+        query = """
+        MATCH (n)-[r]->(m)
+        RETURN ID(n) AS source, ID(m) AS target, TYPE(r) AS type, properties(r) AS properties
+        """
+        result = await self.query(query)
+        edges = [
+            (
+                record["properties"]["source_node_id"],
+                record["properties"]["target_node_id"],
+                record["type"],
+                record["properties"],
+            )
+            for record in result
+        ]
+
+        return (nodes, edges)
+
+    async def get_filtered_graph_data(self, attribute_filters):
+        """
+        Fetches nodes and relationships filtered by specified attribute values.
+
+        Args:
+            attribute_filters (list of dict): A list of dictionaries where keys are attributes and values are lists of values to filter on.
+                                              Example: [{"community": ["1", "2"]}]
+
+        Returns:
+            tuple: A tuple containing two lists: nodes and edges.
+        """
+        where_clauses = []
+        for attribute, values in attribute_filters[0].items():
+            values_str = ", ".join(
+                f"'{value}'" if isinstance(value, str) else str(value) for value in values
+            )
+            where_clauses.append(f"n.{attribute} IN [{values_str}]")
+
+        where_clause = " AND ".join(where_clauses)
+
+        query_nodes = f"""
+        MATCH (n)
+        WHERE {where_clause}
+        RETURN ID(n) AS id, labels(n) AS labels, properties(n) AS properties
+        """
+        result_nodes = await self.query(query_nodes)
+
+        nodes = [
+            (
+                record["id"],
+                record["properties"],
+            )
+            for record in result_nodes
+        ]
+
+        query_edges = f"""
+        MATCH (n)-[r]->(m)
+        WHERE {where_clause} AND {where_clause.replace("n.", "m.")}
+        RETURN ID(n) AS source, ID(m) AS target, TYPE(r) AS type, properties(r) AS properties
+        """
+        result_edges = await self.query(query_edges)
+
+        edges = [
+            (
+                record["source"],
+                record["target"],
+                record["type"],
+                record["properties"],
+            )
+            for record in result_edges
+        ]
+
+        return (nodes, edges)
+
+    async def get_node_labels_string(self):
+        node_labels_query = """
+        MATCH (n)
+        WITH DISTINCT labels(n) AS labelList
+        UNWIND labelList AS label
+        RETURN collect(DISTINCT label) AS labels;
+        """
+        node_labels_result = await self.query(node_labels_query)
+        node_labels = node_labels_result[0]["labels"] if node_labels_result else []
+
+        if not node_labels:
+            raise ValueError("No node labels found in the database")
+
+        node_labels_str = "[" + ", ".join(f"'{label}'" for label in node_labels) + "]"
+        return node_labels_str
+
+    async def get_relationship_labels_string(self):
+        relationship_types_query = (
+            "MATCH ()-[r]->() RETURN collect(DISTINCT type(r)) AS relationships;"
+        )
+        relationship_types_result = await self.query(relationship_types_query)
+        relationship_types = (
+            relationship_types_result[0]["relationships"] if relationship_types_result else []
+        )
+
+        if not relationship_types:
+            raise ValueError("No relationship types found in the database.")
+
+        relationship_types_undirected_str = (
+            "{"
+            + ", ".join(f"{rel}" + ": {orientation: 'UNDIRECTED'}" for rel in relationship_types)
+            + "}"
+        )
+        return relationship_types_undirected_str
+
+    async def get_graph_metrics(self, include_optional=False):
+        """For the definition of these metrics, please refer to
+        https://docs.cognee.ai/core_concepts/graph_generation/descriptive_metrics"""
+
+        try:
+            # Basic metrics
+            node_count = await self.query("MATCH (n) RETURN count(n)")
+            edge_count = await self.query("MATCH ()-[r]->() RETURN count(r)")
+            num_nodes = node_count[0][0] if node_count else 0
+            num_edges = edge_count[0][0] if edge_count else 0
+
+            # Calculate mandatory metrics
+            mandatory_metrics = {
+                "num_nodes": num_nodes,
+                "num_edges": num_edges,
+                "mean_degree": (2 * num_edges) / num_nodes if num_nodes > 0 else 0,
+                "edge_density": (num_edges) / (num_nodes * (num_nodes - 1)) if num_nodes > 1 else 0,
+            }
+
+            # Calculate connected components
+            components_query = """
+            MATCH (n:Node)
+            WITH n.id AS node_id
+            MATCH path = (n)-[:EDGE*0..]-()
+            WITH COLLECT(DISTINCT node_id) AS component
+            RETURN COLLECT(component) AS components
+            """
+            components_result = await self.query(components_query)
+            component_sizes = (
+                [len(comp) for comp in components_result[0][0]] if components_result else []
+            )
+
+            mandatory_metrics.update(
+                {
+                    "num_connected_components": len(component_sizes),
+                    "sizes_of_connected_components": component_sizes,
+                }
+            )
+
+            if include_optional:
+                # Self-loops
+                self_loops_query = """
+                MATCH (n:Node)-[r:EDGE]->(n)
+                RETURN COUNT(r)
+                """
+                self_loops = await self.query(self_loops_query)
+                num_selfloops = self_loops[0][0] if self_loops else 0
+
+                # Shortest paths (simplified for Kuzu)
+                paths_query = """
+                MATCH (n:Node), (m:Node)
+                WHERE n.id < m.id
+                MATCH path = (n)-[:EDGE*]-(m)
+                RETURN MIN(LENGTH(path)) AS length
+                """
+                paths = await self.query(paths_query)
+                path_lengths = [p[0] for p in paths if p[0] is not None]
+
+                # Local clustering coefficient
+                clustering_query = """
+                /// Step 1: Get each node with its neighbors and degree
+                MATCH (n:Node)-[:EDGE]-(neighbor)
+                WITH n, COLLECT(DISTINCT neighbor) AS neighbors, COUNT(DISTINCT neighbor) AS degree
+
+                // Step 2: Pair up neighbors and check if they are connected
+                UNWIND neighbors AS n1
+                UNWIND neighbors AS n2
+                WITH n, degree, n1, n2
+                WHERE id(n1) < id(n2)  // avoid duplicate pairs
+
+                // Step 3: Use OPTIONAL MATCH to see if n1 and n2 are connected
+                OPTIONAL MATCH (n1)-[:EDGE]-(n2)
+                WITH n, degree, COUNT(n2) AS triangle_count
+
+                // Step 4: Compute local clustering coefficient
+                WITH n, degree,
+                    CASE WHEN degree <= 1 THEN 0.0
+                        ELSE (1.0 * triangle_count) / (degree * (degree - 1) / 2.0)
+                    END AS local_cc
+
+                // Step 5: Compute average
+                RETURN AVG(local_cc) AS avg_clustering_coefficient
+                """
+                clustering = await self.query(clustering_query)
+
+                optional_metrics = {
+                    "num_selfloops": num_selfloops,
+                    "diameter": max(path_lengths) if path_lengths else -1,
+                    "avg_shortest_path_length": sum(path_lengths) / len(path_lengths)
+                    if path_lengths
+                    else -1,
+                    "avg_clustering": clustering[0][0] if clustering and clustering[0][0] else -1,
+                }
+            else:
+                optional_metrics = {
+                    "num_selfloops": -1,
+                    "diameter": -1,
+                    "avg_shortest_path_length": -1,
+                    "avg_clustering": -1,
+                }
+
+            return {**mandatory_metrics, **optional_metrics}
+
+        except Exception as e:
+            logger.error(f"Failed to get graph metrics: {e}")
+            return {
+                "num_nodes": 0,
+                "num_edges": 0,
+                "mean_degree": 0,
+                "edge_density": 0,
+                "num_connected_components": 0,
+                "sizes_of_connected_components": [],
+                "num_selfloops": -1,
+                "diameter": -1,
+                "avg_shortest_path_length": -1,
+                "avg_clustering": -1,
+            }

cognee/infrastructure/databases/vector/exceptions/exceptions.py

@@ -9,6 +9,6 @@ class CollectionNotFoundError(CriticalError):
         name: str = "CollectionNotFoundError",
         status_code: int = status.HTTP_422_UNPROCESSABLE_ENTITY,
         log=True,
-        log_level="ERROR",
+        log_level="DEBUG",
     ):
         super().__init__(message, name, status_code, log, log_level)

cognee/infrastructure/databases/vector/lancedb/LanceDBAdapter.py

@@ -204,29 +204,12 @@ class LanceDBAdapter(VectorDBInterface):
             ]
         )
 
-    def delete_data_points(self, collection_name: str, data_point_ids: list[str]):
-        @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
-        async def _delete_data_points():
-            collection = await self.get_collection(collection_name)
+    async def delete_data_points(self, collection_name: str, data_point_ids: list[str]):
+        collection = await self.get_collection(collection_name)
 
-            # Delete one at a time to avoid commit conflicts
-            for data_point_id in data_point_ids:
-                await collection.delete(f"id = '{data_point_id}'")
-
-            return True
-
-        # Check if we're in an event loop
-        try:
-            loop = asyncio.get_event_loop()
-        except RuntimeError:
-            loop = None
-
-        if loop and loop.is_running():
-            # If we're in a running event loop, create a new task
-            return loop.create_task(_delete_data_points())
-        else:
-            # If we're not in an event loop, run it synchronously
-            return asyncio.run(_delete_data_points())
+        # Delete one at a time to avoid commit conflicts
+        for data_point_id in data_point_ids:
+            await collection.delete(f"id = '{data_point_id}'")
 
     async def create_vector_index(self, index_name: str, index_property_name: str):
         await self.create_collection(

cognee/modules/data/methods/__init__.py

@@ -7,6 +7,7 @@ from .get_datasets import get_datasets
 from .get_datasets_by_name import get_datasets_by_name
 from .get_dataset_data import get_dataset_data
 from .get_data import get_data
+from .get_unique_dataset_id import get_unique_dataset_id
 
 # Delete
 from .delete_dataset import delete_dataset

cognee/modules/data/methods/create_dataset.py

@@ -4,8 +4,13 @@ from sqlalchemy import select
 from sqlalchemy.orm import joinedload
 from cognee.modules.data.models import Dataset
 
+from cognee.modules.data.methods.get_unique_dataset_id import get_unique_dataset_id
+from cognee.modules.users.models import User
+
+
+async def create_dataset(dataset_name: str, user: User, session: AsyncSession) -> Dataset:
+    owner_id = user.id
 
-async def create_dataset(dataset_name: str, owner_id: UUID, session: AsyncSession) -> Dataset:
     dataset = (
         await session.scalars(
             select(Dataset)
@@ -16,10 +21,9 @@ async def create_dataset(dataset_name: str, owner_id: UUID, session: AsyncSessio
     ).first()
 
     if dataset is None:
-        # Dataset id should be generated based on dataset_name and owner_id so multiple users can use the same dataset_name
-        dataset = Dataset(
-            id=uuid5(NAMESPACE_OID, f"{dataset_name}{str(owner_id)}"), name=dataset_name, data=[]
-        )
+        # Dataset id should be generated based on dataset_name and owner_id/user so multiple users can use the same dataset_name
+        dataset_id = await get_unique_dataset_id(dataset_name=dataset_name, user=user)
+        dataset = Dataset(id=dataset_id, name=dataset_name, data=[])
         dataset.owner_id = owner_id
 
         session.add(dataset)

cognee/modules/data/methods/get_unique_dataset_id.py

@@ -0,0 +1,6 @@
+from uuid import UUID, uuid5, NAMESPACE_OID
+from cognee.modules.users.models import User
+
+
+async def get_unique_dataset_id(dataset_name: str, user: User) -> UUID:
+    return uuid5(NAMESPACE_OID, f"{dataset_name}{str(user.id)}")

cognee/modules/pipelines/operations/pipeline.py

@@ -5,6 +5,7 @@ from uuid import uuid5, NAMESPACE_OID
 
 from cognee.modules.data.methods import get_datasets, get_datasets_by_name
 from cognee.modules.data.methods.get_dataset_data import get_dataset_data
+from cognee.modules.data.methods.get_unique_dataset_id import get_unique_dataset_id
 from cognee.modules.data.models import Data, Dataset
 from cognee.modules.pipelines.operations.run_tasks import run_tasks
 from cognee.modules.pipelines.models import PipelineRunStatus
@@ -93,7 +94,7 @@ async def run_pipeline(
     elif isinstance(dataset, str):
         check_dataset_name(dataset)
         # Generate id based on unique dataset_id formula
-        dataset_id = uuid5(NAMESPACE_OID, f"{dataset}{str(user.id)}")
+        dataset_id = await get_unique_dataset_id(dataset_name=dataset, user=user)
 
     if not data:
         data: list[Data] = await get_dataset_data(dataset_id=dataset_id)

cognee/modules/retrieval/utils/brute_force_triplet_search.py

@@ -146,7 +146,7 @@ async def brute_force_search(
     async def search_in_collection(collection_name: str):
         try:
             return await vector_engine.search(
-                collection_name=collection_name, query_text=query, limit=top_k
+                collection_name=collection_name, query_text=query, limit=0
             )
         except CollectionNotFoundError:
             return []

cognee/tasks/ingestion/ingest_data.py

@@ -104,7 +104,7 @@ async def ingest_data(
                 db_engine = get_relational_engine()
 
                 async with db_engine.get_async_session() as session:
-                    dataset = await create_dataset(dataset_name, user.id, session)
+                    dataset = await create_dataset(dataset_name, user, session)
 
                     # Check to see if data should be updated
                     data_point = (

cognee/tasks/temporal_awareness/index_graphiti_objects.py

@@ -20,20 +20,23 @@ async def index_and_transform_graphiti_nodes_and_edges():
         logger.error("Failed to initialize engines: %s", e)
         raise RuntimeError("Initialization error") from e
 
-    await graph_engine.query("""MATCH (n) SET n.id = n.uuid RETURN n""")
+    await graph_engine.query("""MATCH (n) SET n.id = n.uuid RETURN n""", params={})
     await graph_engine.query(
         """MATCH (source)-[r]->(target) SET r.source_node_id = source.id,
                              r.target_node_id = target.id,
-                             r.relationship_name = type(r) RETURN r"""
+                             r.relationship_name = type(r) RETURN r""",
+        params={},
+    )
+    await graph_engine.query(
+        """MATCH (n) SET n.text = COALESCE(n.summary, n.content) RETURN n""", params={}
     )
-    await graph_engine.query("""MATCH (n) SET n.text = COALESCE(n.summary, n.content) RETURN n""")
 
-    nodes_data, edges_data = await graph_engine.get_model_independent_graph_data()
+    nodes_data, edges_data = await graph_engine.get_graph_data()
 
-    for node_data in nodes_data[0]["nodes"]:
+    for node_id, node_data in nodes_data:
         graphiti_node = GraphitiNode(
             **{key: node_data[key] for key in ("content", "name", "summary") if key in node_data},
-            id=node_data.get("uuid"),
+            id=node_id,
         )
 
         data_point_type = type(graphiti_node)
@@ -58,9 +61,8 @@ async def index_and_transform_graphiti_nodes_and_edges():
         await vector_engine.index_data_points(index_name, field_name, indexable_points)
 
     edge_types = Counter(
-        edge[1][1]
-        for edge in edges_data[0]["elements"]
-        if isinstance(edge, list) and len(edge) == 3
+        edge[2]  # The edge key (relationship name) is at index 2
+        for edge in edges_data
     )
 
     for text, count in edge_types.items():

cognee/tests/test_memgraph.py

@@ -0,0 +1,107 @@
+import os
+
+import pathlib
+import cognee
+from cognee.modules.search.operations import get_history
+from cognee.modules.users.methods import get_default_user
+from cognee.shared.logging_utils import get_logger
+from cognee.modules.search.types import SearchType
+
+
+logger = get_logger()
+
+
+async def main():
+    cognee.config.set_graph_database_provider("memgraph")
+    data_directory_path = str(
+        pathlib.Path(
+            os.path.join(pathlib.Path(__file__).parent, ".data_storage/test_memgraph")
+        ).resolve()
+    )
+    cognee.config.data_root_directory(data_directory_path)
+    cognee_directory_path = str(
+        pathlib.Path(
+            os.path.join(pathlib.Path(__file__).parent, ".cognee_system/test_memgraph")
+        ).resolve()
+    )
+    cognee.config.system_root_directory(cognee_directory_path)
+
+    await cognee.prune.prune_data()
+    await cognee.prune.prune_system(metadata=True)
+
+    dataset_name = "cs_explanations"
+
+    explanation_file_path = os.path.join(
+        pathlib.Path(__file__).parent, "test_data/Natural_language_processing.txt"
+    )
+    await cognee.add([explanation_file_path], dataset_name)
+
+    text = """A quantum computer is a computer that takes advantage of quantum mechanical phenomena.
+    At small scales, physical matter exhibits properties of both particles and waves, and quantum computing leverages this behavior, specifically quantum superposition and entanglement, using specialized hardware that supports the preparation and manipulation of quantum states.
+    Classical physics cannot explain the operation of these quantum devices, and a scalable quantum computer could perform some calculations exponentially faster (with respect to input size scaling) than any modern "classical" computer. In particular, a large-scale quantum computer could break widely used encryption schemes and aid physicists in performing physical simulations; however, the current state of the technology is largely experimental and impractical, with several obstacles to useful applications. Moreover, scalable quantum computers do not hold promise for many practical tasks, and for many important tasks quantum speedups are proven impossible.
+    The basic unit of information in quantum computing is the qubit, similar to the bit in traditional digital electronics. Unlike a classical bit, a qubit can exist in a superposition of its two "basis" states. When measuring a qubit, the result is a probabilistic output of a classical bit, therefore making quantum computers nondeterministic in general. If a quantum computer manipulates the qubit in a particular way, wave interference effects can amplify the desired measurement results. The design of quantum algorithms involves creating procedures that allow a quantum computer to perform calculations efficiently and quickly.
+    Physically engineering high-quality qubits has proven challenging. If a physical qubit is not sufficiently isolated from its environment, it suffers from quantum decoherence, introducing noise into calculations. Paradoxically, perfectly isolating qubits is also undesirable because quantum computations typically need to initialize qubits, perform controlled qubit interactions, and measure the resulting quantum states. Each of those operations introduces errors and suffers from noise, and such inaccuracies accumulate.
+    In principle, a non-quantum (classical) computer can solve the same computational problems as a quantum computer, given enough time. Quantum advantage comes in the form of time complexity rather than computability, and quantum complexity theory shows that some quantum algorithms for carefully selected tasks require exponentially fewer computational steps than the best known non-quantum algorithms. Such tasks can in theory be solved on a large-scale quantum computer whereas classical computers would not finish computations in any reasonable amount of time. However, quantum speedup is not universal or even typical across computational tasks, since basic tasks such as sorting are proven to not allow any asymptotic quantum speedup. Claims of quantum supremacy have drawn significant attention to the discipline, but are demonstrated on contrived tasks, while near-term practical use cases remain limited.
+    """
+
+    await cognee.add([text], dataset_name)
+
+    await cognee.cognify([dataset_name])
+
+    from cognee.infrastructure.databases.vector import get_vector_engine
+
+    vector_engine = get_vector_engine()
+    random_node = (await vector_engine.search("Entity_name", "Quantum computer"))[0]
+    random_node_name = random_node.payload["text"]
+
+    search_results = await cognee.search(
+        query_type=SearchType.INSIGHTS, query_text=random_node_name
+    )
+    assert len(search_results) != 0, "The search results list is empty."
+    print("\n\nExtracted sentences are:\n")
+    for result in search_results:
+        print(f"{result}\n")
+
+    search_results = await cognee.search(query_type=SearchType.CHUNKS, query_text=random_node_name)
+    assert len(search_results) != 0, "The search results list is empty."
+    print("\n\nExtracted chunks are:\n")
+    for result in search_results:
+        print(f"{result}\n")
+
+    search_results = await cognee.search(
+        query_type=SearchType.SUMMARIES, query_text=random_node_name
+    )
+    assert len(search_results) != 0, "Query related summaries don't exist."
+    print("\nExtracted results are:\n")
+    for result in search_results:
+        print(f"{result}\n")
+
+    search_results = await cognee.search(
+        query_type=SearchType.NATURAL_LANGUAGE,
+        query_text=f"Find nodes connected to node with name {random_node_name}",
+    )
+    assert len(search_results) != 0, "Query related natural language don't exist."
+    print("\nExtracted results are:\n")
+    for result in search_results:
+        print(f"{result}\n")
+
+    user = await get_default_user()
+    history = await get_history(user.id)
+
+    assert len(history) == 8, "Search history is not correct."
+
+    await cognee.prune.prune_data()
+    assert not os.path.isdir(data_directory_path), "Local data files are not deleted"
+
+    await cognee.prune.prune_system(metadata=True)
+    from cognee.infrastructure.databases.graph import get_graph_engine
+
+    graph_engine = await get_graph_engine()
+    nodes, edges = await graph_engine.get_graph_data()
+    assert len(nodes) == 0 and len(edges) == 0, "Memgraph graph database is not empty"
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

examples/python/graphiti_example.py

@@ -14,6 +14,7 @@ from cognee.modules.retrieval.utils.brute_force_triplet_search import brute_forc
 from cognee.modules.retrieval.graph_completion_retriever import GraphCompletionRetriever
 from cognee.infrastructure.llm.prompts import read_query_prompt, render_prompt
 from cognee.infrastructure.llm.get_llm_client import get_llm_client
+from cognee.modules.users.methods import get_default_user
 
 text_list = [
     "Kamala Harris is the Attorney General of California. She was previously "
@@ -27,6 +28,9 @@ async def main():
     await cognee.prune.prune_system(metadata=True)
     await create_relational_db_and_tables()
 
+    # Initialize default user
+    user = await get_default_user()
+
     for text in text_list:
         await cognee.add(text)
 
@@ -34,7 +38,7 @@ async def main():
         Task(build_graph_with_temporal_awareness, text_list=text_list),
     ]
 
-    pipeline = run_tasks(tasks)
+    pipeline = run_tasks(tasks, user=user)
 
     async for result in pipeline:
         print(result)

notebooks/cognee_demo.ipynb

@@ -467,7 +467,7 @@
     "from cognee.modules.data.models import Dataset, Data\n",
     "from cognee.modules.data.methods.get_dataset_data import get_dataset_data\n",
     "from cognee.modules.cognify.config import get_cognify_config\n",
-    "from cognee.modules.pipelines.tasks.Task import Task\n",
+    "from cognee.modules.pipelines.tasks.task import Task\n",
     "from cognee.modules.pipelines import run_tasks\n",
     "from cognee.modules.users.models import User\n",
     "from cognee.tasks.documents import (\n",
@@ -505,7 +505,7 @@
     "            Task(add_data_points, task_config={\"batch_size\": 10}),\n",
     "        ]\n",
     "\n",
-    "        pipeline_run = run_tasks(tasks, dataset.id, data_documents, \"cognify_pipeline\")\n",
+    "        pipeline_run = run_tasks(tasks, dataset.id, data_documents, user, \"cognify_pipeline\")\n",
     "        pipeline_run_status = None\n",
     "\n",
     "        async for run_status in pipeline_run:\n",
@@ -529,8 +529,11 @@
    "source": [
     "from cognee.modules.users.methods import get_default_user\n",
     "from cognee.modules.data.methods import get_datasets_by_name\n",
+    "from cognee.modules.users.methods import get_user\n",
     "\n",
-    "user = await get_default_user()\n",
+    "default_user = await get_default_user()\n",
+    "\n",
+    "user = await get_user(default_user.id)\n",
     "\n",
     "datasets = await get_datasets_by_name([\"example\"], user.id)\n",
     "\n",
@@ -604,39 +607,6 @@
     "visualization_server(port=8002)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "765bc42a143e98af",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-02-09T21:46:07.783693Z",
-     "start_time": "2025-02-09T21:46:07.780709Z"
-    }
-   },
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e1382358-433c-4cd0-8535-9e103f821034",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6332d5bc-882f-49d5-8496-582e3954567a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from IPython.display import IFrame, display, HTML\n",
-    "\n",
-    "IFrame(\"http://127.0.0.1:8002/.artifacts/graph_visualization.html\", width=800, height=600)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -837,14 +807,6 @@
     "### Give us a star if you like it!\n",
     "https://github.com/topoteretes/cognee"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3c081f2d53512199",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

notebooks/cognee_graphiti_demo.ipynb

@@ -15,7 +15,7 @@
     }
    },
    "source": [
-    "First we import the necessary libaries"
+    "First we import the necessary libraries"
    ]
   },
   {
@@ -24,9 +24,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import asyncio\n",
+    "\n",
     "import cognee\n",
     "from cognee.shared.logging_utils import get_logger, ERROR\n",
-    "import warnings\n",
     "from cognee.modules.pipelines import Task, run_tasks\n",
     "from cognee.tasks.temporal_awareness import build_graph_with_temporal_awareness\n",
     "from cognee.infrastructure.databases.relational import (\n",
@@ -38,7 +39,8 @@
     "from cognee.modules.retrieval.utils.brute_force_triplet_search import brute_force_triplet_search\n",
     "from cognee.modules.retrieval.graph_completion_retriever import GraphCompletionRetriever\n",
     "from cognee.infrastructure.llm.prompts import read_query_prompt, render_prompt\n",
-    "from cognee.infrastructure.llm.get_llm_client import get_llm_client"
+    "from cognee.infrastructure.llm.get_llm_client import get_llm_client\n",
+    "from cognee.modules.users.methods import get_default_user"
    ]
   },
   {
@@ -126,33 +128,25 @@
    },
    "outputs": [],
    "source": [
-    "# 🔧 Setting Up Logging to Suppress Errors\n",
-    "logger = get_logger(level=ERROR) # Keeping logs clean and focused\n",
-    "\n",
-    "# 🧹 Pruning Old Data and Metadata\n",
-    "await cognee.prune.prune_data()  # Removing outdated data\n",
+    "await cognee.prune.prune_data()\n",
     "await cognee.prune.prune_system(metadata=True)\n",
-    "\n",
-    "# 🏗️ Creating Relational Database and Tables\n",
     "await create_relational_db_and_tables()\n",
     "\n",
-    "# 📚 Adding Text Data to Cognee\n",
+    "# Initialize default user\n",
+    "user = await get_default_user()\n",
+    "\n",
     "for text in text_list:\n",
     "    await cognee.add(text)\n",
     "\n",
-    "# 🕰️ Building Temporal-Aware Graphs\n",
     "tasks = [\n",
     "    Task(build_graph_with_temporal_awareness, text_list=text_list),\n",
-    "]\n",
+    "    ]\n",
     "\n",
-    "# 🚀 Running the Task Pipeline\n",
-    "pipeline = run_tasks(tasks)\n",
+    "pipeline = run_tasks(tasks, user=user)\n",
     "\n",
-    "# 🌟 Processing Pipeline Results\n",
     "async for result in pipeline:\n",
-    "    print(f\"✅ Result Processed: {result}\")\n",
+    "    print(result)\n",
     "\n",
-    "# 🔄 Indexing and Transforming Graph Data\n",
     "await index_and_transform_graphiti_nodes_and_edges()"
    ]
   },

notebooks/cognee_llama_index.ipynb

@@ -12,7 +12,9 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": "%pip install llama-index-core\n"
+   "source": [
+    "%pip install llama-index-core\n"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -116,10 +118,10 @@
    ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
-   "outputs": [],
    "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
     "from typing import Union, BinaryIO\n",
     "\n",

notebooks/cognee_simple_demo.ipynb

@@ -10,11 +10,13 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "982b897a29a26f7d",
    "metadata": {},
-   "source": "!pip install cognee==0.1.36",
    "outputs": [],
-   "execution_count": null
+   "source": [
+    "!pip install cognee==0.1.39"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -28,15 +30,15 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "initial_id",
    "metadata": {},
+   "outputs": [],
    "source": [
     "import os\n",
     "\n",
     "os.environ[\"LLM_API_KEY\"] = \"\""
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
    "cell_type": "markdown",
@@ -48,14 +50,14 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "5805c346f03d8070",
    "metadata": {},
+   "outputs": [],
    "source": [
     "current_directory = os.getcwd()\n",
     "file_path = os.path.join(current_directory, \"data\", \"alice_in_wonderland.txt\")"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
    "cell_type": "markdown",
@@ -67,15 +69,15 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "875763366723ee48",
    "metadata": {},
+   "outputs": [],
    "source": [
     "import cognee\n",
     "await cognee.add(file_path)\n",
     "await cognee.cognify()"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
    "cell_type": "markdown",
@@ -87,33 +89,33 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "29b3a1e3279100d2",
    "metadata": {},
+   "outputs": [],
    "source": [
     "await cognee.search(\"List me all the influential characters in Alice in Wonderland.\")"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "883ce50d2d9dc584",
    "metadata": {},
+   "outputs": [],
    "source": [
     "await cognee.search(\"How did Alice end up in Wonderland?\")"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "677e1bc52aa078b6",
    "metadata": {},
+   "outputs": [],
    "source": [
     "await cognee.search(\"Tell me about Alice's personality.\")"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
    "cell_type": "markdown",
@@ -125,8 +127,10 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "id": "6effdae590b795d3",
    "metadata": {},
+   "outputs": [],
    "source": [
     "import webbrowser\n",
     "import os\n",
@@ -136,9 +140,7 @@
     "html_file = os.path.join(home_dir, \"graph_visualization.html\")\n",
     "display(html_file)\n",
     "webbrowser.open(f\"file://{html_file}\")"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
    "cell_type": "markdown",

notebooks/github_graph_visualization.html

@@ -1,128 +0,0 @@
-
-    <!DOCTYPE html>
-    <html>
-    <head>
-        <meta charset="utf-8">
-        <script src="https://d3js.org/d3.v5.min.js"></script>
-        <style>
-            body, html { margin: 0; padding: 0; width: 100%; height: 100%; overflow: hidden; background: linear-gradient(90deg, #101010, #1a1a2e); color: white; font-family: 'Inter', sans-serif; }
-
-            svg { width: 100vw; height: 100vh; display: block; }
-            .links line { stroke: rgba(255, 255, 255, 0.4); stroke-width: 2px; }
-            .nodes circle { stroke: white; stroke-width: 0.5px; filter: drop-shadow(0 0 5px rgba(255,255,255,0.3)); }
-            .node-label { font-size: 5px; font-weight: bold; fill: white; text-anchor: middle; dominant-baseline: middle; font-family: 'Inter', sans-serif; pointer-events: none; }
-            .edge-label { font-size: 3px; fill: rgba(255, 255, 255, 0.7); text-anchor: middle; dominant-baseline: middle; font-family: 'Inter', sans-serif; pointer-events: none; }
-        </style>
-    </head>
-    <body>
-        <svg></svg>
-        <script>
-            var nodes = [];
-            var links = [];
-
-            var svg = d3.select("svg"),
-                width = window.innerWidth,
-                height = window.innerHeight;
-
-            var container = svg.append("g");
-
-            var simulation = d3.forceSimulation(nodes)
-                .force("link", d3.forceLink(links).id(d => d.id).strength(0.1))
-                .force("charge", d3.forceManyBody().strength(-275))
-                .force("center", d3.forceCenter(width / 2, height / 2))
-                .force("x", d3.forceX().strength(0.1).x(width / 2))
-                .force("y", d3.forceY().strength(0.1).y(height / 2));
-
-            var link = container.append("g")
-                .attr("class", "links")
-                .selectAll("line")
-                .data(links)
-                .enter().append("line")
-                .attr("stroke-width", 2);
-
-            var edgeLabels = container.append("g")
-                .attr("class", "edge-labels")
-                .selectAll("text")
-                .data(links)
-                .enter().append("text")
-                .attr("class", "edge-label")
-                .text(d => d.relation);
-
-            var nodeGroup = container.append("g")
-                .attr("class", "nodes")
-                .selectAll("g")
-                .data(nodes)
-                .enter().append("g");
-
-            var node = nodeGroup.append("circle")
-                .attr("r", 13)
-                .attr("fill", d => d.color)
-                .call(d3.drag()
-                    .on("start", dragstarted)
-                    .on("drag", dragged)
-                    .on("end", dragended));
-
-            nodeGroup.append("text")
-                .attr("class", "node-label")
-                .attr("dy", 4)
-                .attr("text-anchor", "middle")
-                .text(d => d.name);
-
-            node.append("title").text(d => JSON.stringify(d));
-
-            simulation.on("tick", function() {
-                link.attr("x1", d => d.source.x)
-                    .attr("y1", d => d.source.y)
-                    .attr("x2", d => d.target.x)
-                    .attr("y2", d => d.target.y);
-
-                edgeLabels
-                    .attr("x", d => (d.source.x + d.target.x) / 2)
-                    .attr("y", d => (d.source.y + d.target.y) / 2 - 5);
-
-                node.attr("cx", d => d.x)
-                    .attr("cy", d => d.y);
-
-                nodeGroup.select("text")
-                    .attr("x", d => d.x)
-                    .attr("y", d => d.y)
-                    .attr("dy", 4)
-                    .attr("text-anchor", "middle");
-            });
-
-            svg.call(d3.zoom().on("zoom", function() {
-                container.attr("transform", d3.event.transform);
-            }));
-
-            function dragstarted(d) {
-                if (!d3.event.active) simulation.alphaTarget(0.3).restart();
-                d.fx = d.x;
-                d.fy = d.y;
-            }
-
-            function dragged(d) {
-                d.fx = d3.event.x;
-                d.fy = d3.event.y;
-            }
-
-            function dragended(d) {
-                if (!d3.event.active) simulation.alphaTarget(0);
-                d.fx = null;
-                d.fy = null;
-            }
-
-            window.addEventListener("resize", function() {
-                width = window.innerWidth;
-                height = window.innerHeight;
-                svg.attr("width", width).attr("height", height);
-                simulation.force("center", d3.forceCenter(width / 2, height / 2));
-                simulation.alpha(1).restart();
-            });
-        </script>
-
-        <svg style="position: fixed; bottom: 10px; right: 10px; width: 150px; height: auto; z-index: 9999;" viewBox="0 0 158 44" fill="none" xmlns="http://www.w3.org/2000/svg">
-            <path fill-rule="evenodd" clip-rule="evenodd" d="M11.7496 4.92654C7.83308 4.92654 4.8585 7.94279 4.8585 11.3612V14.9304C4.8585 18.3488 7.83308 21.3651 11.7496 21.3651C13.6831 21.3651 15.0217 20.8121 16.9551 19.3543C18.0458 18.5499 19.5331 18.8013 20.3263 19.9072C21.1195 21.0132 20.8717 22.5213 19.781 23.3257C17.3518 25.0851 15.0217 26.2414 11.7 26.2414C5.35425 26.2414 0 21.2646 0 14.9304V11.3612C0 4.97681 5.35425 0.0502739 11.7 0.0502739C15.0217 0.0502739 17.3518 1.2065 19.781 2.96598C20.8717 3.77032 21.1195 5.27843 20.3263 6.38439C19.5331 7.49035 18.0458 7.69144 16.9551 6.93737C15.0217 5.52979 13.6831 4.92654 11.7496 4.92654ZM35.5463 4.92654C31.7289 4.92654 28.6552 8.04333 28.6552 11.8639V14.478C28.6552 18.2986 31.7289 21.4154 35.5463 21.4154C39.3141 21.4154 42.3878 18.2986 42.3878 14.478V11.8639C42.3878 8.04333 39.3141 4.92654 35.5463 4.92654ZM23.7967 11.8639C23.7967 5.32871 29.0518 0 35.5463 0C42.0408 0 47.2463 5.32871 47.2463 11.8639V14.478C47.2463 21.0132 42.0408 26.3419 35.5463 26.3419C29.0518 26.3419 23.7967 21.0635 23.7967 14.478V11.8639ZM63.3091 5.07736C59.4917 5.07736 56.418 8.19415 56.418 12.0147C56.418 15.8353 59.4917 18.9521 63.3091 18.9521C67.1265 18.9521 70.1506 15.8856 70.1506 12.0147C70.1506 8.14388 67.0769 5.07736 63.3091 5.07736ZM51.5595 11.9645C51.5595 5.42925 56.8146 0.150814 63.3091 0.150814C66.0854 0.150814 68.5642 1.10596 70.5968 2.71463L72.4311 0.904876C73.3731 -0.0502693 74.9099 -0.0502693 75.8519 0.904876C76.7938 1.86002 76.7938 3.41841 75.8519 4.37356L73.7201 6.53521C74.5629 8.19414 75.0587 10.0542 75.0587 12.0147C75.0587 18.4997 69.8532 23.8284 63.3587 23.8284C63.3091 23.8284 63.2099 23.8284 63.1603 23.8284H58.0044C57.1616 23.8284 56.4675 24.5322 56.4675 25.3868C56.4675 26.2414 57.1616 26.9452 58.0044 26.9452H64.6476H66.7794C68.5146 26.9452 70.3489 27.4479 71.7866 28.6041C73.2739 29.8106 74.2159 31.5701 74.4142 33.7317C74.7116 37.6026 72.0345 40.2166 69.8532 41.0713L63.8048 43.7859C62.5654 44.3389 61.1277 43.7859 60.6319 42.5291C60.0866 41.2723 60.6319 39.8648 61.8714 39.3118L68.0188 36.5972C68.0684 36.5972 68.118 36.5469 68.1675 36.5469C68.4154 36.4463 68.8616 36.1447 69.2087 35.6923C69.5061 35.2398 69.7044 34.7371 69.6548 34.1339C69.6053 33.229 69.2582 32.7263 68.8616 32.4247C68.4154 32.0728 67.7214 31.8214 66.8786 31.8214H58.2027C58.1531 31.8214 58.1531 31.8214 58.1035 31.8214H58.054C54.534 31.8214 51.6586 28.956 51.6586 25.3868C51.6586 23.0743 52.8485 21.0635 54.6828 19.9072C52.6997 17.7959 51.5595 15.031 51.5595 11.9645ZM90.8736 5.07736C87.0562 5.07736 83.9824 8.19415 83.9824 12.0147V23.9289C83.9824 25.2862 82.8917 26.3922 81.5532 26.3922C80.2146 26.3922 79.1239 25.2862 79.1239 23.9289V11.9645C79.1239 5.42925 84.379 0.150814 90.824 0.150814C97.2689 0.150814 102.524 5.42925 102.524 11.9645V23.8786C102.524 25.2359 101.433 26.3419 100.095 26.3419C98.7562 26.3419 97.6655 25.2359 97.6655 23.8786V11.9645C97.7647 8.14387 94.6414 5.07736 90.8736 5.07736ZM119.43 5.07736C115.513 5.07736 112.39 8.24441 112.39 12.065V14.5785C112.39 18.4494 115.513 21.5662 119.43 21.5662C120.768 21.5662 122.057 21.164 123.098 20.5105C124.238 19.8067 125.726 20.1586 126.42 21.3148C127.114 22.4711 126.767 23.9792 125.627 24.683C123.842 25.7889 121.71 26.4425 119.43 26.4425C112.885 26.4425 107.581 21.1137 107.581 14.5785V12.065C107.581 5.47952 112.935 0.201088 119.43 0.201088C125.032 0.201088 129.692 4.07194 130.931 9.3001L131.427 11.3612L121.115 15.584C119.876 16.0867 118.488 15.4834 117.942 14.2266C117.447 12.9699 118.041 11.5623 119.281 11.0596L125.478 8.54604C124.238 6.43466 122.008 5.07736 119.43 5.07736ZM146.003 5.07736C142.086 5.07736 138.963 8.24441 138.963 12.065V14.5785C138.963 18.4494 142.086 21.5662 146.003 21.5662C147.341 21.5662 148.63 21.164 149.671 20.5105C150.217 20.1586 150.663 19.8067 151.109 19.304C152.001 18.2986 153.538 18.2483 154.53 19.2034C155.521 20.1083 155.571 21.6667 154.629 22.6721C153.935 23.4262 153.092 24.13 152.2 24.683C150.415 25.7889 148.283 26.4425 146.003 26.4425C139.458 26.4425 134.154 21.1137 134.154 14.5785V12.065C134.154 5.47952 139.508 0.201088 146.003 0.201088C151.605 0.201088 156.265 4.07194 157.504 9.3001L158 11.3612L147.688 15.584C146.449 16.0867 145.061 15.4834 144.515 14.2266C144.019 12.9699 144.614 11.5623 145.854 11.0596L152.051 8.54604C150.762 6.43466 148.58 5.07736 146.003 5.07736Z" fill="white"/>
-        </svg>
-    </body>
-    </html>
-    

notebooks/graphrag_vs_rag.ipynb

@@ -1,8 +1,8 @@
 {
  "cells": [
   {
-   "metadata": {},
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "## Cognee GraphRAG\n",
     "\n",
@@ -48,15 +48,19 @@
    ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
-   "source": "!pip install cognee==0.1.24",
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
-   "execution_count": null
+   "source": [
+    "!pip install cognee==0.1.39"
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
     "import os\n",
     "import cognee\n",
@@ -66,13 +70,11 @@
     "\n",
     "if \"OPENAI_API_KEY\" not in os.environ:\n",
     "    os.environ[\"OPENAI_API_KEY\"] = \"\""
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "Ensure you’ve set up your API keys and installed necessary dependencies.\n",
     "\n",
@@ -82,19 +84,19 @@
    ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
     "documents = [\"Jessica Miller, Experienced Sales Manager with a strong track record in building high-performing teams.\",\n",
     "             \"David Thompson, Creative Graphic Designer with over 8 years of experience in visual design and branding.\"\n",
     "    ]"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "### 3. Adding Data to Cognee\n",
     "\n",
@@ -102,15 +104,17 @@
    ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
-   "source": "await cognee.add(documents)",
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
-   "execution_count": null
+   "source": [
+    "await cognee.add(documents)"
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "This step prepares the data for graph-based processing.\n",
     "\n",
@@ -120,15 +124,17 @@
    ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
-   "source": "await cognee.cognify()",
+   "execution_count": null,
+   "metadata": {},
    "outputs": [],
-   "execution_count": null
+   "source": [
+    "await cognee.cognify()"
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "The graph now contains nodes and relationships derived from the dataset, creating a powerful structure for exploration.\n",
     "\n",
@@ -138,45 +144,49 @@
    ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "from cognee.modules.search.types import SearchType\n",
-    "search_results = await cognee.search(SearchType.GRAPH_COMPLETION, \"Tell me who are the people mentioned?\")\n",
+    "from cognee.api.v1.search import SearchType\n",
+    "search_results = await cognee.search(query_type=SearchType.GRAPH_COMPLETION, query_text=\"Tell me who are the people mentioned?\")\n",
     "\n",
     "print(\"\\n\\nAnswer based on knowledge graph:\\n\")\n",
     "for result in search_results:\n",
     "    print(f\"{result}\\n\")"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
-   "source": "### Answer prompt based on RAG approach:"
+   "metadata": {},
+   "source": [
+    "### Answer prompt based on RAG approach:"
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "search_results = await cognee.search(SearchType.COMPLETION, \"Tell me who are the people mentioned?\")\n",
+    "search_results = await cognee.search(query_type=SearchType.RAG_COMPLETION, query_text=\"Tell me who are the people mentioned?\")\n",
     "\n",
     "print(\"\\n\\nAnswer based on RAG:\\n\")\n",
     "for result in search_results:\n",
     "    print(f\"{result}\\n\")"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
-   "source": "In conclusion, the results demonstrate a significant advantage of the knowledge graph-based approach (Graphrag) over the RAG approach. Graphrag successfully identified all the mentioned individuals across multiple documents, showcasing its ability to aggregate and infer information from a global context. In contrast, the RAG approach was limited to identifying individuals within a single document due to its chunking-based processing constraints. This highlights Graphrag's superior capability in comprehensively resolving queries that span across a broader corpus of interconnected data."
+   "metadata": {},
+   "source": [
+    "In conclusion, the results demonstrate a significant advantage of the knowledge graph-based approach (Graphrag) over the RAG approach. Graphrag successfully identified all the mentioned individuals across multiple documents, showcasing its ability to aggregate and infer information from a global context. In contrast, the RAG approach was limited to identifying individuals within a single document due to its chunking-based processing constraints. This highlights Graphrag's superior capability in comprehensively resolving queries that span across a broader corpus of interconnected data."
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "### 7. Finding Related Nodes\n",
     "\n",
@@ -184,21 +194,21 @@
    ]
   },
   {
-   "metadata": {},
    "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "related_nodes = await cognee.search(SearchType.INSIGHTS, \"person\")\n",
+    "related_nodes = await cognee.search(query_type=SearchType.INSIGHTS, query_text=\"person\")\n",
     "\n",
     "print(\"\\n\\nRelated nodes are:\\n\")\n",
     "for node in related_nodes:\n",
     "    print(f\"{node}\\n\")"
-   ],
-   "outputs": [],
-   "execution_count": null
+   ]
   },
   {
-   "metadata": {},
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "## Why Choose Cognee?\n",
     "\n",
@@ -233,9 +243,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "name": "python3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "display_name": "Python 3 (ipykernel)"
+   "name": "python3"
   }
  },
  "nbformat": 4,

notebooks/hr_demo.ipynb

@@ -1,978 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "d35ac8ce-0f92-46f5-9ba4-a46970f0ce19",
-   "metadata": {},
-   "source": [
-    "# Cognee - Get Started"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "074f0ea8-c659-4736-be26-be4b0e5ac665",
-   "metadata": {},
-   "source": [
-    "# Demo time"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0587d91d",
-   "metadata": {},
-   "source": [
-    "#### First let's define some data that we will cognify and perform a search on"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "df16431d0f48b006",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-09-20T14:02:48.519686Z",
-     "start_time": "2024-09-20T14:02:48.515589Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "job_position = \"\"\"Senior Data Scientist (Machine Learning)\n",
-    "\n",
-    "Company: TechNova Solutions\n",
-    "Location: San Francisco, CA\n",
-    "\n",
-    "Job Description:\n",
-    "\n",
-    "TechNova Solutions is seeking a Senior Data Scientist specializing in Machine Learning to join our dynamic analytics team. The ideal candidate will have a strong background in developing and deploying machine learning models, working with large datasets, and translating complex data into actionable insights.\n",
-    "\n",
-    "Responsibilities:\n",
-    "\n",
-    "Develop and implement advanced machine learning algorithms and models.\n",
-    "Analyze large, complex datasets to extract meaningful patterns and insights.\n",
-    "Collaborate with cross-functional teams to integrate predictive models into products.\n",
-    "Stay updated with the latest advancements in machine learning and data science.\n",
-    "Mentor junior data scientists and provide technical guidance.\n",
-    "Qualifications:\n",
-    "\n",
-    "Master’s or Ph.D. in Data Science, Computer Science, Statistics, or a related field.\n",
-    "5+ years of experience in data science and machine learning.\n",
-    "Proficient in Python, R, and SQL.\n",
-    "Experience with deep learning frameworks (e.g., TensorFlow, PyTorch).\n",
-    "Strong problem-solving skills and attention to detail.\n",
-    "Candidate CVs\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9086abf3af077ab4",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-09-20T14:02:49.120838Z",
-     "start_time": "2024-09-20T14:02:49.118294Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "job_1 = \"\"\"\n",
-    "CV 1: Relevant\n",
-    "Name: Dr. Emily Carter\n",
-    "Contact Information:\n",
-    "\n",
-    "Email: emily.carter@example.com\n",
-    "Phone: (555) 123-4567\n",
-    "Summary:\n",
-    "\n",
-    "Senior Data Scientist with over 8 years of experience in machine learning and predictive analytics. Expertise in developing advanced algorithms and deploying scalable models in production environments.\n",
-    "\n",
-    "Education:\n",
-    "\n",
-    "Ph.D. in Computer Science, Stanford University (2014)\n",
-    "B.S. in Mathematics, University of California, Berkeley (2010)\n",
-    "Experience:\n",
-    "\n",
-    "Senior Data Scientist, InnovateAI Labs (2016 – Present)\n",
-    "Led a team in developing machine learning models for natural language processing applications.\n",
-    "Implemented deep learning algorithms that improved prediction accuracy by 25%.\n",
-    "Collaborated with cross-functional teams to integrate models into cloud-based platforms.\n",
-    "Data Scientist, DataWave Analytics (2014 – 2016)\n",
-    "Developed predictive models for customer segmentation and churn analysis.\n",
-    "Analyzed large datasets using Hadoop and Spark frameworks.\n",
-    "Skills:\n",
-    "\n",
-    "Programming Languages: Python, R, SQL\n",
-    "Machine Learning: TensorFlow, Keras, Scikit-Learn\n",
-    "Big Data Technologies: Hadoop, Spark\n",
-    "Data Visualization: Tableau, Matplotlib\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a9de0cc07f798b7f",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-09-20T14:02:49.675003Z",
-     "start_time": "2024-09-20T14:02:49.671615Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "job_2 = \"\"\"\n",
-    "CV 2: Relevant\n",
-    "Name: Michael Rodriguez\n",
-    "Contact Information:\n",
-    "\n",
-    "Email: michael.rodriguez@example.com\n",
-    "Phone: (555) 234-5678\n",
-    "Summary:\n",
-    "\n",
-    "Data Scientist with a strong background in machine learning and statistical modeling. Skilled in handling large datasets and translating data into actionable business insights.\n",
-    "\n",
-    "Education:\n",
-    "\n",
-    "M.S. in Data Science, Carnegie Mellon University (2013)\n",
-    "B.S. in Computer Science, University of Michigan (2011)\n",
-    "Experience:\n",
-    "\n",
-    "Senior Data Scientist, Alpha Analytics (2017 – Present)\n",
-    "Developed machine learning models to optimize marketing strategies.\n",
-    "Reduced customer acquisition cost by 15% through predictive modeling.\n",
-    "Data Scientist, TechInsights (2013 – 2017)\n",
-    "Analyzed user behavior data to improve product features.\n",
-    "Implemented A/B testing frameworks to evaluate product changes.\n",
-    "Skills:\n",
-    "\n",
-    "Programming Languages: Python, Java, SQL\n",
-    "Machine Learning: Scikit-Learn, XGBoost\n",
-    "Data Visualization: Seaborn, Plotly\n",
-    "Databases: MySQL, MongoDB\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "185ff1c102d06111",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-09-20T14:02:50.286828Z",
-     "start_time": "2024-09-20T14:02:50.284369Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "job_3 = \"\"\"\n",
-    "CV 3: Relevant\n",
-    "Name: Sarah Nguyen\n",
-    "Contact Information:\n",
-    "\n",
-    "Email: sarah.nguyen@example.com\n",
-    "Phone: (555) 345-6789\n",
-    "Summary:\n",
-    "\n",
-    "Data Scientist specializing in machine learning with 6 years of experience. Passionate about leveraging data to drive business solutions and improve product performance.\n",
-    "\n",
-    "Education:\n",
-    "\n",
-    "M.S. in Statistics, University of Washington (2014)\n",
-    "B.S. in Applied Mathematics, University of Texas at Austin (2012)\n",
-    "Experience:\n",
-    "\n",
-    "Data Scientist, QuantumTech (2016 – Present)\n",
-    "Designed and implemented machine learning algorithms for financial forecasting.\n",
-    "Improved model efficiency by 20% through algorithm optimization.\n",
-    "Junior Data Scientist, DataCore Solutions (2014 – 2016)\n",
-    "Assisted in developing predictive models for supply chain optimization.\n",
-    "Conducted data cleaning and preprocessing on large datasets.\n",
-    "Skills:\n",
-    "\n",
-    "Programming Languages: Python, R\n",
-    "Machine Learning Frameworks: PyTorch, Scikit-Learn\n",
-    "Statistical Analysis: SAS, SPSS\n",
-    "Cloud Platforms: AWS, Azure\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d55ce4c58f8efb67",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-09-20T14:02:50.950343Z",
-     "start_time": "2024-09-20T14:02:50.946378Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "job_4 = \"\"\"\n",
-    "CV 4: Not Relevant\n",
-    "Name: David Thompson\n",
-    "Contact Information:\n",
-    "\n",
-    "Email: david.thompson@example.com\n",
-    "Phone: (555) 456-7890\n",
-    "Summary:\n",
-    "\n",
-    "Creative Graphic Designer with over 8 years of experience in visual design and branding. Proficient in Adobe Creative Suite and passionate about creating compelling visuals.\n",
-    "\n",
-    "Education:\n",
-    "\n",
-    "B.F.A. in Graphic Design, Rhode Island School of Design (2012)\n",
-    "Experience:\n",
-    "\n",
-    "Senior Graphic Designer, CreativeWorks Agency (2015 – Present)\n",
-    "Led design projects for clients in various industries.\n",
-    "Created branding materials that increased client engagement by 30%.\n",
-    "Graphic Designer, Visual Innovations (2012 – 2015)\n",
-    "Designed marketing collateral, including brochures, logos, and websites.\n",
-    "Collaborated with the marketing team to develop cohesive brand strategies.\n",
-    "Skills:\n",
-    "\n",
-    "Design Software: Adobe Photoshop, Illustrator, InDesign\n",
-    "Web Design: HTML, CSS\n",
-    "Specialties: Branding and Identity, Typography\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ca4ecc32721ad332",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-09-20T14:02:51.548191Z",
-     "start_time": "2024-09-20T14:02:51.545520Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "job_5 = \"\"\"\n",
-    "CV 5: Not Relevant\n",
-    "Name: Jessica Miller\n",
-    "Contact Information:\n",
-    "\n",
-    "Email: jessica.miller@example.com\n",
-    "Phone: (555) 567-8901\n",
-    "Summary:\n",
-    "\n",
-    "Experienced Sales Manager with a strong track record in driving sales growth and building high-performing teams. Excellent communication and leadership skills.\n",
-    "\n",
-    "Education:\n",
-    "\n",
-    "B.A. in Business Administration, University of Southern California (2010)\n",
-    "Experience:\n",
-    "\n",
-    "Sales Manager, Global Enterprises (2015 – Present)\n",
-    "Managed a sales team of 15 members, achieving a 20% increase in annual revenue.\n",
-    "Developed sales strategies that expanded customer base by 25%.\n",
-    "Sales Representative, Market Leaders Inc. (2010 – 2015)\n",
-    "Consistently exceeded sales targets and received the 'Top Salesperson' award in 2013.\n",
-    "Skills:\n",
-    "\n",
-    "Sales Strategy and Planning\n",
-    "Team Leadership and Development\n",
-    "CRM Software: Salesforce, Zoho\n",
-    "Negotiation and Relationship Building\n",
-    "\"\"\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4415446a",
-   "metadata": {},
-   "source": [
-    "#### Please add the necessary environment information bellow:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "bce39dc6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "\n",
-    "# Setting environment variables\n",
-    "if \"GRAPHISTRY_USERNAME\" not in os.environ:\n",
-    "    os.environ[\"GRAPHISTRY_USERNAME\"] = \"\"\n",
-    "\n",
-    "if \"GRAPHISTRY_PASSWORD\" not in os.environ:\n",
-    "    os.environ[\"GRAPHISTRY_PASSWORD\"] = \"\"\n",
-    "\n",
-    "if \"LLM_API_KEY\" not in os.environ:\n",
-    "    os.environ[\"LLM_API_KEY\"] = \"\"\n",
-    "\n",
-    "# \"neo4j\" or \"networkx\"\n",
-    "os.environ[\"GRAPH_DATABASE_PROVIDER\"] = \"networkx\"\n",
-    "# Not needed if using networkx\n",
-    "# os.environ[\"GRAPH_DATABASE_URL\"]=\"\"\n",
-    "# os.environ[\"GRAPH_DATABASE_USERNAME\"]=\"\"\n",
-    "# os.environ[\"GRAPH_DATABASE_PASSWORD\"]=\"\"\n",
-    "\n",
-    "# \"pgvector\", \"qdrant\", \"weaviate\" or \"lancedb\"\n",
-    "os.environ[\"VECTOR_DB_PROVIDER\"] = \"lancedb\"\n",
-    "# Not needed if using \"lancedb\" or \"pgvector\"\n",
-    "# os.environ[\"VECTOR_DB_URL\"]=\"\"\n",
-    "# os.environ[\"VECTOR_DB_KEY\"]=\"\"\n",
-    "\n",
-    "# Relational Database provider \"sqlite\" or \"postgres\"\n",
-    "os.environ[\"DB_PROVIDER\"] = \"sqlite\"\n",
-    "\n",
-    "# Database name\n",
-    "os.environ[\"DB_NAME\"] = \"cognee_db\"\n",
-    "\n",
-    "# Postgres specific parameters (Only if Postgres or PGVector is used)\n",
-    "# os.environ[\"DB_HOST\"]=\"127.0.0.1\"\n",
-    "# os.environ[\"DB_PORT\"]=\"5432\"\n",
-    "# os.environ[\"DB_USERNAME\"]=\"cognee\"\n",
-    "# os.environ[\"DB_PASSWORD\"]=\"cognee\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9f1a1dbd",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Reset the cognee system with the following command:\n",
-    "\n",
-    "import cognee\n",
-    "\n",
-    "await cognee.prune.prune_data()\n",
-    "await cognee.prune.prune_system(metadata=True)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "383d6971",
-   "metadata": {},
-   "source": [
-    "#### After we have defined and gathered our data let's add it to cognee "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "904df61ba484a8e5",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-09-20T14:02:54.243987Z",
-     "start_time": "2024-09-20T14:02:52.498195Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "import cognee\n",
-    "\n",
-    "await cognee.add([job_1, job_2, job_3, job_4, job_5, job_position], \"example\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0f15c5b1",
-   "metadata": {},
-   "source": [
-    "#### All good, let's cognify it."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7c431fdef4921ae0",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-09-20T14:02:57.925667Z",
-     "start_time": "2024-09-20T14:02:57.922353Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "from cognee.shared.data_models import KnowledgeGraph\n",
-    "from cognee.modules.data.models import Dataset, Data\n",
-    "from cognee.modules.data.methods.get_dataset_data import get_dataset_data\n",
-    "from cognee.modules.cognify.config import get_cognify_config\n",
-    "from cognee.modules.pipelines.tasks.Task import Task\n",
-    "from cognee.modules.pipelines import run_tasks\n",
-    "from cognee.modules.users.models import User\n",
-    "from cognee.tasks.documents import (\n",
-    "    check_permissions_on_documents,\n",
-    "    classify_documents,\n",
-    "    extract_chunks_from_documents,\n",
-    ")\n",
-    "from cognee.tasks.graph import extract_graph_from_data\n",
-    "from cognee.tasks.storage import add_data_points\n",
-    "from cognee.tasks.summarization import summarize_text\n",
-    "\n",
-    "\n",
-    "async def run_cognify_pipeline(dataset: Dataset, user: User = None):\n",
-    "    data_documents: list[Data] = await get_dataset_data(dataset_id=dataset.id)\n",
-    "\n",
-    "    try:\n",
-    "        cognee_config = get_cognify_config()\n",
-    "\n",
-    "        tasks = [\n",
-    "            Task(classify_documents),\n",
-    "            Task(check_permissions_on_documents, user=user, permissions=[\"write\"]),\n",
-    "            Task(extract_chunks_from_documents),  # Extract text chunks based on the document type.\n",
-    "            Task(\n",
-    "                extract_graph_from_data, graph_model=KnowledgeGraph,\n",
-    "                task_config={\"batch_size\": 10}\n",
-    "            ),  # Generate knowledge graphs from the document chunks.\n",
-    "            Task(\n",
-    "                summarize_text,\n",
-    "                summarization_model=cognee_config.summarization_model,\n",
-    "                task_config={\"batch_size\": 10},\n",
-    "            ),\n",
-    "            Task(add_data_points, task_config={\"batch_size\": 10}),\n",
-    "        ]\n",
-    "\n",
-    "        pipeline = run_tasks(tasks, data_documents)\n",
-    "\n",
-    "        async for result in pipeline:\n",
-    "            print(result)\n",
-    "    except Exception as error:\n",
-    "        raise error"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f0a91b99c6215e09",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-09-20T14:02:58.905774Z",
-     "start_time": "2024-09-20T14:02:58.625915Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "from cognee.modules.users.methods import get_default_user\n",
-    "from cognee.modules.data.methods import get_datasets_by_name\n",
-    "\n",
-    "user = await get_default_user()\n",
-    "\n",
-    "datasets = await get_datasets_by_name([\"example\"], user.id)\n",
-    "\n",
-    "await run_cognify_pipeline(datasets[0], user)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "219a6d41",
-   "metadata": {},
-   "source": [
-    "#### We get the url to the graph on graphistry in the notebook cell bellow, showing nodes and connections made by the cognify process."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "080389e5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "from cognee.shared.utils import render_graph\n",
-    "from cognee.infrastructure.databases.graph import get_graph_engine\n",
-    "import graphistry\n",
-    "\n",
-    "graphistry.login(\n",
-    "    username=os.getenv(\"GRAPHISTRY_USERNAME\"), password=os.getenv(\"GRAPHISTRY_PASSWORD\")\n",
-    ")\n",
-    "\n",
-    "graph_engine = await get_graph_engine()\n",
-    "\n",
-    "graph_url = await render_graph(graph_engine.graph)\n",
-    "print(graph_url)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "59e6c3c3",
-   "metadata": {},
-   "source": [
-    "#### We can also do a search on the data to explore the knowledge."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e5e7dfc8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "async def search(\n",
-    "    vector_engine,\n",
-    "    collection_name: str,\n",
-    "    query_text: str = None,\n",
-    "):\n",
-    "    query_vector = (await vector_engine.embedding_engine.embed_text([query_text]))[0]\n",
-    "\n",
-    "    connection = await vector_engine.get_connection()\n",
-    "    collection = await connection.open_table(collection_name)\n",
-    "\n",
-    "    results = await collection.vector_search(query_vector).limit(10).to_pandas()\n",
-    "\n",
-    "    result_values = list(results.to_dict(\"index\").values())\n",
-    "\n",
-    "    return [\n",
-    "        dict(\n",
-    "            id=str(result[\"id\"]),\n",
-    "            payload=result[\"payload\"],\n",
-    "            score=result[\"_distance\"],\n",
-    "        )\n",
-    "        for result in result_values\n",
-    "    ]\n",
-    "\n",
-    "\n",
-    "from cognee.infrastructure.databases.vector import get_vector_engine\n",
-    "\n",
-    "vector_engine = get_vector_engine()\n",
-    "results = await search(vector_engine, \"Entity_name\", \"sarah.nguyen@example.com\")\n",
-    "for result in results:\n",
-    "    print(result)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "81fa2b00",
-   "metadata": {},
-   "source": [
-    "#### We normalize search output scores so the lower the score of the search result is the higher the chance that it's what you're looking for. In the example above we have searched for node entities in the knowledge graph related to \"sarah.nguyen@example.com\""
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "1b94ff96",
-   "metadata": {},
-   "source": [
-    "#### In the example bellow we'll use cognee search to summarize information regarding the node most related to \"sarah.nguyen@example.com\" in the knowledge graph"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "21a3e9a6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from cognee.api.v1.search import SearchType\n",
-    "\n",
-    "node = (await vector_engine.search(\"Entity_name\", \"sarah.nguyen@example.com\"))[0]\n",
-    "node_name = node.payload[\"text\"]\n",
-    "\n",
-    "search_results = await cognee.search(query_type=SearchType.SUMMARIES, query_text=node_name)\n",
-    "print(\"\\n\\Extracted summaries are:\\n\")\n",
-    "for result in search_results:\n",
-    "    print(f\"{result}\\n\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "fd6e5fe2",
-   "metadata": {},
-   "source": [
-    "#### In this example we'll use cognee search to find chunks in which the node most related to \"sarah.nguyen@example.com\" is a part of"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c7a8abff",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "search_results = await cognee.search(query_type=SearchType.CHUNKS, query_text=node_name)\n",
-    "print(\"\\n\\nExtracted chunks are:\\n\")\n",
-    "for result in search_results:\n",
-    "    print(f\"{result}\\n\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "47f0112f",
-   "metadata": {},
-   "source": [
-    "#### In this example we'll use cognee search to give us insights from the knowledge graph related to the node most related to \"sarah.nguyen@example.com\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "706a3954",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "search_results = await cognee.search(query_type=SearchType.INSIGHTS, query_text=node_name)\n",
-    "print(\"\\n\\nExtracted sentences are:\\n\")\n",
-    "for result in search_results:\n",
-    "    print(f\"{result}\\n\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e519e30c0423c2a",
-   "metadata": {},
-   "source": [
-    "## Let's add evals"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3845443e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!pip install \"cognee[deepeval]\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7a2c3c70",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from evals.eval_on_hotpot import deepeval_answers, answer_qa_instance\n",
-    "from evals.qa_dataset_utils import load_qa_dataset\n",
-    "from evals.qa_metrics_utils import get_metrics\n",
-    "from evals.qa_context_provider_utils import qa_context_providers\n",
-    "from pathlib import Path\n",
-    "from tqdm import tqdm\n",
-    "import statistics\n",
-    "import random"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "53a609d8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "num_samples = 10  # With cognee, it takes ~1m10s per sample\n",
-    "dataset_name_or_filename = \"hotpotqa\"\n",
-    "dataset = load_qa_dataset(dataset_name_or_filename)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7351ab8f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "context_provider_name = \"cognee\"\n",
-    "context_provider = qa_context_providers[context_provider_name]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9346115b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "random.seed(42)\n",
-    "instances = dataset if not num_samples else random.sample(dataset, num_samples)\n",
-    "\n",
-    "out_path = \"out\"\n",
-    "if not Path(out_path).exists():\n",
-    "    Path(out_path).mkdir()\n",
-    "contexts_filename = out_path / Path(\n",
-    "    f\"contexts_{dataset_name_or_filename.split('.')[0]}_{context_provider_name}.json\"\n",
-    ")\n",
-    "\n",
-    "answers = []\n",
-    "for instance in tqdm(instances, desc=\"Getting answers\"):\n",
-    "    answer = await answer_qa_instance(instance, context_provider, contexts_filename)\n",
-    "    answers.append(answer)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "1e7d872d",
-   "metadata": {},
-   "source": [
-    "#### Define Metrics for Evaluation and Calculate Score\n",
-    "**Options**: \n",
-    "- **Correctness**: Is the actual output factually correct based on the expected output?\n",
-    "- **Comprehensiveness**: How much detail does the answer provide to cover all aspects and details of the question?\n",
-    "- **Diversity**: How varied and rich is the answer in providing different perspectives and insights on the question?\n",
-    "- **Empowerment**: How well does the answer help the reader understand and make informed judgements about the topic?\n",
-    "- **Directness**: How specifically and clearly does the answer address the question?\n",
-    "- **F1 Score**: the harmonic mean of the precision and recall, using word-level Exact Match\n",
-    "- **EM Score**: the rate at which the predicted strings exactly match their references, ignoring white spaces and capitalization."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "c81e2b46",
-   "metadata": {},
-   "source": [
-    "##### Calculate `\"Correctness\"`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ae728344",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metric_name_list = [\"Correctness\"]\n",
-    "eval_metrics = get_metrics(metric_name_list)\n",
-    "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "764aac6d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "Correctness = statistics.mean(\n",
-    "    [result.metrics_data[0].score for result in eval_results.test_results]\n",
-    ")\n",
-    "print(Correctness)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "6d3bbdc5",
-   "metadata": {},
-   "source": [
-    "##### Calculating `\"Comprehensiveness\"`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9793ef78",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metric_name_list = [\"Comprehensiveness\"]\n",
-    "eval_metrics = get_metrics(metric_name_list)\n",
-    "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9add448a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "Comprehensiveness = statistics.mean(\n",
-    "    [result.metrics_data[0].score for result in eval_results.test_results]\n",
-    ")\n",
-    "print(Comprehensiveness)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "bce2fa25",
-   "metadata": {},
-   "source": [
-    "##### Calculating `\"Diversity\"`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f60a179e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metric_name_list = [\"Diversity\"]\n",
-    "eval_metrics = get_metrics(metric_name_list)\n",
-    "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7ccbd0ab",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "Diversity = statistics.mean([result.metrics_data[0].score for result in eval_results.test_results])\n",
-    "print(Diversity)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "191cab63",
-   "metadata": {},
-   "source": [
-    "##### Calculating`\"Empowerment\"`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "66bec0bf",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metric_name_list = [\"Empowerment\"]\n",
-    "eval_metrics = get_metrics(metric_name_list)\n",
-    "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1b043a8f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "Empowerment = statistics.mean(\n",
-    "    [result.metrics_data[0].score for result in eval_results.test_results]\n",
-    ")\n",
-    "print(Empowerment)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "2cac3be9",
-   "metadata": {},
-   "source": [
-    "##### Calculating `\"Directness\"`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "adaa17c0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metric_name_list = [\"Directness\"]\n",
-    "eval_metrics = get_metrics(metric_name_list)\n",
-    "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3a8f97c9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "Directness = statistics.mean([result.metrics_data[0].score for result in eval_results.test_results])\n",
-    "print(Directness)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "1ad6feb8",
-   "metadata": {},
-   "source": [
-    "##### Calculating `\"F1\"`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "bdc48259",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metric_name_list = [\"F1\"]\n",
-    "eval_metrics = get_metrics(metric_name_list)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c43c17c8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8bfcc46d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "F1_score = statistics.mean([result.metrics_data[0].score for result in eval_results.test_results])\n",
-    "print(F1_score)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "2583f948",
-   "metadata": {},
-   "source": [
-    "##### Calculating `\"EM\"`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "90a8f630",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "metric_name_list = [\"EM\"]\n",
-    "eval_metrics = get_metrics(metric_name_list)\n",
-    "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8d1b1ea1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "EM = statistics.mean([result.metrics_data[0].score for result in eval_results.test_results])\n",
-    "print(EM)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "288ab570",
-   "metadata": {},
-   "source": [
-    "# Give us a star if you like it!\n",
-    "https://github.com/topoteretes/cognee"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "cognee-c83GrcRT-py3.11",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.10"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

notebooks/llama_index_cognee_integration.ipynb

@@ -3,7 +3,9 @@
   {
    "cell_type": "markdown",
    "metadata": {},
-   "source": "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1EpokQ8Y_5jIJ7HdixZms81Oqgh2sp7-E?usp=sharing)"
+   "source": [
+    "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1EpokQ8Y_5jIJ7HdixZms81Oqgh2sp7-E?usp=sharing)"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -57,7 +59,9 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": "!pip install llama-index-graph-rag-cognee==0.1.3"
+   "source": [
+    "!pip install llama-index-graph-rag-cognee==0.1.3"
+   ]
   },
   {
    "cell_type": "code",
@@ -192,7 +196,9 @@
   {
    "cell_type": "markdown",
    "metadata": {},
-   "source": "### Answer prompt based on RAG approach:"
+   "source": [
+    "### Answer prompt based on RAG approach:"
+   ]
   },
   {
    "cell_type": "code",
@@ -210,7 +216,9 @@
   {
    "cell_type": "markdown",
    "metadata": {},
-   "source": "In conclusion, the results demonstrate a significant advantage of the knowledge graph-based approach (Graphrag) over the RAG approach. Graphrag successfully identified all the mentioned individuals across multiple documents, showcasing its ability to aggregate and infer information from a global context. In contrast, the RAG approach was limited to identifying individuals within a single document due to its chunking-based processing constraints. This highlights Graphrag's superior capability in comprehensively resolving queries that span across a broader corpus of interconnected data."
+   "source": [
+    "In conclusion, the results demonstrate a significant advantage of the knowledge graph-based approach (Graphrag) over the RAG approach. Graphrag successfully identified all the mentioned individuals across multiple documents, showcasing its ability to aggregate and infer information from a global context. In contrast, the RAG approach was limited to identifying individuals within a single document due to its chunking-based processing constraints. This highlights Graphrag's superior capability in comprehensively resolving queries that span across a broader corpus of interconnected data."
+   ]
   },
   {
    "cell_type": "markdown",
@@ -271,7 +279,8 @@
  ],
  "metadata": {
   "language_info": {
-   "name": "python"
+   "name": "python",
+   "version": "3.12.9"
   }
  },
  "nbformat": 4,

pyproject.toml

@@ -20,34 +20,32 @@ classifiers = [
 ]
 dependencies = [
     "openai>=1.59.4,<2",
-    "python-dotenv==1.0.1",
+    "python-dotenv>=1.0.1",
     "pydantic==2.10.5",
     "pydantic-settings>=2.2.1,<3",
     "typing_extensions==4.12.2",
     "nltk==3.9.1",
     "numpy>=1.26.4, <=2.1",
-    "pandas==2.2.3",
+    "pandas>=2.2.2",
     # Note: New s3fs and boto3 versions don't work well together
     # Always use comaptible fixed versions of these two dependencies
-    "s3fs==2025.3.2",
-    "boto3==1.37.1",
-    "botocore>=1.35.54,<2",
+    "s3fs[boto3]==2025.3.2",
     "sqlalchemy==2.0.39",
     "aiosqlite>=0.20.0,<0.21",
     "tiktoken<=0.9.0",
     "litellm>=1.57.4",
     "instructor==1.7.2",
-    "langfuse>=2.32.0,<3",
-    "filetype>=1.2.0,<2",
-    "aiohttp>=3.11.14,<4",
-    "aiofiles>=23.2.1,<24",
+    "langfuse>=2.32.0",
+    "filetype>=1.2.0",
+    "aiohttp>=3.11.14",
+    "aiofiles>=23.2.1",
     "owlready2>=0.47,<0.48",
     "graphistry>=0.33.5,<0.34",
     "pypdf>=4.1.0,<6.0.0",
     "jinja2>=3.1.3,<4",
     "matplotlib>=3.8.3,<4",
-    "networkx>=3.2.1,<4",
-    "lancedb==0.16.0",
+    "networkx>=3.4.2,<4",
+    "lancedb==0.21.0",
     "alembic>=1.13.3,<2",
     "pre-commit>=4.0.1,<5",
     "scikit-learn>=1.6.1,<2",
@@ -58,6 +56,8 @@ dependencies = [
     "dlt[sqlalchemy]>=1.9.0,<2",
     "sentry-sdk[fastapi]>=2.9.0,<3",
     "structlog>=25.2.0,<26",
+    "onnxruntime<=1.21.1",
+    "pylance==0.22.0",
 ]
 
 [project.optional-dependencies]
@@ -123,7 +123,6 @@ dev = [
     "ruff>=0.9.2,<1.0.0",
     "tweepy==4.14.0",
     "gitpython>=3.1.43,<4",
-    "pylance==0.19.2",
     "mkdocs-material>=9.5.42,<10",
     "mkdocs-minify-plugin>=0.8.0,<0.9",
     "mkdocstrings[python]>=0.26.2,<0.27",

uv.lock

@@ -26,6 +26,11 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/95/67/026598918f92145156f2feb7957f57daefda20375cc2ac1a0692a9b8010b/aiobotocore-2.21.1-py3-none-any.whl", hash = "sha256:bd7c49a6d6f8a3d9444b0a94417c8da13813b5c7eec1c4f0ec2db7e8ce8f23e7", size = 78313 },
 ]
 
+[package.optional-dependencies]
+boto3 = [
+    { name = "boto3" },
+]
+
 [[package]]
 name = "aiofiles"
 version = "23.2.1"
@@ -861,8 +866,6 @@ dependencies = [
     { name = "aiohttp" },
     { name = "aiosqlite" },
     { name = "alembic" },
-    { name = "boto3" },
-    { name = "botocore" },
     { name = "dlt", extra = ["sqlalchemy"] },
     { name = "fastapi" },
     { name = "fastapi-users", extra = ["sqlalchemy"] },
@@ -878,16 +881,18 @@ dependencies = [
     { name = "networkx" },
     { name = "nltk" },
     { name = "numpy" },
+    { name = "onnxruntime" },
     { name = "openai" },
     { name = "owlready2" },
     { name = "pandas" },
     { name = "pre-commit" },
     { name = "pydantic" },
     { name = "pydantic-settings" },
+    { name = "pylance" },
     { name = "pypdf" },
     { name = "python-dotenv" },
     { name = "python-multipart" },
-    { name = "s3fs" },
+    { name = "s3fs", extra = ["boto3"] },
     { name = "scikit-learn" },
     { name = "sentry-sdk", extra = ["fastapi"] },
     { name = "sqlalchemy" },
@@ -909,8 +914,7 @@ chromadb = [
     { name = "pypika" },
 ]
 codegraph = [
-    { name = "fastembed", version = "0.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*'" },
-    { name = "fastembed", version = "0.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
+    { name = "fastembed", marker = "python_full_version < '3.13'" },
     { name = "transformers" },
     { name = "tree-sitter" },
     { name = "tree-sitter-python" },
@@ -928,7 +932,6 @@ dev = [
     { name = "mkdocstrings", extra = ["python"] },
     { name = "mypy" },
     { name = "notebook" },
-    { name = "pylance" },
     { name = "pylint" },
     { name = "pytest" },
     { name = "pytest-asyncio" },
@@ -1004,14 +1007,12 @@ weaviate = [
 
 [package.metadata]
 requires-dist = [
-    { name = "aiofiles", specifier = ">=23.2.1,<24" },
-    { name = "aiohttp", specifier = ">=3.11.14,<4" },
+    { name = "aiofiles", specifier = ">=23.2.1" },
+    { name = "aiohttp", specifier = ">=3.11.14" },
     { name = "aiosqlite", specifier = ">=0.20.0,<0.21" },
     { name = "alembic", specifier = ">=1.13.3,<2" },
     { name = "anthropic", marker = "extra == 'anthropic'", specifier = ">=0.26.1,<0.27" },
     { name = "asyncpg", marker = "extra == 'postgres'", specifier = "==0.30.0" },
-    { name = "boto3", specifier = "==1.37.1" },
-    { name = "botocore", specifier = ">=1.35.54,<2" },
     { name = "chromadb", marker = "extra == 'chromadb'", specifier = ">=0.3.0,<0.7" },
     { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.3.2,<8" },
     { name = "debugpy", marker = "extra == 'dev'", specifier = "==1.8.9" },
@@ -1022,7 +1023,7 @@ requires-dist = [
     { name = "fastapi", specifier = "==0.115.7" },
     { name = "fastapi-users", extras = ["sqlalchemy"], specifier = "==14.0.1" },
     { name = "fastembed", marker = "python_full_version < '3.13' and extra == 'codegraph'", specifier = "<=0.6.0" },
-    { name = "filetype", specifier = ">=1.2.0,<2" },
+    { name = "filetype", specifier = ">=1.2.0" },
     { name = "gdown", marker = "extra == 'evals'", specifier = ">=5.2.0,<6" },
     { name = "gitpython", marker = "extra == 'dev'", specifier = ">=3.1.43,<4" },
     { name = "google-generativeai", marker = "extra == 'gemini'", specifier = ">=0.8.4,<0.9" },
@@ -1033,9 +1034,9 @@ requires-dist = [
     { name = "instructor", specifier = "==1.7.2" },
     { name = "jinja2", specifier = ">=3.1.3,<4" },
     { name = "kuzu", marker = "extra == 'kuzu'", specifier = "==0.8.2" },
-    { name = "lancedb", specifier = "==0.16.0" },
+    { name = "lancedb", specifier = "==0.21.0" },
     { name = "langchain-text-splitters", marker = "extra == 'langchain'", specifier = "==0.3.2" },
-    { name = "langfuse", specifier = ">=2.32.0,<3" },
+    { name = "langfuse", specifier = ">=2.32.0" },
     { name = "langsmith", marker = "extra == 'langchain'", specifier = "==0.2.3" },
     { name = "limits", specifier = ">=4.4.1,<5" },
     { name = "litellm", specifier = ">=1.57.4" },
@@ -1047,14 +1048,15 @@ requires-dist = [
     { name = "mkdocstrings", extras = ["python"], marker = "extra == 'dev'", specifier = ">=0.26.2,<0.27" },
     { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.7.1,<2" },
     { name = "neo4j", marker = "extra == 'neo4j'", specifier = ">=5.20.0,<6" },
-    { name = "networkx", specifier = ">=3.2.1,<4" },
+    { name = "networkx", specifier = ">=3.4.2,<4" },
     { name = "nltk", specifier = "==3.9.1" },
     { name = "notebook", marker = "extra == 'dev'", specifier = ">=7.1.0,<8" },
     { name = "notebook", marker = "extra == 'notebook'", specifier = ">=7.1.0,<8" },
     { name = "numpy", specifier = ">=1.26.4,<=2.1" },
+    { name = "onnxruntime", specifier = "<=1.21.1" },
     { name = "openai", specifier = ">=1.59.4,<2" },
     { name = "owlready2", specifier = ">=0.47,<0.48" },
-    { name = "pandas", specifier = "==2.2.3" },
+    { name = "pandas", specifier = ">=2.2.2" },
     { name = "pgvector", marker = "extra == 'postgres'", specifier = ">=0.3.5,<0.4" },
     { name = "plotly", marker = "extra == 'evals'", specifier = ">=6.0.0,<7" },
     { name = "posthog", marker = "extra == 'posthog'", specifier = ">=3.5.0,<4" },
@@ -1062,7 +1064,7 @@ requires-dist = [
     { name = "psycopg2", marker = "extra == 'postgres'", specifier = ">=2.9.10,<3" },
     { name = "pydantic", specifier = "==2.10.5" },
     { name = "pydantic-settings", specifier = ">=2.2.1,<3" },
-    { name = "pylance", marker = "extra == 'dev'", specifier = "==0.19.2" },
+    { name = "pylance", specifier = "==0.22.0" },
     { name = "pylint", marker = "extra == 'dev'", specifier = ">=3.0.3,<4" },
     { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=2.5.0,<3" },
     { name = "pypdf", specifier = ">=4.1.0,<6.0.0" },
@@ -1071,12 +1073,12 @@ requires-dist = [
     { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.4.0,<8" },
     { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.21.1,<0.22" },
     { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.1.1" },
-    { name = "python-dotenv", specifier = "==1.0.1" },
+    { name = "python-dotenv", specifier = ">=1.0.1" },
     { name = "python-multipart", specifier = "==0.0.20" },
     { name = "qasync", marker = "extra == 'gui'", specifier = ">=0.27.1,<0.28" },
     { name = "qdrant-client", marker = "extra == 'qdrant'", specifier = ">=1.9.0,<2" },
     { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.2,<1.0.0" },
-    { name = "s3fs", specifier = "==2025.3.2" },
+    { name = "s3fs", extras = ["boto3"], specifier = "==2025.3.2" },
     { name = "scikit-learn", specifier = ">=1.6.1,<2" },
     { name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.9.0,<3" },
     { name = "sqlalchemy", specifier = "==2.0.39" },
@@ -1808,51 +1810,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a6/08/9968963c1fb8c34627b7f1fbcdfe9438540f87dc7c9bfb59bb4fd19a4ecf/fastapi_users_db_sqlalchemy-7.0.0-py3-none-any.whl", hash = "sha256:5fceac018e7cfa69efc70834dd3035b3de7988eb4274154a0dbe8b14f5aa001e", size = 6891 },
 ]
 
-[[package]]
-name = "fastembed"
-version = "0.4.2"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version >= '3.12.4' and python_full_version < '3.13'",
-    "python_full_version >= '3.12' and python_full_version < '3.12.4'",
-]
-dependencies = [
-    { name = "huggingface-hub", marker = "python_full_version == '3.12.*'" },
-    { name = "loguru", marker = "python_full_version == '3.12.*'" },
-    { name = "mmh3", marker = "python_full_version == '3.12.*'" },
-    { name = "numpy", marker = "python_full_version == '3.12.*'" },
-    { name = "onnx", marker = "python_full_version == '3.12.*'" },
-    { name = "onnxruntime", marker = "python_full_version == '3.12.*'" },
-    { name = "pillow", marker = "python_full_version == '3.12.*'" },
-    { name = "py-rust-stemmers", marker = "python_full_version == '3.12.*'" },
-    { name = "requests", marker = "python_full_version == '3.12.*'" },
-    { name = "tokenizers", marker = "python_full_version == '3.12.*'" },
-    { name = "tqdm", marker = "python_full_version == '3.12.*'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/02/f0/8d935e8ea2408ccd34405ccb61bbcc340633597841e56e621cc5c4768405/fastembed-0.4.2.tar.gz", hash = "sha256:4065344ed795c2c860f31953ab9ead91291ce77952a3f7823ae64e3c8dc1a21c", size = 42084 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c2/cd/115de8124008e6bb9aa0b871409a65e6e737192c0732b58ea2fd458777fc/fastembed-0.4.2-py3-none-any.whl", hash = "sha256:b72a5bde7261fa01a4dd74c234f97eff6f6e869307aadaed1c6e37dc9fc80a0a", size = 67043 },
-]
-
 [[package]]
 name = "fastembed"
 version = "0.6.0"
 source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version == '3.11.*'",
-    "python_full_version < '3.11'",
-]
 dependencies = [
-    { name = "huggingface-hub", marker = "python_full_version < '3.12'" },
-    { name = "loguru", marker = "python_full_version < '3.12'" },
-    { name = "mmh3", marker = "python_full_version < '3.12'" },
-    { name = "numpy", marker = "python_full_version < '3.12'" },
-    { name = "onnxruntime", marker = "python_full_version < '3.12'" },
-    { name = "pillow", marker = "python_full_version < '3.12'" },
-    { name = "py-rust-stemmers", marker = "python_full_version < '3.12'" },
-    { name = "requests", marker = "python_full_version < '3.12'" },
-    { name = "tokenizers", marker = "python_full_version < '3.12'" },
-    { name = "tqdm", marker = "python_full_version < '3.12'" },
+    { name = "huggingface-hub", marker = "python_full_version < '3.13'" },
+    { name = "loguru", marker = "python_full_version < '3.13'" },
+    { name = "mmh3", marker = "python_full_version < '3.13'" },
+    { name = "numpy", marker = "python_full_version < '3.13'" },
+    { name = "onnxruntime", marker = "python_full_version < '3.13'" },
+    { name = "pillow", marker = "python_full_version < '3.13'" },
+    { name = "py-rust-stemmers", marker = "python_full_version < '3.13'" },
+    { name = "requests", marker = "python_full_version < '3.13'" },
+    { name = "tokenizers", marker = "python_full_version < '3.13'" },
+    { name = "tqdm", marker = "python_full_version < '3.13'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/c6/f4/036a656c605f63dc25f11284f60f69900a54a19c513e1ae60d21d6977e75/fastembed-0.6.0.tar.gz", hash = "sha256:5c9ead25f23449535b07243bbe1f370b820dcc77ec2931e61674e3fe7ff24733", size = 50731 }
 wheels = [
@@ -3384,24 +3356,24 @@ wheels = [
 
 [[package]]
 name = "lancedb"
-version = "0.16.0"
+version = "0.21.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "deprecation" },
-    { name = "nest-asyncio" },
     { name = "overrides" },
     { name = "packaging" },
+    { name = "pyarrow" },
     { name = "pydantic" },
-    { name = "pylance" },
     { name = "tqdm" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/73/a6/a9c5077e8f0391b637c5a6e7a2adfc43418e1949a931149bd14f10c87871/lancedb-0.16.0-cp38-abi3-macosx_10_15_x86_64.whl", hash = "sha256:3521c53a116bfbb054318a35b2297cd01d57e1db500de4ba3cc7fad6c4add98c", size = 24138660 },
-    { url = "https://files.pythonhosted.org/packages/b9/25/25f8494e164ec83212002018053271865ae06ca4cc5976e3987515fdeb2a/lancedb-0.16.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:8e0968e6b7a3611437dc4c4f468aafb4e665aa315ee0b201e589ea1fa433b5b6", size = 22616630 },
-    { url = "https://files.pythonhosted.org/packages/ac/8d/e3afab600d9437db4d4f8eb1dac9b493778ec0d37e374313b39fa146917e/lancedb-0.16.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e466b2c734f8dde5d037b082cff1ccd9a913e7b87ecc73efa7d921ed1aa6ded", size = 27874448 },
-    { url = "https://files.pythonhosted.org/packages/f6/0d/39692b1e43e02860b0fc827b5cf2d61b377de8096adc44f0ce2b4cd42d97/lancedb-0.16.0-cp38-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:84980ccd4a170a5a07f83c85f90841e6995a05dc92cadedcb806401bc60f832b", size = 25920255 },
-    { url = "https://files.pythonhosted.org/packages/d7/9e/e027daeec233623fa1a0bee9479f40ef819f9af0dd873ed93062f78b72e0/lancedb-0.16.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:7135cd4733c3f7bcff7bc5f017bb79dda8e23dc3530476e0b91a4cb3ffef9e2e", size = 27354357 },
-    { url = "https://files.pythonhosted.org/packages/fc/88/65a3540a0e66f5a1636b0fea715c475e733ca4394452c1095dadfcfe7708/lancedb-0.16.0-cp38-abi3-win_amd64.whl", hash = "sha256:357a4cd2d7c5bf9677f84f7aa408d6c98c9d9012b3110979b126d40d30530e12", size = 25073405 },
+    { url = "https://files.pythonhosted.org/packages/3b/b0/68b605ce247002b505daeaf371d922cc1fea1a00a7b4d8ecd750ae6172f9/lancedb-0.21.0-cp39-abi3-macosx_10_15_x86_64.whl", hash = "sha256:a6df4a58177f5f30208cbdd639a1abb07bc8224c069d8d8c17ce90735f48f0f3", size = 29769342 },
+    { url = "https://files.pythonhosted.org/packages/9c/e0/48eae648799526f68d32bfee00581f967deb864ee2aa47faab25716fea06/lancedb-0.21.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:270bc89f87a797d06f2dccb6b39ad961d5f957270347de9dac80dc096163ae11", size = 27761929 },
+    { url = "https://files.pythonhosted.org/packages/51/77/1fd2d7fd73ee19f118917c0445528a16ad5a12c95099eba70808c9470b8e/lancedb-0.21.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdce9b7cc2252dd48e4149df8d3d07fd3d6672c309c5ea23ab40abd775467a26", size = 31239576 },
+    { url = "https://files.pythonhosted.org/packages/c1/39/a288bcfdac7575373c66045344aae78e975de799627fba9722c797b4f45d/lancedb-0.21.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:470d22547025ece6b27581c8fa29d2cc0a772003f1a90a51ba6bec09102c8778", size = 33747301 },
+    { url = "https://files.pythonhosted.org/packages/6a/be/809217d5fad83604883167bceca875933b9fa9036f558451e8d126c7bab5/lancedb-0.21.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6333f731c0b04acd4f180a63cca0b2e4e412cb4425baa24cce748ad717b19b5d", size = 30940024 },
+    { url = "https://files.pythonhosted.org/packages/3b/b0/977134e883b6ab0873d82e9f14ec410eee52153fc6733eeec06873e346b7/lancedb-0.21.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ea89a7542f84fe1111d056259df6b0434a1bb4ec128850720f9159b3bd5a4089", size = 33231475 },
+    { url = "https://files.pythonhosted.org/packages/b2/dd/d5a264d27cfa45258e2b4422ef42e9ebb47336e0c51f1d4f05d31d63deda/lancedb-0.21.0-cp39-abi3-win_amd64.whl", hash = "sha256:18a48ad207234680afdf5a7677050ac6b51fda055483ac3959d8e6b5eb89e457", size = 30873192 },
 ]
 
 [[package]]
@@ -4796,36 +4768,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/33/3f/164de150e983b3a16e8bf3d4355625e51a357e7b3b1deebe9cc1f7cb9af8/ollama-0.4.8-py3-none-any.whl", hash = "sha256:04312af2c5e72449aaebac4a2776f52ef010877c554103419d3f36066fe8af4c", size = 13325 },
 ]
 
-[[package]]
-name = "onnx"
-version = "1.17.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "numpy", marker = "python_full_version == '3.12.*'" },
-    { name = "protobuf", marker = "python_full_version == '3.12.*'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9a/54/0e385c26bf230d223810a9c7d06628d954008a5e5e4b73ee26ef02327282/onnx-1.17.0.tar.gz", hash = "sha256:48ca1a91ff73c1d5e3ea2eef20ae5d0e709bb8a2355ed798ffc2169753013fd3", size = 12165120 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/2e/29/57053ba7787788ac75efb095cfc1ae290436b6d3a26754693cd7ed1b4fac/onnx-1.17.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:38b5df0eb22012198cdcee527cc5f917f09cce1f88a69248aaca22bd78a7f023", size = 16645616 },
-    { url = "https://files.pythonhosted.org/packages/75/0d/831807a18db2a5e8f7813848c59272b904a4ef3939fe4d1288cbce9ea735/onnx-1.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d545335cb49d4d8c47cc803d3a805deb7ad5d9094dc67657d66e568610a36d7d", size = 15908420 },
-    { url = "https://files.pythonhosted.org/packages/dd/5b/c4f95dbe652d14aeba9afaceb177e9ffc48ac3c03048dd3f872f26f07e34/onnx-1.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3193a3672fc60f1a18c0f4c93ac81b761bc72fd8a6c2035fa79ff5969f07713e", size = 16046244 },
-    { url = "https://files.pythonhosted.org/packages/08/a9/c1f218085043dccc6311460239e253fa6957cf12ee4b0a56b82014938d0b/onnx-1.17.0-cp310-cp310-win32.whl", hash = "sha256:0141c2ce806c474b667b7e4499164227ef594584da432fd5613ec17c1855e311", size = 14423516 },
-    { url = "https://files.pythonhosted.org/packages/0e/d3/d26ebf590a65686dde6b27fef32493026c5be9e42083340d947395f93405/onnx-1.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:dfd777d95c158437fda6b34758f0877d15b89cbe9ff45affbedc519b35345cf9", size = 14528496 },
-    { url = "https://files.pythonhosted.org/packages/e5/a9/8d1b1d53aec70df53e0f57e9f9fcf47004276539e29230c3d5f1f50719ba/onnx-1.17.0-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:d6fc3a03fc0129b8b6ac03f03bc894431ffd77c7d79ec023d0afd667b4d35869", size = 16647991 },
-    { url = "https://files.pythonhosted.org/packages/7b/e3/cc80110e5996ca61878f7b4c73c7a286cd88918ff35eacb60dc75ab11ef5/onnx-1.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01a4b63d4e1d8ec3e2f069e7b798b2955810aa434f7361f01bc8ca08d69cce4", size = 15908949 },
-    { url = "https://files.pythonhosted.org/packages/b1/2f/91092557ed478e323a2b4471e2081fdf88d1dd52ae988ceaf7db4e4506ff/onnx-1.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a183c6178be001bf398260e5ac2c927dc43e7746e8638d6c05c20e321f8c949", size = 16048190 },
-    { url = "https://files.pythonhosted.org/packages/ac/59/9ea23fc22d0bb853133f363e6248e31bcbc6c1c90543a3938c00412ac02a/onnx-1.17.0-cp311-cp311-win32.whl", hash = "sha256:081ec43a8b950171767d99075b6b92553901fa429d4bc5eb3ad66b36ef5dbe3a", size = 14424299 },
-    { url = "https://files.pythonhosted.org/packages/51/a5/19b0dfcb567b62e7adf1a21b08b23224f0c2d13842aee4d0abc6f07f9cf5/onnx-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:95c03e38671785036bb704c30cd2e150825f6ab4763df3a4f1d249da48525957", size = 14529142 },
-    { url = "https://files.pythonhosted.org/packages/b4/dd/c416a11a28847fafb0db1bf43381979a0f522eb9107b831058fde012dd56/onnx-1.17.0-cp312-cp312-macosx_12_0_universal2.whl", hash = "sha256:0e906e6a83437de05f8139ea7eaf366bf287f44ae5cc44b2850a30e296421f2f", size = 16651271 },
-    { url = "https://files.pythonhosted.org/packages/f0/6c/f040652277f514ecd81b7251841f96caa5538365af7df07f86c6018cda2b/onnx-1.17.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d955ba2939878a520a97614bcf2e79c1df71b29203e8ced478fa78c9a9c63c2", size = 15907522 },
-    { url = "https://files.pythonhosted.org/packages/3d/7c/67f4952d1b56b3f74a154b97d0dd0630d525923b354db117d04823b8b49b/onnx-1.17.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f3fb5cc4e2898ac5312a7dc03a65133dd2abf9a5e520e69afb880a7251ec97a", size = 16046307 },
-    { url = "https://files.pythonhosted.org/packages/ae/20/6da11042d2ab870dfb4ce4a6b52354d7651b6b4112038b6d2229ab9904c4/onnx-1.17.0-cp312-cp312-win32.whl", hash = "sha256:317870fca3349d19325a4b7d1b5628f6de3811e9710b1e3665c68b073d0e68d7", size = 14424235 },
-    { url = "https://files.pythonhosted.org/packages/35/55/c4d11bee1fdb0c4bd84b4e3562ff811a19b63266816870ae1f95567aa6e1/onnx-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:659b8232d627a5460d74fd3c96947ae83db6d03f035ac633e20cd69cfa029227", size = 14530453 },
-]
-
 [[package]]
 name = "onnxruntime"
-version = "1.19.2"
+version = "1.21.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "coloredlogs" },
@@ -4836,21 +4781,24 @@ dependencies = [
     { name = "sympy" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/39/18/272d3d7406909141d3c9943796e3e97cafa53f4342d9231c0cfd8cb05702/onnxruntime-1.19.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:84fa57369c06cadd3c2a538ae2a26d76d583e7c34bdecd5769d71ca5c0fc750e", size = 16776408 },
-    { url = "https://files.pythonhosted.org/packages/d8/d3/eb93f4ae511cfc725d0c69e07008800f8ac018de19ea1e497b306f174ccc/onnxruntime-1.19.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdc471a66df0c1cdef774accef69e9f2ca168c851ab5e4f2f3341512c7ef4666", size = 11491779 },
-    { url = "https://files.pythonhosted.org/packages/ca/4b/ce5958074abe4b6e8d1da9c10e443e01a681558a9ec17e5cc7619438e094/onnxruntime-1.19.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e3a4ce906105d99ebbe817f536d50a91ed8a4d1592553f49b3c23c4be2560ae6", size = 13170428 },
-    { url = "https://files.pythonhosted.org/packages/ce/0f/6df82dfe02467d12adbaa05c2bd17519c29c7df531ed600231f0c741ad22/onnxruntime-1.19.2-cp310-cp310-win32.whl", hash = "sha256:4b3d723cc154c8ddeb9f6d0a8c0d6243774c6b5930847cc83170bfe4678fafb3", size = 9591305 },
-    { url = "https://files.pythonhosted.org/packages/3c/d8/68b63dc86b502169d017a86fe8bc718f4b0055ef1f6895bfaddd04f2eead/onnxruntime-1.19.2-cp310-cp310-win_amd64.whl", hash = "sha256:17ed7382d2c58d4b7354fb2b301ff30b9bf308a1c7eac9546449cd122d21cae5", size = 11084902 },
-    { url = "https://files.pythonhosted.org/packages/f0/ff/77bee5df55f034ee81d2e1bc58b2b8511b9c54f06ce6566cb562c5d95aa5/onnxruntime-1.19.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:d863e8acdc7232d705d49e41087e10b274c42f09e259016a46f32c34e06dc4fd", size = 16779187 },
-    { url = "https://files.pythonhosted.org/packages/f3/78/e29f5fb76e0f6524f3520e8e5b9d53282784b45d14068c5112db9f712b0a/onnxruntime-1.19.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1dfe4f660a71b31caa81fc298a25f9612815215a47b286236e61d540350d7b6", size = 11496005 },
-    { url = "https://files.pythonhosted.org/packages/60/ce/be4152da5c1030ab5a159a4a792ed9abad6ba498d79ef0aeba593ff7b5bf/onnxruntime-1.19.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a36511dc07c5c964b916697e42e366fa43c48cdb3d3503578d78cef30417cb84", size = 13167809 },
-    { url = "https://files.pythonhosted.org/packages/e1/00/9740a074eb0e0a21ff13a2c4f32aecc5b21110b2c9b9177d8ac132b66e2d/onnxruntime-1.19.2-cp311-cp311-win32.whl", hash = "sha256:50cbb8dc69d6befad4746a69760e5b00cc3ff0a59c6c3fb27f8afa20e2cab7e7", size = 9591445 },
-    { url = "https://files.pythonhosted.org/packages/1e/f5/9d995a685f97508b3254f17015b4a78641b0625e79480a7aed7a7a105d7c/onnxruntime-1.19.2-cp311-cp311-win_amd64.whl", hash = "sha256:1c3e5d415b78337fa0b1b75291e9ea9fb2a4c1f148eb5811e7212fed02cfffa8", size = 11085695 },
-    { url = "https://files.pythonhosted.org/packages/f2/a5/2a02687a88fc8a2507bef65876c90e96b9f8de5ba1f810acbf67c140fc67/onnxruntime-1.19.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:68e7051bef9cfefcbb858d2d2646536829894d72a4130c24019219442b1dd2ed", size = 16790434 },
-    { url = "https://files.pythonhosted.org/packages/47/64/da42254ec14452cad2cdd4cf407094841c0a378c0d08944e9a36172197e9/onnxruntime-1.19.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d2d366fbcc205ce68a8a3bde2185fd15c604d9645888703785b61ef174265168", size = 11486028 },
-    { url = "https://files.pythonhosted.org/packages/b2/92/3574f6836f33b1b25f272293e72538c38451b12c2d9aa08630bb6bc0f057/onnxruntime-1.19.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:477b93df4db467e9cbf34051662a4b27c18e131fa1836e05974eae0d6e4cf29b", size = 13175054 },
-    { url = "https://files.pythonhosted.org/packages/ff/c9/8c37e413a830cac7f7dc094fffbd0c998c8bcb66a6f0b0a3201a49bc742b/onnxruntime-1.19.2-cp312-cp312-win32.whl", hash = "sha256:9a174073dc5608fad05f7cf7f320b52e8035e73d80b0a23c80f840e5a97c0147", size = 9592681 },
-    { url = "https://files.pythonhosted.org/packages/44/c0/59768846533786a82cafb38d8d2f900ad666bc91f0ae634774d286fa3c47/onnxruntime-1.19.2-cp312-cp312-win_amd64.whl", hash = "sha256:190103273ea4507638ffc31d66a980594b237874b65379e273125150eb044857", size = 11086411 },
+    { url = "https://files.pythonhosted.org/packages/06/72/09d8f206402cd91805828354ad1d7473b1bace60fc54a11971012906d9b7/onnxruntime-1.21.1-cp310-cp310-macosx_13_0_universal2.whl", hash = "sha256:daedb5d33d8963062a25f4a3c788262074587f685a19478ef759a911b4b12c25", size = 33639134 },
+    { url = "https://files.pythonhosted.org/packages/1f/66/31384dc7beea89f21ec7d1582c1b50e9d047d505db38f32cf49693fad1b4/onnxruntime-1.21.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a402f9bda0b1cc791d9cf31d23c471e8189a55369b49ef2b9d0854eb11d22c4", size = 14162243 },
+    { url = "https://files.pythonhosted.org/packages/a9/fb/76597b77785b2012317ffdd817101ccfab784e2c125645d002c4c9cd377b/onnxruntime-1.21.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15656a2d0126f4f66295381e39c8812a6d845ccb1bb1f7bf6dd0a46d7d602e7f", size = 16000498 },
+    { url = "https://files.pythonhosted.org/packages/91/83/c7287845f22f2e1d37a54b5997e9589b6931e264cc0f16250d1706eadf79/onnxruntime-1.21.1-cp310-cp310-win_amd64.whl", hash = "sha256:79bbedfd1263065532967a2132fb365a27ffe5f7ed962e16fec55cca741f72aa", size = 12300918 },
+    { url = "https://files.pythonhosted.org/packages/70/ba/13c46c22fb52d8fea53575da163399a7d75fe61223aba685370f047a0882/onnxruntime-1.21.1-cp311-cp311-macosx_13_0_universal2.whl", hash = "sha256:8bee9b5ba7b88ae7bfccb4f97bbe1b4bae801b0fb05d686b28a722cb27c89931", size = 33643424 },
+    { url = "https://files.pythonhosted.org/packages/18/4f/68985138c507b6ad34061aa4f330b8fbd30b0c5c299be53f0c829420528e/onnxruntime-1.21.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b6a29a1767b92d543091349f5397a1c7619eaca746cd1bc47f8b4ec5a9f1a6c", size = 14162437 },
+    { url = "https://files.pythonhosted.org/packages/0f/76/7dfa4b63f95a17eaf881c9c464feaa59a25bbfb578db204fc22d522b5199/onnxruntime-1.21.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:982dcc04a6688e1af9e3da1d4ef2bdeb11417cf3f8dde81f8f721043c1919a4f", size = 16002403 },
+    { url = "https://files.pythonhosted.org/packages/80/85/397406e758d6c30fb6d0d0152041c6b9ee835c3584765837ce54230c8bc9/onnxruntime-1.21.1-cp311-cp311-win_amd64.whl", hash = "sha256:2b6052c04b9125319293abb9bdcce40e806db3e097f15b82242d4cd72d81fd0c", size = 12301824 },
+    { url = "https://files.pythonhosted.org/packages/a5/42/274438bbc259439fa1606d0d6d2eef4171cdbd2d7a1c3b249b4ba440424b/onnxruntime-1.21.1-cp312-cp312-macosx_13_0_universal2.whl", hash = "sha256:f615c05869a523a94d0a4de1f0936d0199a473cf104d630fc26174bebd5759bd", size = 33658457 },
+    { url = "https://files.pythonhosted.org/packages/9c/93/76f629d4f22571b0b3a29a9d375204faae2bd2b07d557043b56df5848779/onnxruntime-1.21.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79dfb1f47386c4edd115b21015354b2f05f5566c40c98606251f15a64add3cbe", size = 14164881 },
+    { url = "https://files.pythonhosted.org/packages/1b/86/75cbaa4058758fa8ef912dfebba2d5a4e4fd6738615c15b6a2262d076198/onnxruntime-1.21.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2742935d6610fe0f58e1995018d9db7e8239d0201d9ebbdb7964a61386b5390a", size = 16019966 },
+    { url = "https://files.pythonhosted.org/packages/5f/9d/fb8895b2cb38c9965d4b4e0a9aa1398f3e3f16c4acb75cf3b61689780a65/onnxruntime-1.21.1-cp312-cp312-win_amd64.whl", hash = "sha256:a7afdb3fcb162f5536225e13c2b245018068964b1d0eee05303ea6823ca6785e", size = 12302925 },
+    { url = "https://files.pythonhosted.org/packages/6d/7e/8445eb44ba9fe0ce0bc77c4b569d79f7e3efd6da2dd87c5a04347e6c134e/onnxruntime-1.21.1-cp313-cp313-macosx_13_0_universal2.whl", hash = "sha256:ed4f9771233a92edcab9f11f537702371d450fe6cd79a727b672d37b9dab0cde", size = 33658643 },
+    { url = "https://files.pythonhosted.org/packages/ce/46/9c4026d302f1c7e8427bf9fa3da2d7526d9c5200242bde6adee7928ef1c9/onnxruntime-1.21.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bc100fd1f4f95258e7d0f7068ec69dec2a47cc693f745eec9cf4561ee8d952a", size = 14165205 },
+    { url = "https://files.pythonhosted.org/packages/44/b2/4e4c6b5c03be752d74cb20937961c76f53fe87a9760d5b7345629d35bb31/onnxruntime-1.21.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0fea0d2b98eecf4bebe01f7ce9a265a5d72b3050e9098063bfe65fa2b0633a8e", size = 16019529 },
+    { url = "https://files.pythonhosted.org/packages/ec/1d/afca646af339cc6735f3fb7fafb9ca94b578c5b6a0ebd63a312468767bdb/onnxruntime-1.21.1-cp313-cp313-win_amd64.whl", hash = "sha256:da606061b9ed1b05b63a37be38c2014679a3e725903f58036ffd626df45c0e47", size = 12303603 },
+    { url = "https://files.pythonhosted.org/packages/a5/12/a01e38c9a6b8d7c28e04d9eb83ad9143d568b961474ba49f0f18a3eeec82/onnxruntime-1.21.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94674315d40d521952bfc28007ce9b6728e87753e1f18d243c8cd953f25903b8", size = 14176329 },
+    { url = "https://files.pythonhosted.org/packages/3a/72/5ff85c540fd6a465610ce47e4cee8fccb472952fc1d589112f51ae2520a5/onnxruntime-1.21.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c9e4571ff5b2a5d377d414bc85cd9450ba233a9a92f766493874f1093976453", size = 15990556 },
 ]
 
 [[package]]
@@ -5886,19 +5834,19 @@ crypto = [
 
 [[package]]
 name = "pylance"
-version = "0.19.2"
+version = "0.22.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "numpy" },
     { name = "pyarrow" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/11/47/5eb617889ad15fc07bceb419fdc9e42c67de878f2d081e30637af9e5e735/pylance-0.19.2-cp39-abi3-macosx_10_15_x86_64.whl", hash = "sha256:659b2ba45b7c905a2873bb36e9b4a6ec4634690723d45af0b469a502acacf5eb", size = 28776069 },
-    { url = "https://files.pythonhosted.org/packages/8b/f5/6c2f04869747cb382f0f561362d354e132c2adb9b299aa28f28bb1847209/pylance-0.19.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:6a15b8b09e015feb11307ff63ef0742f9e120100e17476b1091d3db543c19bdf", size = 26733988 },
-    { url = "https://files.pythonhosted.org/packages/34/35/722388309957c2dbab64a6901152c794dce01978e859a35989f95678af81/pylance-0.19.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf1732ff03acceacc6793f6b209357a757ce3cfd5a94369a81b3d15e8e425f9a", size = 30626161 },
-    { url = "https://files.pythonhosted.org/packages/e9/d2/38d523007824ef975fca496022e2583df833226f128399d4daf0a046896d/pylance-0.19.2-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:c3be7883ae860c39186f41798cd752b93298450cc09488108f2aa738aa352f0e", size = 29172165 },
-    { url = "https://files.pythonhosted.org/packages/90/48/f41b76e478651afee95f3a558db1d8ea9b85e1b31511b4340ea6224491fe/pylance-0.19.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:48a618dfc493932b49a8c1f50ad031e39c4d505d30c968d6467da1a03720a523", size = 30527798 },
-    { url = "https://files.pythonhosted.org/packages/dd/24/126b0e37b30bf567ed31151f8adf009ee5d8eb68c613bc33e5f523f0d710/pylance-0.19.2-cp39-abi3-win_amd64.whl", hash = "sha256:70d47d94521fc973460c8d765c3960db79a1f676aab658434693ab3e5a7112c1", size = 28575346 },
+    { url = "https://files.pythonhosted.org/packages/9e/22/ad54cfda2bbf7e217de0cc131e0ed2c879af7728d6331903e44dee8f8dfb/pylance-0.22.0-cp39-abi3-macosx_10_15_x86_64.whl", hash = "sha256:2c0bb6bf7320e500f0f5948e5b23e4d70d9c84bba15a2db2e877be9637c4dc0c", size = 34412591 },
+    { url = "https://files.pythonhosted.org/packages/28/e4/54603e4ad6341240e507cd3b490e34cd0663610b59d5e6ba5a9d317cd421/pylance-0.22.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:341a8cbac762c1f446a05a1513dab1b7930f433a8331b08b0b89a975f3864f6a", size = 31889815 },
+    { url = "https://files.pythonhosted.org/packages/ce/ed/bf2b5e480d9ec620f261d9b5293ebb494934b42f30af62973df476ef8b7d/pylance-0.22.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29848127701f2188b331ad8399036f1fb79bacf5102fd030bfe9fd30cb02cf5b", size = 38929145 },
+    { url = "https://files.pythonhosted.org/packages/bf/6c/069ef2823c7366c529297493719e8a3f6b16a19bbaf42e6f5010307157ec/pylance-0.22.0-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:cd4cc3dd3772600092685282db8cd4c21eaa68f458445b3107bd01b43afb8f11", size = 36272984 },
+    { url = "https://files.pythonhosted.org/packages/50/ff/61e10792edab999d0cc0c89a409446d28bee0f47e157ebc5587c0f8fb332/pylance-0.22.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8999e73ce180c977f91bb4629578d742b1e86fcf53e7d27b14d6d219395c17cd", size = 38322607 },
+    { url = "https://files.pythonhosted.org/packages/61/f0/b62b14630af78d468ff7b15cc21576910edbd73114795b49907b39df2841/pylance-0.22.0-cp39-abi3-win_amd64.whl", hash = "sha256:848f1a74dab14dc14bf05569404977cfcba9a95a44e513e5a3b32f1221bfa00f", size = 34216608 },
 ]
 
 [[package]]
@@ -6889,6 +6837,11 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/66/e1/4db0388df5655de92ce5f9b60d2bef220a58dde130e0453e5433c579986e/s3fs-2025.3.2-py3-none-any.whl", hash = "sha256:81eae3f37b4b04bcc08845d7bcc607c6ca45878813ef7e6a28d77b2688417130", size = 30485 },
 ]
 
+[package.optional-dependencies]
+boto3 = [
+    { name = "aiobotocore", extra = ["boto3"] },
+]
+
 [[package]]
 name = "s3transfer"
 version = "0.11.3"