From 6bb0f3d8f2a306daabc3384c2d0c38cc8f3cb60f Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Wed, 4 Dec 2024 15:53:25 +0100 Subject: [PATCH] feat: Add ability to store single data instance in multiple datasets Added ability to store single data instance in multiple datasets Feature COG-505 --- cognee/tasks/ingestion/ingest_data_with_metadata.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cognee/tasks/ingestion/ingest_data_with_metadata.py b/cognee/tasks/ingestion/ingest_data_with_metadata.py index 0a238b548..7c0b840eb 100644 --- a/cognee/tasks/ingestion/ingest_data_with_metadata.py +++ b/cognee/tasks/ingestion/ingest_data_with_metadata.py @@ -4,6 +4,7 @@ import dlt import cognee.modules.ingestion as ingestion from cognee.infrastructure.databases.relational import get_relational_engine from cognee.modules.data.methods import create_dataset +from cognee.modules.data.models.DatasetData import DatasetData from cognee.modules.data.operations.delete_metadata import delete_metadata from cognee.modules.users.models import User from cognee.modules.users.permissions.methods import give_permission_on_document @@ -89,7 +90,17 @@ async def ingest_data_with_metadata(data: Any, dataset_name: str, user: User): mime_type = file_metadata["mime_type"] ) + + + # Check if data is already in dataset + dataset_data = ( + await session.execute(select(DatasetData).filter(DatasetData.data_id == data_id, + DatasetData.dataset_id == dataset.id)) + ).scalar_one_or_none() + # If data is not present in dataset add it + if dataset_data is None: dataset.data.append(data_point) + await session.commit() await write_metadata(data_item, data_point.id, file_metadata)