From 7100a4994a5dd2b3343c99a136c436f549eba5c8 Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Thu, 12 Dec 2024 17:04:49 +0100 Subject: [PATCH] feat: Add resolving of directories as task for the add pipeline Add resolving of directories as task for the add pipeline Feature COG-656 --- cognee/api/v1/add/add_v2.py | 4 +--- cognee/tasks/ingestion/resolve_data_directories.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cognee/api/v1/add/add_v2.py b/cognee/api/v1/add/add_v2.py index 9d07f31e4..637c4a187 100644 --- a/cognee/api/v1/add/add_v2.py +++ b/cognee/api/v1/add/add_v2.py @@ -13,10 +13,8 @@ async def add(data: Union[BinaryIO, list[BinaryIO], str, list[str]], dataset_nam if user is None: user = await get_default_user() - # Resolve all directories from data to files - data = resolve_data_directories(data) - tasks = [ + Task(resolve_data_directories), Task(ingest_data_with_metadata, dataset_name, user) ] diff --git a/cognee/tasks/ingestion/resolve_data_directories.py b/cognee/tasks/ingestion/resolve_data_directories.py index 52ab27111..599a9b342 100644 --- a/cognee/tasks/ingestion/resolve_data_directories.py +++ b/cognee/tasks/ingestion/resolve_data_directories.py @@ -1,7 +1,7 @@ import os from typing import List, Union, BinaryIO -def resolve_data_directories(data: Union[BinaryIO, List[BinaryIO], str, List[str]], include_subdirectories: bool = True): +async def resolve_data_directories(data: Union[BinaryIO, List[BinaryIO], str, List[str]], include_subdirectories: bool = True): """ Resolves directories by replacing them with their contained files.