refactor: make prefererred_loaders easier to define on user facing api
This commit is contained in:
parent
738759bc5b
commit
344fbbdc29
1 changed files with 10 additions and 1 deletions
|
|
@ -23,7 +23,7 @@ async def add(
|
||||||
vector_db_config: dict = None,
|
vector_db_config: dict = None,
|
||||||
graph_db_config: dict = None,
|
graph_db_config: dict = None,
|
||||||
dataset_id: Optional[UUID] = None,
|
dataset_id: Optional[UUID] = None,
|
||||||
preferred_loaders: dict[str, dict[str, Any]] = None,
|
preferred_loaders: Optional[List[Union[str, dict[str, dict[str, Any]]]]] = None,
|
||||||
incremental_loading: bool = True,
|
incremental_loading: bool = True,
|
||||||
data_per_batch: Optional[int] = 20,
|
data_per_batch: Optional[int] = 20,
|
||||||
):
|
):
|
||||||
|
|
@ -164,6 +164,15 @@ async def add(
|
||||||
- TAVILY_API_KEY: YOUR_TAVILY_API_KEY
|
- TAVILY_API_KEY: YOUR_TAVILY_API_KEY
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
if preferred_loaders:
|
||||||
|
transformed = {}
|
||||||
|
for item in preferred_loaders:
|
||||||
|
if isinstance(item, dict):
|
||||||
|
transformed.update(item)
|
||||||
|
else:
|
||||||
|
transformed[item] = {}
|
||||||
|
preferred_loaders = transformed
|
||||||
|
|
||||||
tasks = [
|
tasks = [
|
||||||
Task(resolve_data_directories, include_subdirectories=True),
|
Task(resolve_data_directories, include_subdirectories=True),
|
||||||
Task(
|
Task(
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue