refactor: remove filestream arg from LoaderEngine.load_file(...)

This commit is contained in:
Daulet Amirkhanov 2025-10-17 17:33:25 +01:00
parent 6a693d319a
commit 50aa8aac11
2 changed files with 9 additions and 10 deletions

View file

@ -105,7 +105,6 @@ class LoaderEngine:
async def load_file(
self,
file_path: str,
file_stream: Optional[Any],
preferred_loaders: Optional[List[str]] = None,
**kwargs,
):

View file

@ -48,17 +48,17 @@ async def data_item_to_text_file(
await pull_from_s3(data_item_path, temp_file)
temp_file.flush() # Data needs to be saved to local storage
loader = get_loader_engine()
return await loader.load_file(
temp_file.name, None, preferred_loaders
), loader.get_loader(temp_file.name, preferred_loaders)
return await loader.load_file(temp_file.name, preferred_loaders), loader.get_loader(
temp_file.name, preferred_loaders
)
# data is local file path
elif parsed_url.scheme == "file":
if settings.accept_local_file_path:
loader = get_loader_engine()
return await loader.load_file(
data_item_path, None, preferred_loaders
), loader.get_loader(data_item_path, preferred_loaders)
return await loader.load_file(data_item_path, preferred_loaders), loader.get_loader(
data_item_path, preferred_loaders
)
else:
raise IngestionError(message="Local files are not accepted.")
@ -69,9 +69,9 @@ async def data_item_to_text_file(
# Handle both Unix absolute paths (/path) and Windows absolute paths (C:\path)
if settings.accept_local_file_path:
loader = get_loader_engine()
return await loader.load_file(
data_item_path, None, preferred_loaders
), loader.get_loader(data_item_path, preferred_loaders)
return await loader.load_file(data_item_path, preferred_loaders), loader.get_loader(
data_item_path, preferred_loaders
)
else:
raise IngestionError(message="Local files are not accepted.")