refactor: remove filestream arg from LoaderEngine.load_file(...)
This commit is contained in:
parent
6a693d319a
commit
50aa8aac11
2 changed files with 9 additions and 10 deletions
|
|
@ -105,7 +105,6 @@ class LoaderEngine:
|
|||
async def load_file(
|
||||
self,
|
||||
file_path: str,
|
||||
file_stream: Optional[Any],
|
||||
preferred_loaders: Optional[List[str]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
|
|
|
|||
|
|
@ -48,17 +48,17 @@ async def data_item_to_text_file(
|
|||
await pull_from_s3(data_item_path, temp_file)
|
||||
temp_file.flush() # Data needs to be saved to local storage
|
||||
loader = get_loader_engine()
|
||||
return await loader.load_file(
|
||||
temp_file.name, None, preferred_loaders
|
||||
), loader.get_loader(temp_file.name, preferred_loaders)
|
||||
return await loader.load_file(temp_file.name, preferred_loaders), loader.get_loader(
|
||||
temp_file.name, preferred_loaders
|
||||
)
|
||||
|
||||
# data is local file path
|
||||
elif parsed_url.scheme == "file":
|
||||
if settings.accept_local_file_path:
|
||||
loader = get_loader_engine()
|
||||
return await loader.load_file(
|
||||
data_item_path, None, preferred_loaders
|
||||
), loader.get_loader(data_item_path, preferred_loaders)
|
||||
return await loader.load_file(data_item_path, preferred_loaders), loader.get_loader(
|
||||
data_item_path, preferred_loaders
|
||||
)
|
||||
else:
|
||||
raise IngestionError(message="Local files are not accepted.")
|
||||
|
||||
|
|
@ -69,9 +69,9 @@ async def data_item_to_text_file(
|
|||
# Handle both Unix absolute paths (/path) and Windows absolute paths (C:\path)
|
||||
if settings.accept_local_file_path:
|
||||
loader = get_loader_engine()
|
||||
return await loader.load_file(
|
||||
data_item_path, None, preferred_loaders
|
||||
), loader.get_loader(data_item_path, preferred_loaders)
|
||||
return await loader.load_file(data_item_path, preferred_loaders), loader.get_loader(
|
||||
data_item_path, preferred_loaders
|
||||
)
|
||||
else:
|
||||
raise IngestionError(message="Local files are not accepted.")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue