refactor: remove filestream arg from LoaderEngine.load_file(...)
This commit is contained in:
parent
6a693d319a
commit
50aa8aac11
2 changed files with 9 additions and 10 deletions
|
|
@ -105,7 +105,6 @@ class LoaderEngine:
|
||||||
async def load_file(
|
async def load_file(
|
||||||
self,
|
self,
|
||||||
file_path: str,
|
file_path: str,
|
||||||
file_stream: Optional[Any],
|
|
||||||
preferred_loaders: Optional[List[str]] = None,
|
preferred_loaders: Optional[List[str]] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
|
|
||||||
|
|
@ -48,17 +48,17 @@ async def data_item_to_text_file(
|
||||||
await pull_from_s3(data_item_path, temp_file)
|
await pull_from_s3(data_item_path, temp_file)
|
||||||
temp_file.flush() # Data needs to be saved to local storage
|
temp_file.flush() # Data needs to be saved to local storage
|
||||||
loader = get_loader_engine()
|
loader = get_loader_engine()
|
||||||
return await loader.load_file(
|
return await loader.load_file(temp_file.name, preferred_loaders), loader.get_loader(
|
||||||
temp_file.name, None, preferred_loaders
|
temp_file.name, preferred_loaders
|
||||||
), loader.get_loader(temp_file.name, preferred_loaders)
|
)
|
||||||
|
|
||||||
# data is local file path
|
# data is local file path
|
||||||
elif parsed_url.scheme == "file":
|
elif parsed_url.scheme == "file":
|
||||||
if settings.accept_local_file_path:
|
if settings.accept_local_file_path:
|
||||||
loader = get_loader_engine()
|
loader = get_loader_engine()
|
||||||
return await loader.load_file(
|
return await loader.load_file(data_item_path, preferred_loaders), loader.get_loader(
|
||||||
data_item_path, None, preferred_loaders
|
data_item_path, preferred_loaders
|
||||||
), loader.get_loader(data_item_path, preferred_loaders)
|
)
|
||||||
else:
|
else:
|
||||||
raise IngestionError(message="Local files are not accepted.")
|
raise IngestionError(message="Local files are not accepted.")
|
||||||
|
|
||||||
|
|
@ -69,9 +69,9 @@ async def data_item_to_text_file(
|
||||||
# Handle both Unix absolute paths (/path) and Windows absolute paths (C:\path)
|
# Handle both Unix absolute paths (/path) and Windows absolute paths (C:\path)
|
||||||
if settings.accept_local_file_path:
|
if settings.accept_local_file_path:
|
||||||
loader = get_loader_engine()
|
loader = get_loader_engine()
|
||||||
return await loader.load_file(
|
return await loader.load_file(data_item_path, preferred_loaders), loader.get_loader(
|
||||||
data_item_path, None, preferred_loaders
|
data_item_path, preferred_loaders
|
||||||
), loader.get_loader(data_item_path, preferred_loaders)
|
)
|
||||||
else:
|
else:
|
||||||
raise IngestionError(message="Local files are not accepted.")
|
raise IngestionError(message="Local files are not accepted.")
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue