gmakstutis/LightRAG
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

easier version: detect chunking_func result is coroutine or not

Browse source
This commit is contained in:
Tong Da 2025-11-10 20:49:50 +08:00 • committed by yangdx
parent 7740500693
commit 5016025453
1 changed files with 11 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
lightrag/lightrag.py
View file

@ -1,6 +1,5 @@
from __future__ import annotations from __future__ import annotations
from inspect import iscoroutinefunction
import traceback import traceback
import asyncio import asyncio
import configparser import configparser
@ -1780,24 +1779,17 @@ class LightRAG:
content = content_data["content"] content = content_data["content"]
# Generate chunks from document # Generate chunks from document
if iscoroutinefunction(self.chunking_func): chunks = self.chunking_func(
chunks = await self.chunking_func( self.tokenizer,
self.tokenizer, content,
content, split_by_character,
split_by_character, split_by_character_only,
split_by_character_only, self.chunk_overlap_token_size,
self.chunk_overlap_token_size, self.chunk_token_size,
self.chunk_token_size, )
) # 判断chunks是否是异步异步函数的返回
else: if asyncio.iscoroutine(chunks):
chunks = self.chunking_func( chunks = await chunks
self.tokenizer,
content,
split_by_character,
split_by_character_only,
self.chunk_overlap_token_size,
self.chunk_token_size,
)
chunks: dict[str, Any] = { chunks: dict[str, Any] = {
compute_mdhash_id(dp["content"], prefix="chunk-"): { compute_mdhash_id(dp["content"], prefix="chunk-"): {
**dp, **dp,