gmakstutis/ragflow
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove delimiter for naive parser

Browse source
This commit is contained in:
Kevin Hu 2024-03-04 17:07:17 +08:00
parent 0cc0a91cbf
commit 4a053b8c08
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
rag/nlp/__init__.py
View file

@ -246,7 +246,7 @@ def naive_merge(sections, chunk_token_num=128, delimiter="\n。；！？"):
tk_nums[-1] += tnum tk_nums[-1] += tnum
for sec, pos in sections: for sec, pos in sections:
add_chunk(sec[s: e], pos) add_chunk(sec, pos)
continue continue
s, e = 0, 1 s, e = 0, 1
while e < len(sec): while e < len(sec):