remove delimiter for naive parser

2024-03-04 17:07:17 +08:00 · 2024-03-04 17:07:17 +08:00 · 4a053b8c08
commit 4a053b8c08
parent 0cc0a91cbf
1 changed files with 1 additions and 1 deletions
--- a/rag/nlp/init.py
+++ b/rag/nlp/init.py
@ -246,7 +246,7 @@ def naive_merge(sections, chunk_token_num=128, delimiter="\n。；！？"):
            tk_nums[-1] += tnum

    for sec, pos in sections:
-        add_chunk(sec[s: e], pos)
+        add_chunk(sec, pos)
        continue
        s, e = 0, 1
        while e < len(sec):