From 4a053b8c088b30a22d4f13268b7e2dc4a272f0f9 Mon Sep 17 00:00:00 2001
From: Kevin Hu <kevinhu.sh@gmail.com>
Date: Mon, 4 Mar 2024 17:07:17 +0800
Subject: [PATCH] remove delimiter for naive parser

---
 rag/nlp/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py
index e8cdf481e..f2c9f436f 100644
--- a/rag/nlp/__init__.py
+++ b/rag/nlp/__init__.py
@@ -246,7 +246,7 @@ def naive_merge(sections, chunk_token_num=128, delimiter="\n。；！？"):
             tk_nums[-1] += tnum
 
     for sec, pos in sections:
-        add_chunk(sec[s: e], pos)
+        add_chunk(sec, pos)
         continue
         s, e = 0, 1
         while e < len(sec):