From 871055b0fc5fc98ecc5fc5bdfc90f7012320ef09 Mon Sep 17 00:00:00 2001 From: buua436 <66937541+buua436@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:17:52 +0800 Subject: [PATCH 01/15] Feat:support API for generating knowledge graph and raptor (#11229) ### What problem does this PR solve? issue: [#11195](https://github.com/infiniflow/ragflow/issues/11195) change: support API for generating knowledge graph and raptor ### Type of change - [x] New Feature (non-breaking change which adds functionality) - [x] Documentation Update --- api/apps/sdk/dataset.py | 159 +++++++++++++++++- docs/references/http_api_reference.md | 231 ++++++++++++++++++++++++++ 2 files changed, 387 insertions(+), 3 deletions(-) diff --git a/api/apps/sdk/dataset.py b/api/apps/sdk/dataset.py index 8a315ce69..de5434de7 100644 --- a/api/apps/sdk/dataset.py +++ b/api/apps/sdk/dataset.py @@ -21,10 +21,11 @@ import json from flask import request from peewee import OperationalError from api.db.db_models import File -from api.db.services.document_service import DocumentService +from api.db.services.document_service import DocumentService, queue_raptor_o_graphrag_tasks from api.db.services.file2document_service import File2DocumentService from api.db.services.file_service import FileService from api.db.services.knowledgebase_service import KnowledgebaseService +from api.db.services.task_service import GRAPH_RAPTOR_FAKE_DOC_ID, TaskService from api.db.services.user_service import TenantService from common.constants import RetCode, FileSource, StatusEnum from api.utils.api_utils import ( @@ -118,7 +119,6 @@ def create(tenant_id): req, err = validate_and_parse_json_request(request, CreateDatasetReq) if err is not None: return get_error_argument_result(err) - req = KnowledgebaseService.create_with_name( name = req.pop("name", None), tenant_id = tenant_id, @@ -144,7 +144,6 @@ def create(tenant_id): ok, k = KnowledgebaseService.get_by_id(req["id"]) if not ok: return get_error_data_result(message="Dataset created failed") - response_data = remap_dictionary_keys(k.to_dict()) return get_result(data=response_data) except Exception as e: @@ -532,3 +531,157 @@ def delete_knowledge_graph(tenant_id, dataset_id): search.index_name(kb.tenant_id), dataset_id) return get_result(data=True) + + +@manager.route("/datasets//run_graphrag", methods=["POST"]) # noqa: F821 +@token_required +def run_graphrag(tenant_id,dataset_id): + if not dataset_id: + return get_error_data_result(message='Lack of "Dataset ID"') + if not KnowledgebaseService.accessible(dataset_id, tenant_id): + return get_result( + data=False, + message='No authorization.', + code=RetCode.AUTHENTICATION_ERROR + ) + + ok, kb = KnowledgebaseService.get_by_id(dataset_id) + if not ok: + return get_error_data_result(message="Invalid Dataset ID") + + task_id = kb.graphrag_task_id + if task_id: + ok, task = TaskService.get_by_id(task_id) + if not ok: + logging.warning(f"A valid GraphRAG task id is expected for Dataset {dataset_id}") + + if task and task.progress not in [-1, 1]: + return get_error_data_result(message=f"Task {task_id} in progress with status {task.progress}. A Graph Task is already running.") + + documents, _ = DocumentService.get_by_kb_id( + kb_id=dataset_id, + page_number=0, + items_per_page=0, + orderby="create_time", + desc=False, + keywords="", + run_status=[], + types=[], + suffix=[], + ) + if not documents: + return get_error_data_result(message=f"No documents in Dataset {dataset_id}") + + sample_document = documents[0] + document_ids = [document["id"] for document in documents] + + task_id = queue_raptor_o_graphrag_tasks(sample_doc_id=sample_document, ty="graphrag", priority=0, fake_doc_id=GRAPH_RAPTOR_FAKE_DOC_ID, doc_ids=list(document_ids)) + + if not KnowledgebaseService.update_by_id(kb.id, {"graphrag_task_id": task_id}): + logging.warning(f"Cannot save graphrag_task_id for Dataset {dataset_id}") + + return get_result(data={"graphrag_task_id": task_id}) + + +@manager.route("/datasets//trace_graphrag", methods=["GET"]) # noqa: F821 +@token_required +def trace_graphrag(tenant_id,dataset_id): + if not dataset_id: + return get_error_data_result(message='Lack of "Dataset ID"') + if not KnowledgebaseService.accessible(dataset_id, tenant_id): + return get_result( + data=False, + message='No authorization.', + code=RetCode.AUTHENTICATION_ERROR + ) + + ok, kb = KnowledgebaseService.get_by_id(dataset_id) + if not ok: + return get_error_data_result(message="Invalid Dataset ID") + + task_id = kb.graphrag_task_id + if not task_id: + return get_result(data={}) + + ok, task = TaskService.get_by_id(task_id) + if not ok: + return get_result(data={}) + + return get_result(data=task.to_dict()) + + +@manager.route("/datasets//run_raptor", methods=["POST"]) # noqa: F821 +@token_required +def run_raptor(tenant_id,dataset_id): + if not dataset_id: + return get_error_data_result(message='Lack of "Dataset ID"') + if not KnowledgebaseService.accessible(dataset_id, tenant_id): + return get_result( + data=False, + message='No authorization.', + code=RetCode.AUTHENTICATION_ERROR + ) + + ok, kb = KnowledgebaseService.get_by_id(dataset_id) + if not ok: + return get_error_data_result(message="Invalid Dataset ID") + + task_id = kb.raptor_task_id + if task_id: + ok, task = TaskService.get_by_id(task_id) + if not ok: + logging.warning(f"A valid RAPTOR task id is expected for Dataset {dataset_id}") + + if task and task.progress not in [-1, 1]: + return get_error_data_result(message=f"Task {task_id} in progress with status {task.progress}. A RAPTOR Task is already running.") + + documents, _ = DocumentService.get_by_kb_id( + kb_id=dataset_id, + page_number=0, + items_per_page=0, + orderby="create_time", + desc=False, + keywords="", + run_status=[], + types=[], + suffix=[], + ) + if not documents: + return get_error_data_result(message=f"No documents in Dataset {dataset_id}") + + sample_document = documents[0] + document_ids = [document["id"] for document in documents] + + task_id = queue_raptor_o_graphrag_tasks(sample_doc_id=sample_document, ty="raptor", priority=0, fake_doc_id=GRAPH_RAPTOR_FAKE_DOC_ID, doc_ids=list(document_ids)) + + if not KnowledgebaseService.update_by_id(kb.id, {"raptor_task_id": task_id}): + logging.warning(f"Cannot save raptor_task_id for Dataset {dataset_id}") + + return get_result(data={"raptor_task_id": task_id}) + + +@manager.route("/datasets//trace_raptor", methods=["GET"]) # noqa: F821 +@token_required +def trace_raptor(tenant_id,dataset_id): + if not dataset_id: + return get_error_data_result(message='Lack of "Dataset ID"') + + if not KnowledgebaseService.accessible(dataset_id, tenant_id): + return get_result( + data=False, + message='No authorization.', + code=RetCode.AUTHENTICATION_ERROR + ) + ok, kb = KnowledgebaseService.get_by_id(dataset_id) + if not ok: + return get_error_data_result(message="Invalid Dataset ID") + + task_id = kb.raptor_task_id + if not task_id: + return get_result(data={}) + + ok, task = TaskService.get_by_id(task_id) + if not ok: + return get_error_data_result(message="RAPTOR Task Not Found or Error Occurred") + + return get_result(data=task.to_dict()) \ No newline at end of file diff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.md index f2b86a735..481614d13 100644 --- a/docs/references/http_api_reference.md +++ b/docs/references/http_api_reference.md @@ -974,6 +974,237 @@ Failure: --- +### Construct knowledge graph + +**POST** `/api/v1/datasets/{dataset_id}/run_graphrag` + +Constructs a knowledge graph from a specified dataset. + +#### Request + +- Method: POST +- URL: `/api/v1/datasets/{dataset_id}/run_graphrag` +- Headers: + - `'Authorization: Bearer '` + +##### Request example + +```bash +curl --request POST \ + --url http://{address}/api/v1/datasets/{dataset_id}/run_graphrag \ + --header 'Authorization: Bearer ' +``` + +##### Request parameters + +- `dataset_id`: (*Path parameter*) + The ID of the target dataset. + +#### Response + +Success: + +```json +{ + "code":0, + "data":{ + "graphrag_task_id":"e498de54bfbb11f0ba028f704583b57b" + } +} +``` + +Failure: + +```json +{ + "code": 102, + "message": "Invalid Dataset ID" +} +``` + +--- + +### Get knowledge graph construction status + +**GET** `/api/v1/datasets/{dataset_id}/trace_graphrag` + +Retrieves the knowledge graph construction status for a specified dataset. + +#### Request + +- Method: GET +- URL: `/api/v1/datasets/{dataset_id}/trace_graphrag` +- Headers: + - `'Authorization: Bearer '` + +##### Request example + +```bash +curl --request GET \ + --url http://{address}/api/v1/datasets/{dataset_id}/trace_graphrag \ + --header 'Authorization: Bearer ' +``` + +##### Request parameters + +- `dataset_id`: (*Path parameter*) + The ID of the target dataset. + +#### Response + +Success: + +```json +{ + "code":0, + "data":{ + "begin_at":"Wed, 12 Nov 2025 19:36:56 GMT", + "chunk_ids":"", + "create_date":"Wed, 12 Nov 2025 19:36:56 GMT", + "create_time":1762947416350, + "digest":"39e43572e3dcd84f", + "doc_id":"44661c10bde211f0bc93c164a47ffc40", + "from_page":100000000, + "id":"e498de54bfbb11f0ba028f704583b57b", + "priority":0, + "process_duration":2.45419, + "progress":1.0, + "progress_msg":"19:36:56 created task graphrag\n19:36:57 Task has been received.\n19:36:58 [GraphRAG] doc:083661febe2411f0bc79456921e5745f has no available chunks, skip generation.\n19:36:58 [GraphRAG] build_subgraph doc:44661c10bde211f0bc93c164a47ffc40 start (chunks=1, timeout=10000000000s)\n19:36:58 Graph already contains 44661c10bde211f0bc93c164a47ffc40\n19:36:58 [GraphRAG] build_subgraph doc:44661c10bde211f0bc93c164a47ffc40 empty\n19:36:58 [GraphRAG] kb:33137ed0bde211f0bc93c164a47ffc40 no subgraphs generated successfully, end.\n19:36:58 Knowledge Graph done (0.72s)","retry_count":1, + "task_type":"graphrag", + "to_page":100000000, + "update_date":"Wed, 12 Nov 2025 19:36:58 GMT", + "update_time":1762947418454 + } +} +``` + +Failure: + +```json +{ + "code": 102, + "message": "Invalid Dataset ID" +} +``` + +--- + +### Construct RAPTOR + +**POST** `/api/v1/datasets/{dataset_id}/run_raptor` + +Construct a RAPTOR from a specified dataset. + +#### Request + +- Method: POST +- URL: `/api/v1/datasets/{dataset_id}/run_raptor` +- Headers: + - `'Authorization: Bearer '` + +##### Request example + +```bash +curl --request POST \ + --url http://{address}/api/v1/datasets/{dataset_id}/run_raptor \ + --header 'Authorization: Bearer ' +``` + +##### Request parameters + +- `dataset_id`: (*Path parameter*) + The ID of the target dataset. + +#### Response + +Success: + +```json +{ + "code":0, + "data":{ + "raptor_task_id":"50d3c31cbfbd11f0ba028f704583b57b" + } +} +``` + +Failure: + +```json +{ + "code": 102, + "message": "Invalid Dataset ID" +} +``` + +--- + +### Get RAPTOR construction status + +**GET** `/api/v1/datasets/{dataset_id}/trace_raptor` + +Retrieves the RAPTOR construction status for a specified dataset. + +#### Request + +- Method: GET +- URL: `/api/v1/datasets/{dataset_id}/trace_raptor` +- Headers: + - `'Authorization: Bearer '` + +##### Request example + +```bash +curl --request GET \ + --url http://{address}/api/v1/datasets/{dataset_id}/trace_raptor \ + --header 'Authorization: Bearer ' +``` + +##### Request parameters + +- `dataset_id`: (*Path parameter*) + The ID of the target dataset. + +#### Response + +Success: + +```json +{ + "code":0, + "data":{ + "begin_at":"Wed, 12 Nov 2025 19:47:07 GMT", + "chunk_ids":"", + "create_date":"Wed, 12 Nov 2025 19:47:07 GMT", + "create_time":1762948027427, + "digest":"8b279a6248cb8fc6", + "doc_id":"44661c10bde211f0bc93c164a47ffc40", + "from_page":100000000, + "id":"50d3c31cbfbd11f0ba028f704583b57b", + "priority":0, + "process_duration":0.948244, + "progress":1.0, + "progress_msg":"19:47:07 created task raptor\n19:47:07 Task has been received.\n19:47:07 Processing...\n19:47:07 Processing...\n19:47:07 Indexing done (0.01s).\n19:47:07 Task done (0.29s)", + "retry_count":1, + "task_type":"raptor", + "to_page":100000000, + "update_date":"Wed, 12 Nov 2025 19:47:07 GMT", + "update_time":1762948027948 + } +} +``` + +Failure: + +```json +{ + "code": 102, + "message": "Invalid Dataset ID" +} +``` + +--- + ## FILE MANAGEMENT WITHIN DATASET --- From bfc84ba95bff47eb839076eb01077add64400205 Mon Sep 17 00:00:00 2001 From: Liu An Date: Thu, 13 Nov 2025 15:18:32 +0800 Subject: [PATCH 02/15] Test: handle duplicate names by appending "(1)" (#11244) ### What problem does this PR solve? - Updated tests to reflect new behavior of handling duplicate dataset names - Instead of returning an error, the system now appends "(1)" to duplicate names - This problem was introduced by PR #10960 ### Type of change - [x] Testcase update --- .../test_dataset_mangement/test_create_dataset.py | 15 ++++++++------- .../test_dataset_mangement/test_create_dataset.py | 14 ++++++-------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/test/testcases/test_http_api/test_dataset_mangement/test_create_dataset.py b/test/testcases/test_http_api/test_dataset_mangement/test_create_dataset.py index 507570fba..559b41f3c 100644 --- a/test/testcases/test_http_api/test_dataset_mangement/test_create_dataset.py +++ b/test/testcases/test_http_api/test_dataset_mangement/test_create_dataset.py @@ -16,14 +16,15 @@ from concurrent.futures import ThreadPoolExecutor, as_completed import pytest -from common import create_dataset -from configs import DATASET_NAME_LIMIT, INVALID_API_TOKEN +from configs import DATASET_NAME_LIMIT, DEFAULT_PARSER_CONFIG, INVALID_API_TOKEN from hypothesis import example, given, settings from libs.auth import RAGFlowHttpApiAuth from utils import encode_avatar from utils.file_utils import create_image_file from utils.hypothesis_utils import valid_names -from configs import DEFAULT_PARSER_CONFIG + +from common import create_dataset + @pytest.mark.usefixtures("clear_datasets") class TestAuthorization: @@ -125,8 +126,8 @@ class TestDatasetCreate: assert res["code"] == 0, res res = create_dataset(HttpApiAuth, payload) - assert res["code"] == 103, res - assert res["message"] == f"Dataset name '{name}' already exists", res + assert res["code"] == 0, res + assert res["data"]["name"] == name + "(1)", res @pytest.mark.p3 def test_name_case_insensitive(self, HttpApiAuth): @@ -137,8 +138,8 @@ class TestDatasetCreate: payload = {"name": name.lower()} res = create_dataset(HttpApiAuth, payload) - assert res["code"] == 103, res - assert res["message"] == f"Dataset name '{name.lower()}' already exists", res + assert res["code"] == 0, res + assert res["data"]["name"] == name.lower() + "(1)", res @pytest.mark.p2 def test_avatar(self, HttpApiAuth, tmp_path): diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py index 049f288b3..a97cb66c8 100644 --- a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py +++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py @@ -17,13 +17,13 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from operator import attrgetter import pytest -from configs import DATASET_NAME_LIMIT, HOST_ADDRESS, INVALID_API_TOKEN +from configs import DATASET_NAME_LIMIT, DEFAULT_PARSER_CONFIG, HOST_ADDRESS, INVALID_API_TOKEN from hypothesis import example, given, settings from ragflow_sdk import DataSet, RAGFlow from utils import encode_avatar from utils.file_utils import create_image_file from utils.hypothesis_utils import valid_names -from configs import DEFAULT_PARSER_CONFIG + @pytest.mark.usefixtures("clear_datasets") class TestAuthorization: @@ -95,9 +95,8 @@ class TestDatasetCreate: payload = {"name": name} client.create_dataset(**payload) - with pytest.raises(Exception) as excinfo: - client.create_dataset(**payload) - assert str(excinfo.value) == f"Dataset name '{name}' already exists", str(excinfo.value) + dataset = client.create_dataset(**payload) + assert dataset.name == name + "(1)", str(dataset) @pytest.mark.p3 def test_name_case_insensitive(self, client): @@ -106,9 +105,8 @@ class TestDatasetCreate: client.create_dataset(**payload) payload = {"name": name.lower()} - with pytest.raises(Exception) as excinfo: - client.create_dataset(**payload) - assert str(excinfo.value) == f"Dataset name '{name.lower()}' already exists", str(excinfo.value) + dataset = client.create_dataset(**payload) + assert dataset.name == name.lower() + "(1)", str(dataset) @pytest.mark.p2 def test_avatar(self, client, tmp_path): From 93422fa8cc121148dfd5ff76b1a1bdc32c42ff48 Mon Sep 17 00:00:00 2001 From: Billy Bao Date: Thu, 13 Nov 2025 15:19:02 +0800 Subject: [PATCH 03/15] Fix: Law parser (#11246) ### What problem does this PR solve? Fix: Law parser ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- rag/nlp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py index 61a3b6f3a..80acf1d8f 100644 --- a/rag/nlp/__init__.py +++ b/rag/nlp/__init__.py @@ -482,7 +482,7 @@ def tree_merge(bull, sections, depth): root = Node(level=0, depth=target_level, texts=[]) root.build_tree(lines) - return [("\n").join(element) for element in root.get_tree() if element] + return [element for element in root.get_tree() if element] def hierarchical_merge(bull, sections, depth): From 70a0f081f68b644ed386381b42dde72ba9ed8558 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Thu, 13 Nov 2025 16:11:07 +0800 Subject: [PATCH 04/15] Minor tweaks (#11249) ### What problem does this PR solve? Fix some IDE warnings ### Type of change - [x] Refactoring --------- Signed-off-by: Jin Hai --- api/apps/__init__.py | 12 +++++++----- api/apps/canvas_app.py | 7 +++---- api/apps/connector_app.py | 1 - api/apps/conversation_app.py | 1 - api/apps/dialog_app.py | 10 +++++----- api/apps/document_app.py | 3 ++- api/apps/file_app.py | 4 ++-- api/apps/kb_app.py | 2 ++ 8 files changed, 21 insertions(+), 19 deletions(-) diff --git a/api/apps/__init__.py b/api/apps/__init__.py index f2009db2c..e6249a443 100644 --- a/api/apps/__init__.py +++ b/api/apps/__init__.py @@ -96,12 +96,12 @@ login_manager.init_app(app) commands.register_commands(app) -def search_pages_path(pages_dir): +def search_pages_path(page_path): app_path_list = [ - path for path in pages_dir.glob("*_app.py") if not path.name.startswith(".") + path for path in page_path.glob("*_app.py") if not path.name.startswith(".") ] api_path_list = [ - path for path in pages_dir.glob("*sdk/*.py") if not path.name.startswith(".") + path for path in page_path.glob("*sdk/*.py") if not path.name.startswith(".") ] app_path_list.extend(api_path_list) return app_path_list @@ -138,7 +138,7 @@ pages_dir = [ ] client_urls_prefix = [ - register_page(path) for dir in pages_dir for path in search_pages_path(dir) + register_page(path) for directory in pages_dir for path in search_pages_path(directory) ] @@ -177,5 +177,7 @@ def load_user(web_request): @app.teardown_request -def _db_close(exc): +def _db_close(exception): + if exception: + logging.exception(f"Request failed: {exception}") close_connection() diff --git a/api/apps/canvas_app.py b/api/apps/canvas_app.py index 0ac2951ae..bc0ea8b80 100644 --- a/api/apps/canvas_app.py +++ b/api/apps/canvas_app.py @@ -426,7 +426,6 @@ def test_db_connect(): try: import trino import os - from trino.auth import BasicAuthentication except Exception as e: return server_error_response(f"Missing dependency 'trino'. Please install: pip install trino, detail: {e}") @@ -438,7 +437,7 @@ def test_db_connect(): auth = None if http_scheme == "https" and req.get("password"): - auth = BasicAuthentication(req.get("username") or "ragflow", req["password"]) + auth = trino.BasicAuthentication(req.get("username") or "ragflow", req["password"]) conn = trino.dbapi.connect( host=req["host"], @@ -471,8 +470,8 @@ def test_db_connect(): @login_required def getlistversion(canvas_id): try: - list =sorted([c.to_dict() for c in UserCanvasVersionService.list_by_canvas_id(canvas_id)], key=lambda x: x["update_time"]*-1) - return get_json_result(data=list) + versions =sorted([c.to_dict() for c in UserCanvasVersionService.list_by_canvas_id(canvas_id)], key=lambda x: x["update_time"]*-1) + return get_json_result(data=versions) except Exception as e: return get_data_error_result(message=f"Error getting history files: {e}") diff --git a/api/apps/connector_app.py b/api/apps/connector_app.py index 23965e617..80f791c93 100644 --- a/api/apps/connector_app.py +++ b/api/apps/connector_app.py @@ -55,7 +55,6 @@ def set_connector(): "timeout_secs": int(req.get("timeout_secs", 60 * 29)), "status": TaskStatus.SCHEDULE, } - conn["status"] = TaskStatus.SCHEDULE ConnectorService.save(**conn) time.sleep(1) diff --git a/api/apps/conversation_app.py b/api/apps/conversation_app.py index 984e57cac..d0465252a 100644 --- a/api/apps/conversation_app.py +++ b/api/apps/conversation_app.py @@ -85,7 +85,6 @@ def get(): if not e: return get_data_error_result(message="Conversation not found!") tenants = UserTenantService.query(user_id=current_user.id) - avatar = None for tenant in tenants: dialog = DialogService.query(tenant_id=tenant.tenant_id, id=conv.dialog_id) if dialog and len(dialog) > 0: diff --git a/api/apps/dialog_app.py b/api/apps/dialog_app.py index 99f700568..82c78ffed 100644 --- a/api/apps/dialog_app.py +++ b/api/apps/dialog_app.py @@ -154,15 +154,15 @@ def get_kb_names(kb_ids): @login_required def list_dialogs(): try: - diags = DialogService.query( + conversations = DialogService.query( tenant_id=current_user.id, status=StatusEnum.VALID.value, reverse=True, order_by=DialogService.model.create_time) - diags = [d.to_dict() for d in diags] - for d in diags: - d["kb_ids"], d["kb_names"] = get_kb_names(d["kb_ids"]) - return get_json_result(data=diags) + conversations = [d.to_dict() for d in conversations] + for conversation in conversations: + conversation["kb_ids"], conversation["kb_names"] = get_kb_names(conversation["kb_ids"]) + return get_json_result(data=conversations) except Exception as e: return server_error_response(e) diff --git a/api/apps/document_app.py b/api/apps/document_app.py index c2e37598e..12c19f978 100644 --- a/api/apps/document_app.py +++ b/api/apps/document_app.py @@ -308,7 +308,7 @@ def get_filter(): @manager.route("/infos", methods=["POST"]) # noqa: F821 @login_required -def docinfos(): +def doc_infos(): req = request.json doc_ids = req["doc_ids"] for doc_id in doc_ids: @@ -544,6 +544,7 @@ def change_parser(): return get_data_error_result(message="Tenant not found!") if settings.docStoreConn.indexExist(search.index_name(tenant_id), doc.kb_id): settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id) + return None try: if "pipeline_id" in req and req["pipeline_id"] != "": diff --git a/api/apps/file_app.py b/api/apps/file_app.py index 279e32525..7daff6ed7 100644 --- a/api/apps/file_app.py +++ b/api/apps/file_app.py @@ -246,8 +246,8 @@ def rm(): try: if file.location: settings.STORAGE_IMPL.rm(file.parent_id, file.location) - except Exception: - logging.exception(f"Fail to remove object: {file.parent_id}/{file.location}") + except Exception as e: + logging.exception(f"Fail to remove object: {file.parent_id}/{file.location}, error: {e}") informs = File2DocumentService.get_by_file_id(file.id) for inform in informs: diff --git a/api/apps/kb_app.py b/api/apps/kb_app.py index 4546b2586..e570debb2 100644 --- a/api/apps/kb_app.py +++ b/api/apps/kb_app.py @@ -731,6 +731,8 @@ def delete_kb_task(): def cancel_task(task_id): REDIS_CONN.set(f"{task_id}-cancel", "x") + kb_task_id_field: str = "" + kb_task_finish_at: str = "" match pipeline_task_type: case PipelineTaskType.GRAPH_RAG: kb_task_id_field = "graphrag_task_id" From 908450509fb656c4ad1c9272ffbdea95bbb5cd6c Mon Sep 17 00:00:00 2001 From: Yongteng Lei Date: Thu, 13 Nov 2025 18:48:07 +0800 Subject: [PATCH 05/15] Feat: add fault-tolerant mechanism to RAPTOR (#11206) ### What problem does this PR solve? Add fault-tolerant mechanism to RAPTOR. ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- common/data_source/google_util/oauth_flow.py | 76 +-------- graphrag/general/extractor.py | 2 +- rag/raptor.py | 154 ++++++++++--------- rag/svr/task_executor.py | 3 + 4 files changed, 86 insertions(+), 149 deletions(-) diff --git a/common/data_source/google_util/oauth_flow.py b/common/data_source/google_util/oauth_flow.py index 7e39e5283..e6ba58274 100644 --- a/common/data_source/google_util/oauth_flow.py +++ b/common/data_source/google_util/oauth_flow.py @@ -3,15 +3,9 @@ import os import threading from typing import Any, Callable -import requests - from common.data_source.config import DocumentSource from common.data_source.google_util.constant import GOOGLE_SCOPES -GOOGLE_DEVICE_CODE_URL = "https://oauth2.googleapis.com/device/code" -GOOGLE_DEVICE_TOKEN_URL = "https://oauth2.googleapis.com/token" -DEFAULT_DEVICE_INTERVAL = 5 - def _get_requested_scopes(source: DocumentSource) -> list[str]: """Return the scopes to request, honoring an optional override env var.""" @@ -55,62 +49,6 @@ def _run_with_timeout(func: Callable[[], Any], timeout_secs: int, timeout_messag return result.get("value") -def _extract_client_info(credentials: dict[str, Any]) -> tuple[str, str | None]: - if "client_id" in credentials: - return credentials["client_id"], credentials.get("client_secret") - for key in ("installed", "web"): - if key in credentials and isinstance(credentials[key], dict): - nested = credentials[key] - if "client_id" not in nested: - break - return nested["client_id"], nested.get("client_secret") - raise ValueError("Provided Google OAuth credentials are missing client_id.") - - -def start_device_authorization_flow( - credentials: dict[str, Any], - source: DocumentSource, -) -> tuple[dict[str, Any], dict[str, Any]]: - client_id, client_secret = _extract_client_info(credentials) - data = { - "client_id": client_id, - "scope": " ".join(_get_requested_scopes(source)), - } - if client_secret: - data["client_secret"] = client_secret - resp = requests.post(GOOGLE_DEVICE_CODE_URL, data=data, timeout=15) - resp.raise_for_status() - payload = resp.json() - state = { - "client_id": client_id, - "client_secret": client_secret, - "device_code": payload.get("device_code"), - "interval": payload.get("interval", DEFAULT_DEVICE_INTERVAL), - } - response_data = { - "user_code": payload.get("user_code"), - "verification_url": payload.get("verification_url") or payload.get("verification_uri"), - "verification_url_complete": payload.get("verification_url_complete") - or payload.get("verification_uri_complete"), - "expires_in": payload.get("expires_in"), - "interval": state["interval"], - } - return state, response_data - - -def poll_device_authorization_flow(state: dict[str, Any]) -> dict[str, Any]: - data = { - "client_id": state["client_id"], - "device_code": state["device_code"], - "grant_type": "urn:ietf:params:oauth:grant-type:device_code", - } - if state.get("client_secret"): - data["client_secret"] = state["client_secret"] - resp = requests.post(GOOGLE_DEVICE_TOKEN_URL, data=data, timeout=20) - resp.raise_for_status() - return resp.json() - - def _run_local_server_flow(client_config: dict[str, Any], source: DocumentSource) -> dict[str, Any]: """Launch the standard Google OAuth local-server flow to mint user tokens.""" from google_auth_oauthlib.flow import InstalledAppFlow # type: ignore @@ -125,10 +63,7 @@ def _run_local_server_flow(client_config: dict[str, Any], source: DocumentSource preferred_port = os.environ.get("GOOGLE_OAUTH_LOCAL_SERVER_PORT") port = int(preferred_port) if preferred_port else 0 timeout_secs = _get_oauth_timeout_secs() - timeout_message = ( - f"Google OAuth verification timed out after {timeout_secs} seconds. " - "Close any pending consent windows and rerun the connector configuration to try again." - ) + timeout_message = f"Google OAuth verification timed out after {timeout_secs} seconds. Close any pending consent windows and rerun the connector configuration to try again." print("Launching Google OAuth flow. A browser window should open shortly.") print("If it does not, copy the URL shown in the console into your browser manually.") @@ -153,11 +88,8 @@ def _run_local_server_flow(client_config: dict[str, Any], source: DocumentSource instructions = [ "Google rejected one or more of the requested OAuth scopes.", "Fix options:", - " 1. In Google Cloud Console, open APIs & Services > OAuth consent screen and add the missing scopes " - " (Drive metadata + Admin Directory read scopes), then re-run the flow.", + " 1. In Google Cloud Console, open APIs & Services > OAuth consent screen and add the missing scopes (Drive metadata + Admin Directory read scopes), then re-run the flow.", " 2. Set GOOGLE_OAUTH_SCOPE_OVERRIDE to a comma-separated list of scopes you are allowed to request.", - " 3. For quick local testing only, export OAUTHLIB_RELAX_TOKEN_SCOPE=1 to accept the reduced scopes " - " (be aware the connector may lose functionality).", ] raise RuntimeError("\n".join(instructions)) from warning raise @@ -184,8 +116,6 @@ def ensure_oauth_token_dict(credentials: dict[str, Any], source: DocumentSource) client_config = {"web": credentials["web"]} if client_config is None: - raise ValueError( - "Provided Google OAuth credentials are missing both tokens and a client configuration." - ) + raise ValueError("Provided Google OAuth credentials are missing both tokens and a client configuration.") return _run_local_server_flow(client_config, source) diff --git a/graphrag/general/extractor.py b/graphrag/general/extractor.py index 1df38ed1c..495e562ed 100644 --- a/graphrag/general/extractor.py +++ b/graphrag/general/extractor.py @@ -114,7 +114,7 @@ class Extractor: async def extract_all(doc_id, chunks, max_concurrency=MAX_CONCURRENT_PROCESS_AND_EXTRACT_CHUNK, task_id=""): out_results = [] error_count = 0 - max_errors = 3 + max_errors = int(os.environ.get("GRAPHRAG_MAX_ERRORS", 3)) limiter = trio.Semaphore(max_concurrency) diff --git a/rag/raptor.py b/rag/raptor.py index e6efe3504..a455d0127 100644 --- a/rag/raptor.py +++ b/rag/raptor.py @@ -15,27 +15,35 @@ # import logging import re -import umap + import numpy as np -from sklearn.mixture import GaussianMixture import trio +import umap +from sklearn.mixture import GaussianMixture from api.db.services.task_service import has_canceled from common.connection_utils import timeout from common.exceptions import TaskCanceledException +from common.token_utils import truncate from graphrag.utils import ( - get_llm_cache, + chat_limiter, get_embed_cache, + get_llm_cache, set_embed_cache, set_llm_cache, - chat_limiter, ) -from common.token_utils import truncate class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval: def __init__( - self, max_cluster, llm_model, embd_model, prompt, max_token=512, threshold=0.1 + self, + max_cluster, + llm_model, + embd_model, + prompt, + max_token=512, + threshold=0.1, + max_errors=3, ): self._max_cluster = max_cluster self._llm_model = llm_model @@ -43,31 +51,35 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval: self._threshold = threshold self._prompt = prompt self._max_token = max_token + self._max_errors = max(1, max_errors) + self._error_count = 0 - @timeout(60*20) + @timeout(60 * 20) async def _chat(self, system, history, gen_conf): - response = await trio.to_thread.run_sync( - lambda: get_llm_cache(self._llm_model.llm_name, system, history, gen_conf) - ) + cached = await trio.to_thread.run_sync(lambda: get_llm_cache(self._llm_model.llm_name, system, history, gen_conf)) + if cached: + return cached - if response: - return response - response = await trio.to_thread.run_sync( - lambda: self._llm_model.chat(system, history, gen_conf) - ) - response = re.sub(r"^.*", "", response, flags=re.DOTALL) - if response.find("**ERROR**") >= 0: - raise Exception(response) - await trio.to_thread.run_sync( - lambda: set_llm_cache(self._llm_model.llm_name, system, response, history, gen_conf) - ) - return response + last_exc = None + for attempt in range(3): + try: + response = await trio.to_thread.run_sync(lambda: self._llm_model.chat(system, history, gen_conf)) + response = re.sub(r"^.*", "", response, flags=re.DOTALL) + if response.find("**ERROR**") >= 0: + raise Exception(response) + await trio.to_thread.run_sync(lambda: set_llm_cache(self._llm_model.llm_name, system, response, history, gen_conf)) + return response + except Exception as exc: + last_exc = exc + logging.warning("RAPTOR LLM call failed on attempt %d/3: %s", attempt + 1, exc) + if attempt < 2: + await trio.sleep(1 + attempt) + + raise last_exc if last_exc else Exception("LLM chat failed without exception") @timeout(20) async def _embedding_encode(self, txt): - response = await trio.to_thread.run_sync( - lambda: get_embed_cache(self._embd_model.llm_name, txt) - ) + response = await trio.to_thread.run_sync(lambda: get_embed_cache(self._embd_model.llm_name, txt)) if response is not None: return response embds, _ = await trio.to_thread.run_sync(lambda: self._embd_model.encode([txt])) @@ -82,7 +94,6 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval: n_clusters = np.arange(1, max_clusters) bics = [] for n in n_clusters: - if task_id: if has_canceled(task_id): logging.info(f"Task {task_id} cancelled during get optimal clusters.") @@ -101,7 +112,7 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval: layers = [(0, len(chunks))] start, end = 0, len(chunks) - @timeout(60*20) + @timeout(60 * 20) async def summarize(ck_idx: list[int]): nonlocal chunks @@ -111,47 +122,50 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval: raise TaskCanceledException(f"Task {task_id} was cancelled") texts = [chunks[i][0] for i in ck_idx] - len_per_chunk = int( - (self._llm_model.max_length - self._max_token) / len(texts) - ) - cluster_content = "\n".join( - [truncate(t, max(1, len_per_chunk)) for t in texts] - ) - async with chat_limiter: + len_per_chunk = int((self._llm_model.max_length - self._max_token) / len(texts)) + cluster_content = "\n".join([truncate(t, max(1, len_per_chunk)) for t in texts]) + try: + async with chat_limiter: + if task_id and has_canceled(task_id): + logging.info(f"Task {task_id} cancelled before RAPTOR LLM call.") + raise TaskCanceledException(f"Task {task_id} was cancelled") - if task_id and has_canceled(task_id): - logging.info(f"Task {task_id} cancelled before RAPTOR LLM call.") - raise TaskCanceledException(f"Task {task_id} was cancelled") + cnt = await self._chat( + "You're a helpful assistant.", + [ + { + "role": "user", + "content": self._prompt.format(cluster_content=cluster_content), + } + ], + {"max_tokens": max(self._max_token, 512)}, # fix issue: #10235 + ) + cnt = re.sub( + "(······\n由于长度的原因，回答被截断了，要继续吗？|For the content length reason, it stopped, continue?)", + "", + cnt, + ) + logging.debug(f"SUM: {cnt}") - cnt = await self._chat( - "You're a helpful assistant.", - [ - { - "role": "user", - "content": self._prompt.format( - cluster_content=cluster_content - ), - } - ], - {"max_tokens": max(self._max_token, 512)}, # fix issue: #10235 - ) - cnt = re.sub( - "(······\n由于长度的原因，回答被截断了，要继续吗？|For the content length reason, it stopped, continue?)", - "", - cnt, - ) - logging.debug(f"SUM: {cnt}") + if task_id and has_canceled(task_id): + logging.info(f"Task {task_id} cancelled before RAPTOR embedding.") + raise TaskCanceledException(f"Task {task_id} was cancelled") - if task_id and has_canceled(task_id): - logging.info(f"Task {task_id} cancelled before RAPTOR embedding.") - raise TaskCanceledException(f"Task {task_id} was cancelled") - - embds = await self._embedding_encode(cnt) - chunks.append((cnt, embds)) + embds = await self._embedding_encode(cnt) + chunks.append((cnt, embds)) + except TaskCanceledException: + raise + except Exception as exc: + self._error_count += 1 + warn_msg = f"[RAPTOR] Skip cluster ({len(ck_idx)} chunks) due to error: {exc}" + logging.warning(warn_msg) + if callback: + callback(msg=warn_msg) + if self._error_count >= self._max_errors: + raise RuntimeError(f"RAPTOR aborted after {self._error_count} errors. Last error: {exc}") from exc labels = [] while end - start > 1: - if task_id: if has_canceled(task_id): logging.info(f"Task {task_id} cancelled during RAPTOR layer processing.") @@ -161,11 +175,7 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval: if len(embeddings) == 2: await summarize([start, start + 1]) if callback: - callback( - msg="Cluster one layer: {} -> {}".format( - end - start, len(chunks) - end - ) - ) + callback(msg="Cluster one layer: {} -> {}".format(end - start, len(chunks) - end)) labels.extend([0, 0]) layers.append((end, len(chunks))) start = end @@ -199,17 +209,11 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval: nursery.start_soon(summarize, ck_idx) - assert len(chunks) - end == n_clusters, "{} vs. {}".format( - len(chunks) - end, n_clusters - ) + assert len(chunks) - end == n_clusters, "{} vs. {}".format(len(chunks) - end, n_clusters) labels.extend(lbls) layers.append((end, len(chunks))) if callback: - callback( - msg="Cluster one layer: {} -> {}".format( - end - start, len(chunks) - end - ) - ) + callback(msg="Cluster one layer: {} -> {}".format(end - start, len(chunks) - end)) start = end end = len(chunks) diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py index d926415e5..a183bf0cf 100644 --- a/rag/svr/task_executor.py +++ b/rag/svr/task_executor.py @@ -649,6 +649,8 @@ async def run_raptor_for_kb(row, kb_parser_config, chat_mdl, embd_mdl, vector_si res = [] tk_count = 0 + max_errors = int(os.environ.get("RAPTOR_MAX_ERRORS", 3)) + async def generate(chunks, did): nonlocal tk_count, res raptor = Raptor( @@ -658,6 +660,7 @@ async def run_raptor_for_kb(row, kb_parser_config, chat_mdl, embd_mdl, vector_si raptor_config["prompt"], raptor_config["max_token"], raptor_config["threshold"], + max_errors=max_errors, ) original_length = len(chunks) chunks = await raptor(chunks, kb_parser_config["raptor"]["random_seed"], callback, row["id"]) From e8f1a245a63090cdf8fbb1d5ec1bd08ac794a398 Mon Sep 17 00:00:00 2001 From: buua436 <66937541+buua436@users.noreply.github.com> Date: Thu, 13 Nov 2025 18:48:25 +0800 Subject: [PATCH 06/15] Feat:update check_embedding api (#11254) ### What problem does this PR solve? pr: #10854 change: update check_embedding api ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- api/apps/kb_app.py | 27 ++++++++++++++++++++++----- rag/svr/task_executor.py | 8 +++++--- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/api/apps/kb_app.py b/api/apps/kb_app.py index e570debb2..b7cf58a20 100644 --- a/api/apps/kb_app.py +++ b/api/apps/kb_app.py @@ -16,6 +16,7 @@ import json import logging import random +import re from flask import request from flask_login import login_required, current_user @@ -847,8 +848,13 @@ def check_embedding(): "position_int": full_doc.get("position_int"), "top_int": full_doc.get("top_int"), "content_with_weight": full_doc.get("content_with_weight") or "", + "question_kwd": full_doc.get("question_kwd") or [] }) return out + + def _clean(s: str) -> str: + s = re.sub(r"]{0,12})?>", " ", s or "") + return s if s else "None" req = request.json kb_id = req.get("kb_id", "") embd_id = req.get("embd_id", "") @@ -861,8 +867,10 @@ def check_embedding(): results, eff_sims = [], [] for ck in samples: - txt = (ck.get("content_with_weight") or "").strip() - if not txt: + title = ck.get("doc_name") or "Title" + txt_in = "\n".join(ck.get("question_kwd") or []) or ck.get("content_with_weight") or "" + txt_in = _clean(txt_in) + if not txt_in: results.append({"chunk_id": ck["chunk_id"], "reason": "no_text"}) continue @@ -871,8 +879,16 @@ def check_embedding(): continue try: - qv, _ = emb_mdl.encode_queries(txt) - sim = _cos_sim(qv, ck["vector"]) + v, _ = emb_mdl.encode([title, txt_in]) + sim_content = _cos_sim(v[1], ck["vector"]) + title_w = 0.1 + qv_mix = title_w * v[0] + (1 - title_w) * v[1] + sim_mix = _cos_sim(qv_mix, ck["vector"]) + sim = sim_content + mode = "content_only" + if sim_mix > sim: + sim = sim_mix + mode = "title+content" except Exception: return get_error_data_result(message="embedding failure") @@ -894,8 +910,9 @@ def check_embedding(): "avg_cos_sim": round(float(np.mean(eff_sims)) if eff_sims else 0.0, 6), "min_cos_sim": round(float(np.min(eff_sims)) if eff_sims else 0.0, 6), "max_cos_sim": round(float(np.max(eff_sims)) if eff_sims else 0.0, 6), + "match_mode": mode, } - if summary["avg_cos_sim"] > 0.99: + if summary["avg_cos_sim"] > 0.9: return get_json_result(data={"summary": summary, "results": results}) return get_json_result(code=RetCode.NOT_EFFECTIVE, message="failed", data={"summary": summary, "results": results}) diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py index a183bf0cf..370bd2a10 100644 --- a/rag/svr/task_executor.py +++ b/rag/svr/task_executor.py @@ -442,7 +442,7 @@ async def embedding(docs, mdl, parser_config=None, callback=None): tk_count = 0 if len(tts) == len(cnts): vts, c = await trio.to_thread.run_sync(lambda: mdl.encode(tts[0: 1])) - tts = np.concatenate([vts[0] for _ in range(len(tts))], axis=0) + tts = np.tile(vts[0], (len(cnts), 1)) tk_count += c @timeout(60) @@ -465,8 +465,10 @@ async def embedding(docs, mdl, parser_config=None, callback=None): if not filename_embd_weight: filename_embd_weight = 0.1 title_w = float(filename_embd_weight) - vects = (title_w * tts + (1 - title_w) * - cnts) if len(tts) == len(cnts) else cnts + if tts.ndim == 2 and cnts.ndim == 2 and tts.shape == cnts.shape: + vects = title_w * tts + (1 - title_w) * cnts + else: + vects = cnts assert len(vects) == len(docs) vector_size = 0 From 63131ec9b24cf51f054a01f53aa209dbfc8b2ba5 Mon Sep 17 00:00:00 2001 From: writinwaters <93570324+writinwaters@users.noreply.github.com> Date: Fri, 14 Nov 2025 09:35:56 +0800 Subject: [PATCH 07/15] Docs: default admin credentials (#11260) ### What problem does this PR solve? ### Type of change - [x] Documentation Update --- .../dataset/add_data_source/_category_.json | 8 + .../add_data_source/add_google_drive.md | 137 ++++++++++++++++++ .../dataset/best_practices/_category_.json | 2 +- docs/guides/manage_users_and_services.md | 3 + 4 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 docs/guides/dataset/add_data_source/_category_.json create mode 100644 docs/guides/dataset/add_data_source/add_google_drive.md diff --git a/docs/guides/dataset/add_data_source/_category_.json b/docs/guides/dataset/add_data_source/_category_.json new file mode 100644 index 000000000..42f2b164a --- /dev/null +++ b/docs/guides/dataset/add_data_source/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Add data source", + "position": 18, + "link": { + "type": "generated-index", + "description": "Add various data sources" + } +} diff --git a/docs/guides/dataset/add_data_source/add_google_drive.md b/docs/guides/dataset/add_data_source/add_google_drive.md new file mode 100644 index 000000000..b4fdf14f4 --- /dev/null +++ b/docs/guides/dataset/add_data_source/add_google_drive.md @@ -0,0 +1,137 @@ +--- +sidebar_position: 3 +slug: /add_google_drive +--- + +# Add Google Drive + +## 1. Create a Google Cloud Project + +You can either create a dedicated project for RAGFlow or use an existing +Google Cloud external project. + +**Steps:** +1. Open the project creation page\ +`https://console.cloud.google.com/projectcreate` +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image1.jpeg?raw=true) +2. Select **External** as the Audience +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image2.png?raw=true) +3. Click **Create** +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image3.jpeg?raw=true) + +------------------------------------------------------------------------ + +## 2. Configure OAuth Consent Screen + +1. Go to **APIs & Services → OAuth consent screen** +2. Ensure **User Type = External** +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image4.jpeg?raw=true) +3. Add your test users under **Test Users** by entering email addresses +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image5.jpeg?raw=true) +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image6.jpeg?raw=true) + +------------------------------------------------------------------------ + +## 3. Create OAuth Client Credentials + +1. Navigate to:\ + `https://console.cloud.google.com/auth/clients` +2. Create a **Web Application** +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image7.png?raw=true) +3. Enter a name for the client +4. Add the following **Authorized Redirect URIs**: + +``` +http://localhost:9380/v1/connector/google-drive/oauth/web/callback +``` + +### If using Docker deployment: + +**Authorized JavaScript origin:** +``` +http://localhost:80 +``` + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image8.png?raw=true) +### If running from source: +**Authorized JavaScript origin:** +``` +http://localhost:9222 +``` + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image9.png?raw=true) +5. After saving, click **Download JSON**. This file will later be + uploaded into RAGFlow. + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image10.png?raw=true) + +------------------------------------------------------------------------ + +## 4. Add Scopes + +1. Open **Data Access → Add or remove scopes** + +2. Paste and add the following entries: + +``` +https://www.googleapis.com/auth/drive.readonly +https://www.googleapis.com/auth/drive.metadata.readonly +https://www.googleapis.com/auth/admin.directory.group.readonly +https://www.googleapis.com/auth/admin.directory.user.readonly +``` + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image11.jpeg?raw=true) +3. Update and Save changes + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image12.jpeg?raw=true) +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image13.jpeg?raw=true) + +------------------------------------------------------------------------ + +## 5. Enable Required APIs +Navigate to the Google API Library:\ +`https://console.cloud.google.com/apis/library` +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image14.png?raw=true) + +Enable the following APIs: + +- Google Drive API +- Admin SDK API +- Google Sheets API +- Google Docs API + + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image15.png?raw=true) + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image16.png?raw=true) + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image17.png?raw=true) + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image18.png?raw=true) + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image19.png?raw=true) + +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image21.png?raw=true) + +------------------------------------------------------------------------ + +## 6. Add Google Drive As a Data Source in RAGFlow + +1. Go to **Data Sources** inside RAGFlow +2. Select **Google Drive** +3. Upload the previously downloaded JSON credentials +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image22.jpeg?raw=true) +4. Enter the shared Google Drive folder link (https://drive.google.com/drive), such as: +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image23.png?raw=true) + +5. Click **Authorize with Google** +A browser window will appear. +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image25.jpeg?raw=true) +Click: - **Continue** - **Select All → Continue** - Authorization should +succeed - Select **OK** to add the data source +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image26.jpeg?raw=true) +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image27.jpeg?raw=true) +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image28.png?raw=true) +![placeholder-image](https://github.com/infiniflow/ragflow-docs/blob/040e4acd4c1eac6dc73dc44e934a6518de78d097/images/google_drive/image29.png?raw=true) + + diff --git a/docs/guides/dataset/best_practices/_category_.json b/docs/guides/dataset/best_practices/_category_.json index f55fe009b..79a1103d5 100644 --- a/docs/guides/dataset/best_practices/_category_.json +++ b/docs/guides/dataset/best_practices/_category_.json @@ -1,6 +1,6 @@ { "label": "Best practices", - "position": 11, + "position": 19, "link": { "type": "generated-index", "description": "Best practices on configuring a dataset." diff --git a/docs/guides/manage_users_and_services.md b/docs/guides/manage_users_and_services.md index 1d7f0fa64..a6e8a3314 100644 --- a/docs/guides/manage_users_and_services.md +++ b/docs/guides/manage_users_and_services.md @@ -64,7 +64,10 @@ The Admin CLI and Admin Service form a client-server architectural suite for RAG - -p: RAGFlow admin server port +## Default administrative account +- Username: admin@ragflow.io +- Password: admin ## Supported Commands From 6b52f7df5ad5c7a3a71f1a0f9fe1da4bff3311e9 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 14 Nov 2025 10:53:09 +0800 Subject: [PATCH 08/15] CI check comments of cheanged Python files --- .github/workflows/tests.yml | 32 ++++++++++++++++++++++++++++++++ check_comment_ascii.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 check_comment_ascii.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4357bf982..2d0804c12 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -95,6 +95,38 @@ jobs: version: ">=0.11.x" args: "check" + - name: Check comments of changed Python files + if: ${{ !cancelled() && !failure() }} + run: | + if [[ ${{ github.event_name }} == 'pull_request_target' ]]; then + CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \ + | grep -E '\.(py)$' || true) + + if [ -n "$CHANGED_FILES" ]; then + echo "Check comments of changed Python files with check_comment_ascii.py" + + readarray -t files <<< "$CHANGED_FILES" + HAS_ERROR=0 + + for file in "${files[@]}"; do + if [ -f "$file" ]; then + if python3 check_comment_ascii.py $file"; then + echo "✅ $file" + else + echo "❌ $file" + HAS_ERROR=1 + fi + fi + done + + if [ $HAS_ERROR -ne 0 ]; then + exit 1 + fi + else + echo "No Python files changed" + fi + fi + - name: Build ragflow:nightly run: | RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}} diff --git a/check_comment_ascii.py b/check_comment_ascii.py new file mode 100644 index 000000000..49cac90d7 --- /dev/null +++ b/check_comment_ascii.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import sys +import tokenize +import ast +import pathlib +import re + +ASCII = re.compile(r"^[ -~]*\Z") # Only printable ASCII + + +def check(src: str, name: str) -> int: + """ + I'm a docstring + """ + ok = 1 + # A common comment begins with `#` + with tokenize.open(src) as fp: + for tk in tokenize.generate_tokens(fp.readline): + if tk.type == tokenize.COMMENT and not ASCII.fullmatch(tk.string): + print(f"{name}:{tk.start[0]}: non-ASCII comment: {tk.string}") + ok = 0 + # A docstring begins and ends with `'''` + for node in ast.walk(ast.parse(pathlib.Path(src).read_text(), filename=name)): + if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.Module)): + if (doc := ast.get_docstring(node)) and not ASCII.fullmatch(doc): + print(f"{name}:{node.lineno}: non-ASCII docstring: {doc}") + ok = 0 + return ok + + +if __name__ == "__main__": + status = 0 + for file in sys.argv[1:]: + if not check(file, file): + status = 1 + sys.exit(status) From 7538e218a531ff24e6c980164718db275f32d92c Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 14 Nov 2025 11:32:55 +0800 Subject: [PATCH 09/15] Fix check_comment_ascii.py --- check_comment_ascii.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/check_comment_ascii.py b/check_comment_ascii.py index 49cac90d7..98bc9c2e3 100644 --- a/check_comment_ascii.py +++ b/check_comment_ascii.py @@ -5,12 +5,13 @@ import ast import pathlib import re -ASCII = re.compile(r"^[ -~]*\Z") # Only printable ASCII +ASCII = re.compile(r"^[\n -~]*\Z") # Printable ASCII + newline def check(src: str, name: str) -> int: """ - I'm a docstring + docstring line 1 + docstring line 2 """ ok = 1 # A common comment begins with `#` From 1d4d67daf846f10e624677df5deb59b2b044736b Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 14 Nov 2025 11:45:32 +0800 Subject: [PATCH 10/15] Fix check_comment_ascii.py --- check_comment_ascii.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/check_comment_ascii.py b/check_comment_ascii.py index 98bc9c2e3..57d188b6c 100644 --- a/check_comment_ascii.py +++ b/check_comment_ascii.py @@ -1,4 +1,15 @@ #!/usr/bin/env python3 + +""" +Check whether given python files contain non-ASCII comments. + +How to check the whole git repo: + +``` +$ git ls-files -z -- '*.py' | xargs -0 python3 check_comment_ascii.py +``` +""" + import sys import tokenize import ast From 3f2472f1b92eecf8cdc035d6b8b1386c3e0e7640 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 14 Nov 2025 11:53:14 +0800 Subject: [PATCH 11/15] Skip checking python comments --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d0804c12..42da89cd8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -96,7 +96,7 @@ jobs: args: "check" - name: Check comments of changed Python files - if: ${{ !cancelled() && !failure() }} + if: ${{ false }} run: | if [[ ${{ github.event_name }} == 'pull_request_target' ]]; then CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \ @@ -110,7 +110,7 @@ jobs: for file in "${files[@]}"; do if [ -f "$file" ]; then - if python3 check_comment_ascii.py $file"; then + if python3 check_comment_ascii.py "$file"; then echo "✅ $file" else echo "❌ $file" From 72c20022f62a9982bce4fc4947f17fe671132094 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Fri, 14 Nov 2025 12:32:08 +0800 Subject: [PATCH 12/15] Refactor service config fetching in admin server (#11267) ### What problem does this PR solve? As title ### Type of change - [x] Refactoring Signed-off-by: Jin Hai Co-authored-by: Zhichang Yu --- admin/client/README.md | 2 +- admin/server/auth.py | 2 +- admin/server/config.py | 30 +++++++++++++++--------------- admin/server/services.py | 22 +++++++++------------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/admin/client/README.md b/admin/client/README.md index 1964a41d4..07de0ab69 100644 --- a/admin/client/README.md +++ b/admin/client/README.md @@ -4,7 +4,7 @@ Admin Service is a dedicated management component designed to monitor, maintain, and administrate the RAGFlow system. It provides comprehensive tools for ensuring system stability, performing operational tasks, and managing users and permissions efficiently. -The service offers real-time monitoring of critical components, including the RAGFlow server, Task Executor processes, and dependent services such as MySQL, Elasticsearch, Redis, and MinIO. It automatically checks their health status, resource usage, and uptime, and performs restarts in case of failures to minimize downtime. +The service offers real-time monitoring of critical components, including the RAGFlow server, Task Executor processes, and dependent services such as MySQL, Infinity, Elasticsearch, Redis, and MinIO. It automatically checks their health status, resource usage, and uptime, and performs restarts in case of failures to minimize downtime. For user and system management, it supports listing, creating, modifying, and deleting users and their associated resources like knowledge bases and Agents. diff --git a/admin/server/auth.py b/admin/server/auth.py index 564c348e3..4217977a2 100644 --- a/admin/server/auth.py +++ b/admin/server/auth.py @@ -169,7 +169,7 @@ def login_verify(f): username = auth.parameters['username'] password = auth.parameters['password'] try: - if check_admin(username, password) is False: + if not check_admin(username, password): return jsonify({ "code": 500, "message": "Access denied", diff --git a/admin/server/config.py b/admin/server/config.py index e2c7d11ef..43f079d4f 100644 --- a/admin/server/config.py +++ b/admin/server/config.py @@ -25,8 +25,21 @@ from common.config_utils import read_config from urllib.parse import urlparse +class BaseConfig(BaseModel): + id: int + name: str + host: str + port: int + service_type: str + detail_func_name: str + + def to_dict(self) -> dict[str, Any]: + return {'id': self.id, 'name': self.name, 'host': self.host, 'port': self.port, + 'service_type': self.service_type} + + class ServiceConfigs: - configs = dict + configs = list[BaseConfig] def __init__(self): self.configs = [] @@ -45,19 +58,6 @@ class ServiceType(Enum): FILE_STORE = "file_store" -class BaseConfig(BaseModel): - id: int - name: str - host: str - port: int - service_type: str - detail_func_name: str - - def to_dict(self) -> dict[str, Any]: - return {'id': self.id, 'name': self.name, 'host': self.host, 'port': self.port, - 'service_type': self.service_type} - - class MetaConfig(BaseConfig): meta_type: str @@ -227,7 +227,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]: ragflow_count = 0 id_count = 0 for k, v in raw_configs.items(): - match (k): + match k: case "ragflow": name: str = f'ragflow_{ragflow_count}' host: str = v['host'] diff --git a/admin/server/services.py b/admin/server/services.py index e8cf4eb5d..4dbbf011e 100644 --- a/admin/server/services.py +++ b/admin/server/services.py @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # - - +import logging import re from werkzeug.security import check_password_hash from common.constants import ActiveEnum @@ -190,7 +189,8 @@ class ServiceMgr: config_dict['status'] = service_detail['status'] else: config_dict['status'] = 'timeout' - except Exception: + except Exception as e: + logging.warning(f"Can't get service details, error: {e}") config_dict['status'] = 'timeout' if not config_dict['host']: config_dict['host'] = '-' @@ -205,17 +205,13 @@ class ServiceMgr: @staticmethod def get_service_details(service_id: int): - service_id = int(service_id) + service_idx = int(service_id) configs = SERVICE_CONFIGS.configs - service_config_mapping = { - c.id: { - 'name': c.name, - 'detail_func_name': c.detail_func_name - } for c in configs - } - service_info = service_config_mapping.get(service_id, {}) - if not service_info: - raise AdminException(f"invalid service_id: {service_id}") + if service_idx < 0 or service_idx >= len(configs): + raise AdminException(f"invalid service_index: {service_idx}") + + service_config = configs[service_idx] + service_info = {'name': service_config.name, 'detail_func_name': service_config.detail_func_name} detail_func = getattr(health_utils, service_info.get('detail_func_name')) res = detail_func() From 87e69868c0b1c817f2fd51f30a2737e6020a728b Mon Sep 17 00:00:00 2001 From: chanx <1243304602@qq.com> Date: Fri, 14 Nov 2025 13:56:56 +0800 Subject: [PATCH 13/15] Fixes: Added session variable types and modified configuration (#11269) ### What problem does this PR solve? Fixes: Added session variable types and modified configuration - Added more types of session variables - Modified the embedding model switching logic in the knowledge base configuration ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- web/src/components/dynamic-form.tsx | 152 ++++++++--- web/src/components/ui/segmented.tsx | 5 +- web/src/locales/en.ts | 1 + web/src/locales/zh.ts | 1 + .../component/add-variable-modal.tsx | 134 ++++++++++ .../{contant.ts => constant.ts} | 26 +- .../gobal-variable-sheet/hooks/use-form.tsx | 41 +++ .../hooks/use-object-fields.tsx | 246 ++++++++++++++++++ .../agent/gobal-variable-sheet/index.tsx | 188 ++++--------- web/src/pages/agent/hooks/use-build-dsl.ts | 10 +- web/src/pages/agent/hooks/use-save-graph.ts | 2 +- web/src/pages/agent/index.tsx | 18 +- web/src/pages/agent/utils.ts | 24 +- .../configuration/common-item.tsx | 46 +++- .../dataset/dataset-setting/general-form.tsx | 2 +- .../pages/dataset/dataset-setting/hooks.ts | 21 ++ web/src/services/knowledge-service.ts | 6 + web/src/utils/api.ts | 2 + 18 files changed, 712 insertions(+), 213 deletions(-) create mode 100644 web/src/pages/agent/gobal-variable-sheet/component/add-variable-modal.tsx rename web/src/pages/agent/gobal-variable-sheet/{contant.ts => constant.ts} (72%) create mode 100644 web/src/pages/agent/gobal-variable-sheet/hooks/use-form.tsx create mode 100644 web/src/pages/agent/gobal-variable-sheet/hooks/use-object-fields.tsx diff --git a/web/src/components/dynamic-form.tsx b/web/src/components/dynamic-form.tsx index f7449ec9f..a90afe287 100644 --- a/web/src/components/dynamic-form.tsx +++ b/web/src/components/dynamic-form.tsx @@ -61,6 +61,12 @@ export interface FormFieldConfig { horizontal?: boolean; onChange?: (value: any) => void; tooltip?: React.ReactNode; + customValidate?: ( + value: any, + formValues: any, + ) => string | boolean | Promise; + dependencies?: string[]; + schema?: ZodSchema; } // Component props interface @@ -94,36 +100,40 @@ const generateSchema = (fields: FormFieldConfig[]): ZodSchema => { let fieldSchema: ZodSchema; // Create base validation schema based on field type - switch (field.type) { - case FormFieldType.Email: - fieldSchema = z.string().email('Please enter a valid email address'); - break; - case FormFieldType.Number: - fieldSchema = z.coerce.number(); - if (field.validation?.min !== undefined) { - fieldSchema = (fieldSchema as z.ZodNumber).min( - field.validation.min, - field.validation.message || - `Value cannot be less than ${field.validation.min}`, - ); - } - if (field.validation?.max !== undefined) { - fieldSchema = (fieldSchema as z.ZodNumber).max( - field.validation.max, - field.validation.message || - `Value cannot be greater than ${field.validation.max}`, - ); - } - break; - case FormFieldType.Checkbox: - fieldSchema = z.boolean(); - break; - case FormFieldType.Tag: - fieldSchema = z.array(z.string()); - break; - default: - fieldSchema = z.string(); - break; + if (field.schema) { + fieldSchema = field.schema; + } else { + switch (field.type) { + case FormFieldType.Email: + fieldSchema = z.string().email('Please enter a valid email address'); + break; + case FormFieldType.Number: + fieldSchema = z.coerce.number(); + if (field.validation?.min !== undefined) { + fieldSchema = (fieldSchema as z.ZodNumber).min( + field.validation.min, + field.validation.message || + `Value cannot be less than ${field.validation.min}`, + ); + } + if (field.validation?.max !== undefined) { + fieldSchema = (fieldSchema as z.ZodNumber).max( + field.validation.max, + field.validation.message || + `Value cannot be greater than ${field.validation.max}`, + ); + } + break; + case FormFieldType.Checkbox: + fieldSchema = z.boolean(); + break; + case FormFieldType.Tag: + fieldSchema = z.array(z.string()); + break; + default: + fieldSchema = z.string(); + break; + } } // Handle required fields @@ -300,10 +310,90 @@ const DynamicForm = { // Initialize form const form = useForm({ - resolver: zodResolver(schema), + resolver: async (data, context, options) => { + const zodResult = await zodResolver(schema)(data, context, options); + + let combinedErrors = { ...zodResult.errors }; + + const fieldErrors: Record = + {}; + for (const field of fields) { + if (field.customValidate && data[field.name] !== undefined) { + try { + const result = await field.customValidate( + data[field.name], + data, + ); + if (typeof result === 'string') { + fieldErrors[field.name] = { + type: 'custom', + message: result, + }; + } else if (result === false) { + fieldErrors[field.name] = { + type: 'custom', + message: + field.validation?.message || `${field.label} is invalid`, + }; + } + } catch (error) { + fieldErrors[field.name] = { + type: 'custom', + message: + error instanceof Error + ? error.message + : 'Validation failed', + }; + } + } + } + + combinedErrors = { + ...combinedErrors, + ...fieldErrors, + } as any; + console.log('combinedErrors', combinedErrors); + return { + values: Object.keys(combinedErrors).length ? {} : data, + errors: combinedErrors, + } as any; + }, defaultValues, }); + useEffect(() => { + const dependencyMap: Record = {}; + + fields.forEach((field) => { + if (field.dependencies && field.dependencies.length > 0) { + field.dependencies.forEach((dep) => { + if (!dependencyMap[dep]) { + dependencyMap[dep] = []; + } + dependencyMap[dep].push(field.name); + }); + } + }); + + const subscriptions = Object.keys(dependencyMap).map((depField) => { + return form.watch((values: any, { name }) => { + if (name === depField && dependencyMap[depField]) { + dependencyMap[depField].forEach((dependentField) => { + form.trigger(dependentField as any); + }); + } + }); + }); + + return () => { + subscriptions.forEach((sub) => { + if (sub.unsubscribe) { + sub.unsubscribe(); + } + }); + }; + }, [fields, form]); + // Expose form methods via ref useImperativeHandle(ref, () => ({ submit: () => form.handleSubmit(onSubmit)(), diff --git a/web/src/components/ui/segmented.tsx b/web/src/components/ui/segmented.tsx index 8aadc3b21..3f9b0cc53 100644 --- a/web/src/components/ui/segmented.tsx +++ b/web/src/components/ui/segmented.tsx @@ -51,6 +51,7 @@ export interface SegmentedProps direction?: 'ltr' | 'rtl'; motionName?: string; activeClassName?: string; + itemClassName?: string; rounded?: keyof typeof segmentedVariants.round; sizeType?: keyof typeof segmentedVariants.size; buttonSize?: keyof typeof segmentedVariants.buttonSize; @@ -62,6 +63,7 @@ export function Segmented({ onChange, className, activeClassName, + itemClassName, rounded = 'default', sizeType = 'default', buttonSize = 'default', @@ -92,12 +94,13 @@ export function Segmented({
void; + visible?: boolean; + hideModal: () => void; + defaultValues?: FieldValues; + setDefaultValues?: (value: FieldValues) => void; +}) => { + const { + fields, + setFields, + visible, + hideModal, + defaultValues, + setDefaultValues, + } = props; + + const { handleSubmit: submitForm, loading } = useHandleForm(); + + const { handleCustomValidate, handleCustomSchema, handleRender } = + useObjectFields(); + + const formRef = useRef(null); + + const handleFieldUpdate = ( + fieldName: string, + updatedField: Partial, + ) => { + setFields((prevFields: any) => + prevFields.map((field: any) => + field.name === fieldName ? { ...field, ...updatedField } : field, + ), + ); + }; + + useEffect(() => { + const typeField = fields?.find((item) => item.name === 'type'); + + if (typeField) { + typeField.onChange = (value) => { + handleFieldUpdate('value', { + type: TypeMaps[value as keyof typeof TypeMaps], + render: handleRender(value), + customValidate: handleCustomValidate(value), + schema: handleCustomSchema(value), + }); + const values = formRef.current?.getValues(); + // setTimeout(() => { + switch (value) { + case TypesWithArray.Boolean: + setDefaultValues?.({ ...values, value: false }); + break; + case TypesWithArray.Number: + setDefaultValues?.({ ...values, value: 0 }); + break; + case TypesWithArray.Object: + setDefaultValues?.({ ...values, value: {} }); + break; + case TypesWithArray.ArrayString: + setDefaultValues?.({ ...values, value: [''] }); + break; + case TypesWithArray.ArrayNumber: + setDefaultValues?.({ ...values, value: [''] }); + break; + case TypesWithArray.ArrayBoolean: + setDefaultValues?.({ ...values, value: [false] }); + break; + case TypesWithArray.ArrayObject: + setDefaultValues?.({ ...values, value: [] }); + break; + default: + setDefaultValues?.({ ...values, value: '' }); + break; + } + // }, 0); + }; + } + }, [fields]); + + const handleSubmit = async (fieldValue: FieldValues) => { + await submitForm(fieldValue); + hideModal(); + }; + + return ( + + { + console.log(data); + }} + defaultValues={defaultValues} + onFieldUpdate={handleFieldUpdate} + > +
+ { + hideModal?.(); + }} + /> + { + handleSubmit(values); + // console.log(values); + // console.log(nodes, edges); + // handleOk(values); + }} + /> +
+ + + ); +}; diff --git a/web/src/pages/agent/gobal-variable-sheet/contant.ts b/web/src/pages/agent/gobal-variable-sheet/constant.ts similarity index 72% rename from web/src/pages/agent/gobal-variable-sheet/contant.ts rename to web/src/pages/agent/gobal-variable-sheet/constant.ts index 2f3bd395f..fc668e330 100644 --- a/web/src/pages/agent/gobal-variable-sheet/contant.ts +++ b/web/src/pages/agent/gobal-variable-sheet/constant.ts @@ -13,14 +13,14 @@ export enum TypesWithArray { String = 'string', Number = 'number', Boolean = 'boolean', - // Object = 'object', - // ArrayString = 'array', - // ArrayNumber = 'array', - // ArrayBoolean = 'array', - // ArrayObject = 'array', + Object = 'object', + ArrayString = 'array', + ArrayNumber = 'array', + ArrayBoolean = 'array', + ArrayObject = 'array', } -export const GobalFormFields = [ +export const GlobalFormFields = [ { label: t('flow.name'), name: 'name', @@ -50,11 +50,11 @@ export const GobalFormFields = [ label: t('flow.description'), name: 'description', placeholder: t('flow.variableDescription'), - type: 'textarea', + type: FormFieldType.Textarea, }, ] as FormFieldConfig[]; -export const GobalVariableFormDefaultValues = { +export const GlobalVariableFormDefaultValues = { name: '', type: TypesWithArray.String, value: '', @@ -65,9 +65,9 @@ export const TypeMaps = { [TypesWithArray.String]: FormFieldType.Textarea, [TypesWithArray.Number]: FormFieldType.Number, [TypesWithArray.Boolean]: FormFieldType.Checkbox, - // [TypesWithArray.Object]: FormFieldType.Textarea, - // [TypesWithArray.ArrayString]: FormFieldType.Textarea, - // [TypesWithArray.ArrayNumber]: FormFieldType.Textarea, - // [TypesWithArray.ArrayBoolean]: FormFieldType.Textarea, - // [TypesWithArray.ArrayObject]: FormFieldType.Textarea, + [TypesWithArray.Object]: FormFieldType.Textarea, + [TypesWithArray.ArrayString]: FormFieldType.Textarea, + [TypesWithArray.ArrayNumber]: FormFieldType.Textarea, + [TypesWithArray.ArrayBoolean]: FormFieldType.Textarea, + [TypesWithArray.ArrayObject]: FormFieldType.Textarea, }; diff --git a/web/src/pages/agent/gobal-variable-sheet/hooks/use-form.tsx b/web/src/pages/agent/gobal-variable-sheet/hooks/use-form.tsx new file mode 100644 index 000000000..cb38012f3 --- /dev/null +++ b/web/src/pages/agent/gobal-variable-sheet/hooks/use-form.tsx @@ -0,0 +1,41 @@ +import { useFetchAgent } from '@/hooks/use-agent-request'; +import { GlobalVariableType } from '@/interfaces/database/agent'; +import { useCallback } from 'react'; +import { FieldValues } from 'react-hook-form'; +import { useSaveGraph } from '../../hooks/use-save-graph'; +import { TypesWithArray } from '../constant'; + +export const useHandleForm = () => { + const { data, refetch } = useFetchAgent(); + const { saveGraph, loading } = useSaveGraph(); + const handleObjectData = (value: any) => { + try { + return JSON.parse(value); + } catch (error) { + return value; + } + }; + const handleSubmit = useCallback(async (fieldValue: FieldValues) => { + const param = { + ...(data.dsl?.variables || {}), + [fieldValue.name]: { + ...fieldValue, + value: + fieldValue.type === TypesWithArray.Object || + fieldValue.type === TypesWithArray.ArrayObject + ? handleObjectData(fieldValue.value) + : fieldValue.value, + }, + } as Record; + + const res = await saveGraph(undefined, { + globalVariables: param, + }); + + if (res.code === 0) { + refetch(); + } + }, []); + + return { handleSubmit, loading }; +}; diff --git a/web/src/pages/agent/gobal-variable-sheet/hooks/use-object-fields.tsx b/web/src/pages/agent/gobal-variable-sheet/hooks/use-object-fields.tsx new file mode 100644 index 000000000..d8600d568 --- /dev/null +++ b/web/src/pages/agent/gobal-variable-sheet/hooks/use-object-fields.tsx @@ -0,0 +1,246 @@ +import { BlockButton, Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { Segmented } from '@/components/ui/segmented'; +import { Editor } from '@monaco-editor/react'; +import { t } from 'i18next'; +import { Trash2, X } from 'lucide-react'; +import { useCallback } from 'react'; +import { FieldValues } from 'react-hook-form'; +import { z } from 'zod'; +import { TypesWithArray } from '../constant'; + +export const useObjectFields = () => { + const booleanRender = useCallback( + (field: FieldValues, className?: string) => { + const fieldValue = field.value ? true : false; + return ( + + ); + }, + [], + ); + + const objectRender = useCallback((field: FieldValues) => { + const fieldValue = + typeof field.value === 'object' + ? JSON.stringify(field.value, null, 2) + : JSON.stringify({}, null, 2); + console.log('object-render-field', field, fieldValue); + return ( + + ); + }, []); + + const objectValidate = useCallback((value: any) => { + try { + if (!JSON.parse(value)) { + throw new Error(t('knowledgeDetails.formatTypeError')); + } + return true; + } catch (e) { + throw new Error(t('knowledgeDetails.formatTypeError')); + } + }, []); + + const arrayStringRender = useCallback((field: FieldValues, type = 'text') => { + const values = Array.isArray(field.value) + ? field.value + : [type === 'number' ? 0 : '']; + return ( + <> + {values?.map((item: any, index: number) => ( +
+ { + const newValues = [...values]; + newValues[index] = e.target.value; + field.onChange(newValues); + }} + /> + +
+ ))} + { + field.onChange([...field.value, '']); + }} + > + {t('flow.add')} + + + ); + }, []); + + const arrayBooleanRender = useCallback( + (field: FieldValues) => { + // const values = field.value || [false]; + const values = Array.isArray(field.value) ? field.value : [false]; + return ( +
+ {values?.map((item: any, index: number) => ( +
+ {booleanRender( + { + value: item, + onChange: (value) => { + values[index] = !!value; + field.onChange(values); + }, + }, + 'bg-transparent', + )} + +
+ ))} + { + field.onChange([...field.value, false]); + }} + > + {t('flow.add')} + +
+ ); + }, + [booleanRender], + ); + + const arrayNumberRender = useCallback( + (field: FieldValues) => { + return arrayStringRender(field, 'number'); + }, + [arrayStringRender], + ); + + const arrayValidate = useCallback((value: any, type: string = 'string') => { + if (!Array.isArray(value) || !value.every((item) => typeof item === type)) { + throw new Error(t('flow.formatTypeError')); + } + return true; + }, []); + + const arrayStringValidate = useCallback( + (value: any) => { + return arrayValidate(value, 'string'); + }, + [arrayValidate], + ); + + const arrayNumberValidate = useCallback( + (value: any) => { + return arrayValidate(value, 'number'); + }, + [arrayValidate], + ); + + const arrayBooleanValidate = useCallback( + (value: any) => { + return arrayValidate(value, 'boolean'); + }, + [arrayValidate], + ); + + const handleRender = (value: TypesWithArray) => { + switch (value) { + case TypesWithArray.Boolean: + return booleanRender; + case TypesWithArray.Object: + case TypesWithArray.ArrayObject: + return objectRender; + case TypesWithArray.ArrayString: + return arrayStringRender; + case TypesWithArray.ArrayNumber: + return arrayNumberRender; + case TypesWithArray.ArrayBoolean: + return arrayBooleanRender; + default: + return undefined; + } + }; + const handleCustomValidate = (value: TypesWithArray) => { + switch (value) { + case TypesWithArray.Object: + case TypesWithArray.ArrayObject: + return objectValidate; + case TypesWithArray.ArrayString: + return arrayStringValidate; + case TypesWithArray.ArrayNumber: + return arrayNumberValidate; + case TypesWithArray.ArrayBoolean: + return arrayBooleanValidate; + default: + return undefined; + } + }; + const handleCustomSchema = (value: TypesWithArray) => { + switch (value) { + case TypesWithArray.ArrayString: + return z.array(z.string()); + case TypesWithArray.ArrayNumber: + return z.array(z.number()); + case TypesWithArray.ArrayBoolean: + return z.array(z.boolean()); + default: + return undefined; + } + }; + return { + objectRender, + objectValidate, + arrayStringRender, + arrayStringValidate, + arrayNumberRender, + booleanRender, + arrayBooleanRender, + arrayNumberValidate, + arrayBooleanValidate, + handleRender, + handleCustomValidate, + handleCustomSchema, + }; +}; diff --git a/web/src/pages/agent/gobal-variable-sheet/index.tsx b/web/src/pages/agent/gobal-variable-sheet/index.tsx index 454131638..51648b8d1 100644 --- a/web/src/pages/agent/gobal-variable-sheet/index.tsx +++ b/web/src/pages/agent/gobal-variable-sheet/index.tsx @@ -1,12 +1,6 @@ import { ConfirmDeleteDialog } from '@/components/confirm-delete-dialog'; -import { - DynamicForm, - DynamicFormRef, - FormFieldConfig, - FormFieldType, -} from '@/components/dynamic-form'; +import { FormFieldConfig } from '@/components/dynamic-form'; import { BlockButton, Button } from '@/components/ui/button'; -import { Modal } from '@/components/ui/modal/modal'; import { Sheet, SheetContent, @@ -19,117 +13,65 @@ import { GlobalVariableType } from '@/interfaces/database/agent'; import { cn } from '@/lib/utils'; import { t } from 'i18next'; import { Trash2 } from 'lucide-react'; -import { useEffect, useRef, useState } from 'react'; +import { useState } from 'react'; import { FieldValues } from 'react-hook-form'; import { useSaveGraph } from '../hooks/use-save-graph'; +import { AddVariableModal } from './component/add-variable-modal'; import { - GobalFormFields, - GobalVariableFormDefaultValues, + GlobalFormFields, + GlobalVariableFormDefaultValues, TypeMaps, TypesWithArray, -} from './contant'; +} from './constant'; +import { useObjectFields } from './hooks/use-object-fields'; -export type IGobalParamModalProps = { +export type IGlobalParamModalProps = { data: any; hideModal: (open: boolean) => void; }; -export const GobalParamSheet = (props: IGobalParamModalProps) => { +export const GlobalParamSheet = (props: IGlobalParamModalProps) => { const { hideModal } = props; const { data, refetch } = useFetchAgent(); - const [fields, setFields] = useState(GobalFormFields); const { visible, showModal, hideModal: hideAddModal } = useSetModalState(); + const [fields, setFields] = useState(GlobalFormFields); const [defaultValues, setDefaultValues] = useState( - GobalVariableFormDefaultValues, + GlobalVariableFormDefaultValues, ); - const formRef = useRef(null); + const { handleCustomValidate, handleCustomSchema, handleRender } = + useObjectFields(); + const { saveGraph } = useSaveGraph(); - const handleFieldUpdate = ( - fieldName: string, - updatedField: Partial, - ) => { - setFields((prevFields) => - prevFields.map((field) => - field.name === fieldName ? { ...field, ...updatedField } : field, - ), - ); - }; - - useEffect(() => { - const typefileld = fields.find((item) => item.name === 'type'); - - if (typefileld) { - typefileld.onChange = (value) => { - // setWatchType(value); - handleFieldUpdate('value', { - type: TypeMaps[value as keyof typeof TypeMaps], - }); - const values = formRef.current?.getValues(); - setTimeout(() => { - switch (value) { - case TypesWithArray.Boolean: - setDefaultValues({ ...values, value: false }); - break; - case TypesWithArray.Number: - setDefaultValues({ ...values, value: 0 }); - break; - default: - setDefaultValues({ ...values, value: '' }); - } - }, 0); - }; - } - }, [fields]); - - const { saveGraph, loading } = useSaveGraph(); - - const handleSubmit = async (value: FieldValues) => { - const param = { - ...(data.dsl?.variables || {}), - [value.name]: value, - } as Record; - - const res = await saveGraph(undefined, { - gobalVariables: param, - }); - - if (res.code === 0) { - refetch(); - } - hideAddModal(); - }; - - const handleDeleteGobalVariable = async (key: string) => { + const handleDeleteGlobalVariable = async (key: string) => { const param = { ...(data.dsl?.variables || {}), } as Record; delete param[key]; const res = await saveGraph(undefined, { - gobalVariables: param, + globalVariables: param, }); - console.log('delete gobal variable-->', res); if (res.code === 0) { refetch(); } }; - const handleEditGobalVariable = (item: FieldValues) => { - fields.forEach((field) => { - if (field.name === 'value') { - switch (item.type) { - // [TypesWithArray.String]: FormFieldType.Textarea, - // [TypesWithArray.Number]: FormFieldType.Number, - // [TypesWithArray.Boolean]: FormFieldType.Checkbox, - case TypesWithArray.Boolean: - field.type = FormFieldType.Checkbox; - break; - case TypesWithArray.Number: - field.type = FormFieldType.Number; - break; - default: - field.type = FormFieldType.Textarea; - } + const handleEditGlobalVariable = (item: FieldValues) => { + const newFields = fields.map((field) => { + let newField = field; + newField.render = undefined; + newField.schema = undefined; + newField.customValidate = undefined; + if (newField.name === 'value') { + newField = { + ...newField, + type: TypeMaps[item.type as keyof typeof TypeMaps], + render: handleRender(item.type), + customValidate: handleCustomValidate(item.type), + schema: handleCustomSchema(item.type), + }; } + return newField; }); + setFields(newFields); setDefaultValues(item); showModal(); }; @@ -149,8 +91,8 @@ export const GobalParamSheet = (props: IGobalParamModalProps) => {
{ - setFields(GobalFormFields); - setDefaultValues(GobalVariableFormDefaultValues); + setFields(GlobalFormFields); + setDefaultValues(GlobalVariableFormDefaultValues); showModal(); }} > @@ -167,7 +109,7 @@ export const GobalParamSheet = (props: IGobalParamModalProps) => { key={key} className="flex items-center gap-3 min-h-14 justify-between px-5 py-3 border border-border-default rounded-lg hover:bg-bg-card group" onClick={() => { - handleEditGobalVariable(item); + handleEditGlobalVariable(item); }} >
@@ -177,13 +119,23 @@ export const GobalParamSheet = (props: IGobalParamModalProps) => { {item.type}
-
- {item.value} -
+ {![ + TypesWithArray.Object, + TypesWithArray.ArrayObject, + TypesWithArray.ArrayString, + TypesWithArray.ArrayNumber, + TypesWithArray.ArrayBoolean, + ].includes(item.type as TypesWithArray) && ( +
+ + {item.value} + +
+ )}
handleDeleteGobalVariable(key)} + onOk={() => handleDeleteGlobalVariable(key)} >
- - { - console.log(data); - }} - defaultValues={defaultValues} - onFieldUpdate={handleFieldUpdate} - > -
- { - hideAddModal?.(); - }} - /> - { - handleSubmit(values); - // console.log(values); - // console.log(nodes, edges); - // handleOk(values); - }} - /> -
- - + ); diff --git a/web/src/pages/agent/hooks/use-build-dsl.ts b/web/src/pages/agent/hooks/use-build-dsl.ts index 1a8569636..47ec1c225 100644 --- a/web/src/pages/agent/hooks/use-build-dsl.ts +++ b/web/src/pages/agent/hooks/use-build-dsl.ts @@ -4,7 +4,7 @@ import { RAGFlowNodeType } from '@/interfaces/database/flow'; import { useCallback } from 'react'; import { Operator } from '../constant'; import useGraphStore from '../store'; -import { buildDslComponentsByGraph, buildDslGobalVariables } from '../utils'; +import { buildDslComponentsByGraph, buildDslGlobalVariables } from '../utils'; export const useBuildDslData = () => { const { data } = useFetchAgent(); @@ -13,7 +13,7 @@ export const useBuildDslData = () => { const buildDslData = useCallback( ( currentNodes?: RAGFlowNodeType[], - otherParam?: { gobalVariables: Record }, + otherParam?: { globalVariables: Record }, ) => { const nodesToProcess = currentNodes ?? nodes; @@ -41,13 +41,13 @@ export const useBuildDslData = () => { data.dsl.components, ); - const gobalVariables = buildDslGobalVariables( + const globalVariables = buildDslGlobalVariables( data.dsl, - otherParam?.gobalVariables, + otherParam?.globalVariables, ); return { ...data.dsl, - ...gobalVariables, + ...globalVariables, graph: { nodes: filteredNodes, edges: filteredEdges }, components: dslComponents, }; diff --git a/web/src/pages/agent/hooks/use-save-graph.ts b/web/src/pages/agent/hooks/use-save-graph.ts index e59b99193..500baf716 100644 --- a/web/src/pages/agent/hooks/use-save-graph.ts +++ b/web/src/pages/agent/hooks/use-save-graph.ts @@ -21,7 +21,7 @@ export const useSaveGraph = (showMessage: boolean = true) => { const saveGraph = useCallback( async ( currentNodes?: RAGFlowNodeType[], - otherParam?: { gobalVariables: Record }, + otherParam?: { globalVariables: Record }, ) => { return setAgent({ id, diff --git a/web/src/pages/agent/index.tsx b/web/src/pages/agent/index.tsx index 21ecb22e7..b0d2f6f15 100644 --- a/web/src/pages/agent/index.tsx +++ b/web/src/pages/agent/index.tsx @@ -39,7 +39,7 @@ import { useParams } from 'umi'; import AgentCanvas from './canvas'; import { DropdownProvider } from './canvas/context'; import { Operator } from './constant'; -import { GobalParamSheet } from './gobal-variable-sheet'; +import { GlobalParamSheet } from './gobal-variable-sheet'; import { useCancelCurrentDataflow } from './hooks/use-cancel-dataflow'; import { useHandleExportJsonFile } from './hooks/use-export-json'; import { useFetchDataOnMount } from './hooks/use-fetch-data'; @@ -126,9 +126,9 @@ export default function Agent() { } = useSetModalState(); const { - visible: gobalParamSheetVisible, - showModal: showGobalParamSheet, - hideModal: hideGobalParamSheet, + visible: globalParamSheetVisible, + showModal: showGlobalParamSheet, + hideModal: hideGlobalParamSheet, } = useSetModalState(); const { @@ -216,7 +216,7 @@ export default function Agent() { showGobalParamSheet()} + onClick={() => showGlobalParamSheet()} loading={loading} > {t('flow.conversationVariable')} @@ -314,11 +314,11 @@ export default function Agent() { loading={pipelineRunning} > )} - {gobalParamSheetVisible && ( - + hideModal={hideGlobalParamSheet} + > )} ); diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts index 487067ed8..3312b7236 100644 --- a/web/src/pages/agent/utils.ts +++ b/web/src/pages/agent/utils.ts @@ -348,30 +348,30 @@ export const buildDslComponentsByGraph = ( return components; }; -export const buildDslGobalVariables = ( +export const buildDslGlobalVariables = ( dsl: DSL, - gobalVariables?: Record, + globalVariables?: Record, ) => { - if (!gobalVariables) { + if (!globalVariables) { return { globals: dsl.globals, variables: dsl.variables || {} }; } - let gobalVariablesTemp: Record = {}; - let gobalSystem: Record = {}; + let globalVariablesTemp: Record = {}; + let globalSystem: Record = {}; Object.keys(dsl.globals)?.forEach((key) => { if (key.indexOf('sys') > -1) { - gobalSystem[key] = dsl.globals[key]; + globalSystem[key] = dsl.globals[key]; } }); - Object.keys(gobalVariables).forEach((key) => { - gobalVariablesTemp['env.' + key] = gobalVariables[key].value; + Object.keys(globalVariables).forEach((key) => { + globalVariablesTemp['env.' + key] = globalVariables[key].value; }); - const gobalVariablesResult = { - ...gobalSystem, - ...gobalVariablesTemp, + const globalVariablesResult = { + ...globalSystem, + ...globalVariablesTemp, }; - return { globals: gobalVariablesResult, variables: gobalVariables }; + return { globals: globalVariablesResult, variables: globalVariables }; }; export const receiveMessageError = (res: any) => diff --git a/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx b/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx index c63309c50..c6d18af13 100644 --- a/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx +++ b/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx @@ -7,11 +7,14 @@ import { FormMessage, } from '@/components/ui/form'; import { Radio } from '@/components/ui/radio'; +import { Spin } from '@/components/ui/spin'; import { Switch } from '@/components/ui/switch'; import { useTranslate } from '@/hooks/common-hooks'; import { cn } from '@/lib/utils'; +import { useMemo, useState } from 'react'; import { useFormContext } from 'react-hook-form'; import { + useHandleKbEmbedding, useHasParsedDocument, useSelectChunkMethodList, useSelectEmbeddingModelOptions, @@ -62,11 +65,17 @@ export function ChunkMethodItem(props: IProps) { /> ); } -export function EmbeddingModelItem({ line = 1, isEdit = true }: IProps) { +export function EmbeddingModelItem({ line = 1, isEdit }: IProps) { const { t } = useTranslate('knowledgeConfiguration'); const form = useFormContext(); const embeddingModelOptions = useSelectEmbeddingModelOptions(); + const { handleChange } = useHandleKbEmbedding(); const disabled = useHasParsedDocument(isEdit); + const oldValue = useMemo(() => { + const embdStr = form.getValues('embd_id'); + return embdStr || ''; + }, [form]); + const [loading, setLoading] = useState(false); return ( <> - + + { + field.onChange(value); + if (isEdit && disabled) { + setLoading(true); + const res = await handleChange({ + embed_id: value, + callback: field.onChange, + }); + if (res.code !== 0) { + field.onChange(oldValue); + } + setLoading(false); + } + }} + value={field.value} + options={embeddingModelOptions} + placeholder={t('embeddingModelPlaceholder')} + triggerClassName="!bg-bg-base" + /> + diff --git a/web/src/pages/dataset/dataset-setting/general-form.tsx b/web/src/pages/dataset/dataset-setting/general-form.tsx index b4a7b9635..110c03a3e 100644 --- a/web/src/pages/dataset/dataset-setting/general-form.tsx +++ b/web/src/pages/dataset/dataset-setting/general-form.tsx @@ -88,7 +88,7 @@ export function GeneralForm() { }} /> - + diff --git a/web/src/pages/dataset/dataset-setting/hooks.ts b/web/src/pages/dataset/dataset-setting/hooks.ts index 605f91e4d..f9efe1d08 100644 --- a/web/src/pages/dataset/dataset-setting/hooks.ts +++ b/web/src/pages/dataset/dataset-setting/hooks.ts @@ -4,10 +4,12 @@ import { useSetModalState } from '@/hooks/common-hooks'; import { useSelectLlmOptionsByModelType } from '@/hooks/llm-hooks'; import { useFetchKnowledgeBaseConfiguration } from '@/hooks/use-knowledge-request'; import { useSelectParserList } from '@/hooks/user-setting-hooks'; +import kbService from '@/services/knowledge-service'; import { useIsFetching } from '@tanstack/react-query'; import { pick } from 'lodash'; import { useCallback, useEffect, useState } from 'react'; import { UseFormReturn } from 'react-hook-form'; +import { useParams, useSearchParams } from 'umi'; import { z } from 'zod'; import { formSchema } from './form-schema'; @@ -98,3 +100,22 @@ export const useRenameKnowledgeTag = () => { showTagRenameModal: handleShowTagRenameModal, }; }; + +export const useHandleKbEmbedding = () => { + const { id } = useParams(); + const [searchParams] = useSearchParams(); + const knowledgeBaseId = searchParams.get('id') || id; + const handleChange = useCallback( + async ({ embed_id }: { embed_id: string }) => { + const res = await kbService.checkEmbedding({ + kb_id: knowledgeBaseId, + embd_id: embed_id, + }); + return res.data; + }, + [knowledgeBaseId], + ); + return { + handleChange, + }; +}; diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.ts index 350fa4e2a..01b8da127 100644 --- a/web/src/services/knowledge-service.ts +++ b/web/src/services/knowledge-service.ts @@ -47,6 +47,7 @@ const { traceGraphRag, runRaptor, traceRaptor, + check_embedding, } = api; const methods = { @@ -214,6 +215,11 @@ const methods = { url: api.pipelineRerun, method: 'post', }, + + checkEmbedding: { + url: check_embedding, + method: 'post', + }, }; const kbService = registerServer(methods, request); diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts index 0d97801ac..e0afdbeb3 100644 --- a/web/src/utils/api.ts +++ b/web/src/utils/api.ts @@ -49,6 +49,8 @@ export default { llm_tools: `${api_host}/plugin/llm_tools`, // knowledge base + + check_embedding: `${api_host}/kb/check_embedding`, kb_list: `${api_host}/kb/list`, create_kb: `${api_host}/kb/create`, update_kb: `${api_host}/kb/update`, From 5f59418ababc619aa61244dba6772dca424c507b Mon Sep 17 00:00:00 2001 From: redredrrred <1589289338@qq.com> Date: Fri, 14 Nov 2025 13:59:03 +0800 Subject: [PATCH 14/15] Remove leftover account and password from the code (#11248) Remove legacy accounts and passwords. ### What problem does this PR solve? Remove leftover account and password in agent/templates/sql_assistant.json ### Type of change - [x] Other (please describe): --- agent/templates/sql_assistant.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/agent/templates/sql_assistant.json b/agent/templates/sql_assistant.json index 92804abc6..6e7140196 100644 --- a/agent/templates/sql_assistant.json +++ b/agent/templates/sql_assistant.json @@ -83,10 +83,10 @@ "value": [] } }, - "password": "20010812Yy!", + "password": "", "port": 3306, "sql": "{Agent:WickedGoatsDivide@content}", - "username": "13637682833@163.com" + "username": "" } }, "upstream": [ @@ -527,10 +527,10 @@ "value": [] } }, - "password": "20010812Yy!", + "password": "", "port": 3306, "sql": "{Agent:WickedGoatsDivide@content}", - "username": "13637682833@163.com" + "username": "" }, "label": "ExeSQL", "name": "ExeSQL" From e27ff8d3d42ce726941f8494a1a428ebe76587de Mon Sep 17 00:00:00 2001 From: Billy Bao Date: Fri, 14 Nov 2025 13:59:54 +0800 Subject: [PATCH 15/15] Fix: rerank algorithm (#11266) ### What problem does this PR solve? Fix: rerank algorithm #11234 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- rag/nlp/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rag/nlp/search.py b/rag/nlp/search.py index f8b3d513f..4dbd9945c 100644 --- a/rag/nlp/search.py +++ b/rag/nlp/search.py @@ -347,7 +347,7 @@ class Dealer: ## For rank feature(tag_fea) scores. rank_fea = self._rank_feature_scores(rank_feature, sres) - return tkweight * (np.array(tksim)+rank_fea) + vtweight * vtsim, tksim, vtsim + return tkweight * np.array(tksim) + vtweight * vtsim + rank_fea, tksim, vtsim def hybrid_similarity(self, ans_embd, ins_embd, ans, inst): return self.qryr.hybrid_similarity(ans_embd,