From 7538e218a531ff24e6c980164718db275f32d92c Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 14 Nov 2025 11:32:55 +0800 Subject: [PATCH 01/10] Fix check_comment_ascii.py --- check_comment_ascii.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/check_comment_ascii.py b/check_comment_ascii.py index 49cac90d7..98bc9c2e3 100644 --- a/check_comment_ascii.py +++ b/check_comment_ascii.py @@ -5,12 +5,13 @@ import ast import pathlib import re -ASCII = re.compile(r"^[ -~]*\Z") # Only printable ASCII +ASCII = re.compile(r"^[\n -~]*\Z") # Printable ASCII + newline def check(src: str, name: str) -> int: """ - I'm a docstring + docstring line 1 + docstring line 2 """ ok = 1 # A common comment begins with `#` From 1d4d67daf846f10e624677df5deb59b2b044736b Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 14 Nov 2025 11:45:32 +0800 Subject: [PATCH 02/10] Fix check_comment_ascii.py --- check_comment_ascii.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/check_comment_ascii.py b/check_comment_ascii.py index 98bc9c2e3..57d188b6c 100644 --- a/check_comment_ascii.py +++ b/check_comment_ascii.py @@ -1,4 +1,15 @@ #!/usr/bin/env python3 + +""" +Check whether given python files contain non-ASCII comments. + +How to check the whole git repo: + +``` +$ git ls-files -z -- '*.py' | xargs -0 python3 check_comment_ascii.py +``` +""" + import sys import tokenize import ast From 3f2472f1b92eecf8cdc035d6b8b1386c3e0e7640 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 14 Nov 2025 11:53:14 +0800 Subject: [PATCH 03/10] Skip checking python comments --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d0804c12..42da89cd8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -96,7 +96,7 @@ jobs: args: "check" - name: Check comments of changed Python files - if: ${{ !cancelled() && !failure() }} + if: ${{ false }} run: | if [[ ${{ github.event_name }} == 'pull_request_target' ]]; then CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \ @@ -110,7 +110,7 @@ jobs: for file in "${files[@]}"; do if [ -f "$file" ]; then - if python3 check_comment_ascii.py $file"; then + if python3 check_comment_ascii.py "$file"; then echo "✅ $file" else echo "❌ $file" From 72c20022f62a9982bce4fc4947f17fe671132094 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Fri, 14 Nov 2025 12:32:08 +0800 Subject: [PATCH 04/10] Refactor service config fetching in admin server (#11267) ### What problem does this PR solve? As title ### Type of change - [x] Refactoring Signed-off-by: Jin Hai Co-authored-by: Zhichang Yu --- admin/client/README.md | 2 +- admin/server/auth.py | 2 +- admin/server/config.py | 30 +++++++++++++++--------------- admin/server/services.py | 22 +++++++++------------- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/admin/client/README.md b/admin/client/README.md index 1964a41d4..07de0ab69 100644 --- a/admin/client/README.md +++ b/admin/client/README.md @@ -4,7 +4,7 @@ Admin Service is a dedicated management component designed to monitor, maintain, and administrate the RAGFlow system. It provides comprehensive tools for ensuring system stability, performing operational tasks, and managing users and permissions efficiently. -The service offers real-time monitoring of critical components, including the RAGFlow server, Task Executor processes, and dependent services such as MySQL, Elasticsearch, Redis, and MinIO. It automatically checks their health status, resource usage, and uptime, and performs restarts in case of failures to minimize downtime. +The service offers real-time monitoring of critical components, including the RAGFlow server, Task Executor processes, and dependent services such as MySQL, Infinity, Elasticsearch, Redis, and MinIO. It automatically checks their health status, resource usage, and uptime, and performs restarts in case of failures to minimize downtime. For user and system management, it supports listing, creating, modifying, and deleting users and their associated resources like knowledge bases and Agents. diff --git a/admin/server/auth.py b/admin/server/auth.py index 564c348e3..4217977a2 100644 --- a/admin/server/auth.py +++ b/admin/server/auth.py @@ -169,7 +169,7 @@ def login_verify(f): username = auth.parameters['username'] password = auth.parameters['password'] try: - if check_admin(username, password) is False: + if not check_admin(username, password): return jsonify({ "code": 500, "message": "Access denied", diff --git a/admin/server/config.py b/admin/server/config.py index e2c7d11ef..43f079d4f 100644 --- a/admin/server/config.py +++ b/admin/server/config.py @@ -25,8 +25,21 @@ from common.config_utils import read_config from urllib.parse import urlparse +class BaseConfig(BaseModel): + id: int + name: str + host: str + port: int + service_type: str + detail_func_name: str + + def to_dict(self) -> dict[str, Any]: + return {'id': self.id, 'name': self.name, 'host': self.host, 'port': self.port, + 'service_type': self.service_type} + + class ServiceConfigs: - configs = dict + configs = list[BaseConfig] def __init__(self): self.configs = [] @@ -45,19 +58,6 @@ class ServiceType(Enum): FILE_STORE = "file_store" -class BaseConfig(BaseModel): - id: int - name: str - host: str - port: int - service_type: str - detail_func_name: str - - def to_dict(self) -> dict[str, Any]: - return {'id': self.id, 'name': self.name, 'host': self.host, 'port': self.port, - 'service_type': self.service_type} - - class MetaConfig(BaseConfig): meta_type: str @@ -227,7 +227,7 @@ def load_configurations(config_path: str) -> list[BaseConfig]: ragflow_count = 0 id_count = 0 for k, v in raw_configs.items(): - match (k): + match k: case "ragflow": name: str = f'ragflow_{ragflow_count}' host: str = v['host'] diff --git a/admin/server/services.py b/admin/server/services.py index e8cf4eb5d..4dbbf011e 100644 --- a/admin/server/services.py +++ b/admin/server/services.py @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # - - +import logging import re from werkzeug.security import check_password_hash from common.constants import ActiveEnum @@ -190,7 +189,8 @@ class ServiceMgr: config_dict['status'] = service_detail['status'] else: config_dict['status'] = 'timeout' - except Exception: + except Exception as e: + logging.warning(f"Can't get service details, error: {e}") config_dict['status'] = 'timeout' if not config_dict['host']: config_dict['host'] = '-' @@ -205,17 +205,13 @@ class ServiceMgr: @staticmethod def get_service_details(service_id: int): - service_id = int(service_id) + service_idx = int(service_id) configs = SERVICE_CONFIGS.configs - service_config_mapping = { - c.id: { - 'name': c.name, - 'detail_func_name': c.detail_func_name - } for c in configs - } - service_info = service_config_mapping.get(service_id, {}) - if not service_info: - raise AdminException(f"invalid service_id: {service_id}") + if service_idx < 0 or service_idx >= len(configs): + raise AdminException(f"invalid service_index: {service_idx}") + + service_config = configs[service_idx] + service_info = {'name': service_config.name, 'detail_func_name': service_config.detail_func_name} detail_func = getattr(health_utils, service_info.get('detail_func_name')) res = detail_func() From 87e69868c0b1c817f2fd51f30a2737e6020a728b Mon Sep 17 00:00:00 2001 From: chanx <1243304602@qq.com> Date: Fri, 14 Nov 2025 13:56:56 +0800 Subject: [PATCH 05/10] Fixes: Added session variable types and modified configuration (#11269) ### What problem does this PR solve? Fixes: Added session variable types and modified configuration - Added more types of session variables - Modified the embedding model switching logic in the knowledge base configuration ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- web/src/components/dynamic-form.tsx | 152 ++++++++--- web/src/components/ui/segmented.tsx | 5 +- web/src/locales/en.ts | 1 + web/src/locales/zh.ts | 1 + .../component/add-variable-modal.tsx | 134 ++++++++++ .../{contant.ts => constant.ts} | 26 +- .../gobal-variable-sheet/hooks/use-form.tsx | 41 +++ .../hooks/use-object-fields.tsx | 246 ++++++++++++++++++ .../agent/gobal-variable-sheet/index.tsx | 188 ++++--------- web/src/pages/agent/hooks/use-build-dsl.ts | 10 +- web/src/pages/agent/hooks/use-save-graph.ts | 2 +- web/src/pages/agent/index.tsx | 18 +- web/src/pages/agent/utils.ts | 24 +- .../configuration/common-item.tsx | 46 +++- .../dataset/dataset-setting/general-form.tsx | 2 +- .../pages/dataset/dataset-setting/hooks.ts | 21 ++ web/src/services/knowledge-service.ts | 6 + web/src/utils/api.ts | 2 + 18 files changed, 712 insertions(+), 213 deletions(-) create mode 100644 web/src/pages/agent/gobal-variable-sheet/component/add-variable-modal.tsx rename web/src/pages/agent/gobal-variable-sheet/{contant.ts => constant.ts} (72%) create mode 100644 web/src/pages/agent/gobal-variable-sheet/hooks/use-form.tsx create mode 100644 web/src/pages/agent/gobal-variable-sheet/hooks/use-object-fields.tsx diff --git a/web/src/components/dynamic-form.tsx b/web/src/components/dynamic-form.tsx index f7449ec9f..a90afe287 100644 --- a/web/src/components/dynamic-form.tsx +++ b/web/src/components/dynamic-form.tsx @@ -61,6 +61,12 @@ export interface FormFieldConfig { horizontal?: boolean; onChange?: (value: any) => void; tooltip?: React.ReactNode; + customValidate?: ( + value: any, + formValues: any, + ) => string | boolean | Promise; + dependencies?: string[]; + schema?: ZodSchema; } // Component props interface @@ -94,36 +100,40 @@ const generateSchema = (fields: FormFieldConfig[]): ZodSchema => { let fieldSchema: ZodSchema; // Create base validation schema based on field type - switch (field.type) { - case FormFieldType.Email: - fieldSchema = z.string().email('Please enter a valid email address'); - break; - case FormFieldType.Number: - fieldSchema = z.coerce.number(); - if (field.validation?.min !== undefined) { - fieldSchema = (fieldSchema as z.ZodNumber).min( - field.validation.min, - field.validation.message || - `Value cannot be less than ${field.validation.min}`, - ); - } - if (field.validation?.max !== undefined) { - fieldSchema = (fieldSchema as z.ZodNumber).max( - field.validation.max, - field.validation.message || - `Value cannot be greater than ${field.validation.max}`, - ); - } - break; - case FormFieldType.Checkbox: - fieldSchema = z.boolean(); - break; - case FormFieldType.Tag: - fieldSchema = z.array(z.string()); - break; - default: - fieldSchema = z.string(); - break; + if (field.schema) { + fieldSchema = field.schema; + } else { + switch (field.type) { + case FormFieldType.Email: + fieldSchema = z.string().email('Please enter a valid email address'); + break; + case FormFieldType.Number: + fieldSchema = z.coerce.number(); + if (field.validation?.min !== undefined) { + fieldSchema = (fieldSchema as z.ZodNumber).min( + field.validation.min, + field.validation.message || + `Value cannot be less than ${field.validation.min}`, + ); + } + if (field.validation?.max !== undefined) { + fieldSchema = (fieldSchema as z.ZodNumber).max( + field.validation.max, + field.validation.message || + `Value cannot be greater than ${field.validation.max}`, + ); + } + break; + case FormFieldType.Checkbox: + fieldSchema = z.boolean(); + break; + case FormFieldType.Tag: + fieldSchema = z.array(z.string()); + break; + default: + fieldSchema = z.string(); + break; + } } // Handle required fields @@ -300,10 +310,90 @@ const DynamicForm = { // Initialize form const form = useForm({ - resolver: zodResolver(schema), + resolver: async (data, context, options) => { + const zodResult = await zodResolver(schema)(data, context, options); + + let combinedErrors = { ...zodResult.errors }; + + const fieldErrors: Record = + {}; + for (const field of fields) { + if (field.customValidate && data[field.name] !== undefined) { + try { + const result = await field.customValidate( + data[field.name], + data, + ); + if (typeof result === 'string') { + fieldErrors[field.name] = { + type: 'custom', + message: result, + }; + } else if (result === false) { + fieldErrors[field.name] = { + type: 'custom', + message: + field.validation?.message || `${field.label} is invalid`, + }; + } + } catch (error) { + fieldErrors[field.name] = { + type: 'custom', + message: + error instanceof Error + ? error.message + : 'Validation failed', + }; + } + } + } + + combinedErrors = { + ...combinedErrors, + ...fieldErrors, + } as any; + console.log('combinedErrors', combinedErrors); + return { + values: Object.keys(combinedErrors).length ? {} : data, + errors: combinedErrors, + } as any; + }, defaultValues, }); + useEffect(() => { + const dependencyMap: Record = {}; + + fields.forEach((field) => { + if (field.dependencies && field.dependencies.length > 0) { + field.dependencies.forEach((dep) => { + if (!dependencyMap[dep]) { + dependencyMap[dep] = []; + } + dependencyMap[dep].push(field.name); + }); + } + }); + + const subscriptions = Object.keys(dependencyMap).map((depField) => { + return form.watch((values: any, { name }) => { + if (name === depField && dependencyMap[depField]) { + dependencyMap[depField].forEach((dependentField) => { + form.trigger(dependentField as any); + }); + } + }); + }); + + return () => { + subscriptions.forEach((sub) => { + if (sub.unsubscribe) { + sub.unsubscribe(); + } + }); + }; + }, [fields, form]); + // Expose form methods via ref useImperativeHandle(ref, () => ({ submit: () => form.handleSubmit(onSubmit)(), diff --git a/web/src/components/ui/segmented.tsx b/web/src/components/ui/segmented.tsx index 8aadc3b21..3f9b0cc53 100644 --- a/web/src/components/ui/segmented.tsx +++ b/web/src/components/ui/segmented.tsx @@ -51,6 +51,7 @@ export interface SegmentedProps direction?: 'ltr' | 'rtl'; motionName?: string; activeClassName?: string; + itemClassName?: string; rounded?: keyof typeof segmentedVariants.round; sizeType?: keyof typeof segmentedVariants.size; buttonSize?: keyof typeof segmentedVariants.buttonSize; @@ -62,6 +63,7 @@ export function Segmented({ onChange, className, activeClassName, + itemClassName, rounded = 'default', sizeType = 'default', buttonSize = 'default', @@ -92,12 +94,13 @@ export function Segmented({
void; + visible?: boolean; + hideModal: () => void; + defaultValues?: FieldValues; + setDefaultValues?: (value: FieldValues) => void; +}) => { + const { + fields, + setFields, + visible, + hideModal, + defaultValues, + setDefaultValues, + } = props; + + const { handleSubmit: submitForm, loading } = useHandleForm(); + + const { handleCustomValidate, handleCustomSchema, handleRender } = + useObjectFields(); + + const formRef = useRef(null); + + const handleFieldUpdate = ( + fieldName: string, + updatedField: Partial, + ) => { + setFields((prevFields: any) => + prevFields.map((field: any) => + field.name === fieldName ? { ...field, ...updatedField } : field, + ), + ); + }; + + useEffect(() => { + const typeField = fields?.find((item) => item.name === 'type'); + + if (typeField) { + typeField.onChange = (value) => { + handleFieldUpdate('value', { + type: TypeMaps[value as keyof typeof TypeMaps], + render: handleRender(value), + customValidate: handleCustomValidate(value), + schema: handleCustomSchema(value), + }); + const values = formRef.current?.getValues(); + // setTimeout(() => { + switch (value) { + case TypesWithArray.Boolean: + setDefaultValues?.({ ...values, value: false }); + break; + case TypesWithArray.Number: + setDefaultValues?.({ ...values, value: 0 }); + break; + case TypesWithArray.Object: + setDefaultValues?.({ ...values, value: {} }); + break; + case TypesWithArray.ArrayString: + setDefaultValues?.({ ...values, value: [''] }); + break; + case TypesWithArray.ArrayNumber: + setDefaultValues?.({ ...values, value: [''] }); + break; + case TypesWithArray.ArrayBoolean: + setDefaultValues?.({ ...values, value: [false] }); + break; + case TypesWithArray.ArrayObject: + setDefaultValues?.({ ...values, value: [] }); + break; + default: + setDefaultValues?.({ ...values, value: '' }); + break; + } + // }, 0); + }; + } + }, [fields]); + + const handleSubmit = async (fieldValue: FieldValues) => { + await submitForm(fieldValue); + hideModal(); + }; + + return ( + + { + console.log(data); + }} + defaultValues={defaultValues} + onFieldUpdate={handleFieldUpdate} + > +
+ { + hideModal?.(); + }} + /> + { + handleSubmit(values); + // console.log(values); + // console.log(nodes, edges); + // handleOk(values); + }} + /> +
+ + + ); +}; diff --git a/web/src/pages/agent/gobal-variable-sheet/contant.ts b/web/src/pages/agent/gobal-variable-sheet/constant.ts similarity index 72% rename from web/src/pages/agent/gobal-variable-sheet/contant.ts rename to web/src/pages/agent/gobal-variable-sheet/constant.ts index 2f3bd395f..fc668e330 100644 --- a/web/src/pages/agent/gobal-variable-sheet/contant.ts +++ b/web/src/pages/agent/gobal-variable-sheet/constant.ts @@ -13,14 +13,14 @@ export enum TypesWithArray { String = 'string', Number = 'number', Boolean = 'boolean', - // Object = 'object', - // ArrayString = 'array', - // ArrayNumber = 'array', - // ArrayBoolean = 'array', - // ArrayObject = 'array', + Object = 'object', + ArrayString = 'array', + ArrayNumber = 'array', + ArrayBoolean = 'array', + ArrayObject = 'array', } -export const GobalFormFields = [ +export const GlobalFormFields = [ { label: t('flow.name'), name: 'name', @@ -50,11 +50,11 @@ export const GobalFormFields = [ label: t('flow.description'), name: 'description', placeholder: t('flow.variableDescription'), - type: 'textarea', + type: FormFieldType.Textarea, }, ] as FormFieldConfig[]; -export const GobalVariableFormDefaultValues = { +export const GlobalVariableFormDefaultValues = { name: '', type: TypesWithArray.String, value: '', @@ -65,9 +65,9 @@ export const TypeMaps = { [TypesWithArray.String]: FormFieldType.Textarea, [TypesWithArray.Number]: FormFieldType.Number, [TypesWithArray.Boolean]: FormFieldType.Checkbox, - // [TypesWithArray.Object]: FormFieldType.Textarea, - // [TypesWithArray.ArrayString]: FormFieldType.Textarea, - // [TypesWithArray.ArrayNumber]: FormFieldType.Textarea, - // [TypesWithArray.ArrayBoolean]: FormFieldType.Textarea, - // [TypesWithArray.ArrayObject]: FormFieldType.Textarea, + [TypesWithArray.Object]: FormFieldType.Textarea, + [TypesWithArray.ArrayString]: FormFieldType.Textarea, + [TypesWithArray.ArrayNumber]: FormFieldType.Textarea, + [TypesWithArray.ArrayBoolean]: FormFieldType.Textarea, + [TypesWithArray.ArrayObject]: FormFieldType.Textarea, }; diff --git a/web/src/pages/agent/gobal-variable-sheet/hooks/use-form.tsx b/web/src/pages/agent/gobal-variable-sheet/hooks/use-form.tsx new file mode 100644 index 000000000..cb38012f3 --- /dev/null +++ b/web/src/pages/agent/gobal-variable-sheet/hooks/use-form.tsx @@ -0,0 +1,41 @@ +import { useFetchAgent } from '@/hooks/use-agent-request'; +import { GlobalVariableType } from '@/interfaces/database/agent'; +import { useCallback } from 'react'; +import { FieldValues } from 'react-hook-form'; +import { useSaveGraph } from '../../hooks/use-save-graph'; +import { TypesWithArray } from '../constant'; + +export const useHandleForm = () => { + const { data, refetch } = useFetchAgent(); + const { saveGraph, loading } = useSaveGraph(); + const handleObjectData = (value: any) => { + try { + return JSON.parse(value); + } catch (error) { + return value; + } + }; + const handleSubmit = useCallback(async (fieldValue: FieldValues) => { + const param = { + ...(data.dsl?.variables || {}), + [fieldValue.name]: { + ...fieldValue, + value: + fieldValue.type === TypesWithArray.Object || + fieldValue.type === TypesWithArray.ArrayObject + ? handleObjectData(fieldValue.value) + : fieldValue.value, + }, + } as Record; + + const res = await saveGraph(undefined, { + globalVariables: param, + }); + + if (res.code === 0) { + refetch(); + } + }, []); + + return { handleSubmit, loading }; +}; diff --git a/web/src/pages/agent/gobal-variable-sheet/hooks/use-object-fields.tsx b/web/src/pages/agent/gobal-variable-sheet/hooks/use-object-fields.tsx new file mode 100644 index 000000000..d8600d568 --- /dev/null +++ b/web/src/pages/agent/gobal-variable-sheet/hooks/use-object-fields.tsx @@ -0,0 +1,246 @@ +import { BlockButton, Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { Segmented } from '@/components/ui/segmented'; +import { Editor } from '@monaco-editor/react'; +import { t } from 'i18next'; +import { Trash2, X } from 'lucide-react'; +import { useCallback } from 'react'; +import { FieldValues } from 'react-hook-form'; +import { z } from 'zod'; +import { TypesWithArray } from '../constant'; + +export const useObjectFields = () => { + const booleanRender = useCallback( + (field: FieldValues, className?: string) => { + const fieldValue = field.value ? true : false; + return ( + + ); + }, + [], + ); + + const objectRender = useCallback((field: FieldValues) => { + const fieldValue = + typeof field.value === 'object' + ? JSON.stringify(field.value, null, 2) + : JSON.stringify({}, null, 2); + console.log('object-render-field', field, fieldValue); + return ( + + ); + }, []); + + const objectValidate = useCallback((value: any) => { + try { + if (!JSON.parse(value)) { + throw new Error(t('knowledgeDetails.formatTypeError')); + } + return true; + } catch (e) { + throw new Error(t('knowledgeDetails.formatTypeError')); + } + }, []); + + const arrayStringRender = useCallback((field: FieldValues, type = 'text') => { + const values = Array.isArray(field.value) + ? field.value + : [type === 'number' ? 0 : '']; + return ( + <> + {values?.map((item: any, index: number) => ( +
+ { + const newValues = [...values]; + newValues[index] = e.target.value; + field.onChange(newValues); + }} + /> + +
+ ))} + { + field.onChange([...field.value, '']); + }} + > + {t('flow.add')} + + + ); + }, []); + + const arrayBooleanRender = useCallback( + (field: FieldValues) => { + // const values = field.value || [false]; + const values = Array.isArray(field.value) ? field.value : [false]; + return ( +
+ {values?.map((item: any, index: number) => ( +
+ {booleanRender( + { + value: item, + onChange: (value) => { + values[index] = !!value; + field.onChange(values); + }, + }, + 'bg-transparent', + )} + +
+ ))} + { + field.onChange([...field.value, false]); + }} + > + {t('flow.add')} + +
+ ); + }, + [booleanRender], + ); + + const arrayNumberRender = useCallback( + (field: FieldValues) => { + return arrayStringRender(field, 'number'); + }, + [arrayStringRender], + ); + + const arrayValidate = useCallback((value: any, type: string = 'string') => { + if (!Array.isArray(value) || !value.every((item) => typeof item === type)) { + throw new Error(t('flow.formatTypeError')); + } + return true; + }, []); + + const arrayStringValidate = useCallback( + (value: any) => { + return arrayValidate(value, 'string'); + }, + [arrayValidate], + ); + + const arrayNumberValidate = useCallback( + (value: any) => { + return arrayValidate(value, 'number'); + }, + [arrayValidate], + ); + + const arrayBooleanValidate = useCallback( + (value: any) => { + return arrayValidate(value, 'boolean'); + }, + [arrayValidate], + ); + + const handleRender = (value: TypesWithArray) => { + switch (value) { + case TypesWithArray.Boolean: + return booleanRender; + case TypesWithArray.Object: + case TypesWithArray.ArrayObject: + return objectRender; + case TypesWithArray.ArrayString: + return arrayStringRender; + case TypesWithArray.ArrayNumber: + return arrayNumberRender; + case TypesWithArray.ArrayBoolean: + return arrayBooleanRender; + default: + return undefined; + } + }; + const handleCustomValidate = (value: TypesWithArray) => { + switch (value) { + case TypesWithArray.Object: + case TypesWithArray.ArrayObject: + return objectValidate; + case TypesWithArray.ArrayString: + return arrayStringValidate; + case TypesWithArray.ArrayNumber: + return arrayNumberValidate; + case TypesWithArray.ArrayBoolean: + return arrayBooleanValidate; + default: + return undefined; + } + }; + const handleCustomSchema = (value: TypesWithArray) => { + switch (value) { + case TypesWithArray.ArrayString: + return z.array(z.string()); + case TypesWithArray.ArrayNumber: + return z.array(z.number()); + case TypesWithArray.ArrayBoolean: + return z.array(z.boolean()); + default: + return undefined; + } + }; + return { + objectRender, + objectValidate, + arrayStringRender, + arrayStringValidate, + arrayNumberRender, + booleanRender, + arrayBooleanRender, + arrayNumberValidate, + arrayBooleanValidate, + handleRender, + handleCustomValidate, + handleCustomSchema, + }; +}; diff --git a/web/src/pages/agent/gobal-variable-sheet/index.tsx b/web/src/pages/agent/gobal-variable-sheet/index.tsx index 454131638..51648b8d1 100644 --- a/web/src/pages/agent/gobal-variable-sheet/index.tsx +++ b/web/src/pages/agent/gobal-variable-sheet/index.tsx @@ -1,12 +1,6 @@ import { ConfirmDeleteDialog } from '@/components/confirm-delete-dialog'; -import { - DynamicForm, - DynamicFormRef, - FormFieldConfig, - FormFieldType, -} from '@/components/dynamic-form'; +import { FormFieldConfig } from '@/components/dynamic-form'; import { BlockButton, Button } from '@/components/ui/button'; -import { Modal } from '@/components/ui/modal/modal'; import { Sheet, SheetContent, @@ -19,117 +13,65 @@ import { GlobalVariableType } from '@/interfaces/database/agent'; import { cn } from '@/lib/utils'; import { t } from 'i18next'; import { Trash2 } from 'lucide-react'; -import { useEffect, useRef, useState } from 'react'; +import { useState } from 'react'; import { FieldValues } from 'react-hook-form'; import { useSaveGraph } from '../hooks/use-save-graph'; +import { AddVariableModal } from './component/add-variable-modal'; import { - GobalFormFields, - GobalVariableFormDefaultValues, + GlobalFormFields, + GlobalVariableFormDefaultValues, TypeMaps, TypesWithArray, -} from './contant'; +} from './constant'; +import { useObjectFields } from './hooks/use-object-fields'; -export type IGobalParamModalProps = { +export type IGlobalParamModalProps = { data: any; hideModal: (open: boolean) => void; }; -export const GobalParamSheet = (props: IGobalParamModalProps) => { +export const GlobalParamSheet = (props: IGlobalParamModalProps) => { const { hideModal } = props; const { data, refetch } = useFetchAgent(); - const [fields, setFields] = useState(GobalFormFields); const { visible, showModal, hideModal: hideAddModal } = useSetModalState(); + const [fields, setFields] = useState(GlobalFormFields); const [defaultValues, setDefaultValues] = useState( - GobalVariableFormDefaultValues, + GlobalVariableFormDefaultValues, ); - const formRef = useRef(null); + const { handleCustomValidate, handleCustomSchema, handleRender } = + useObjectFields(); + const { saveGraph } = useSaveGraph(); - const handleFieldUpdate = ( - fieldName: string, - updatedField: Partial, - ) => { - setFields((prevFields) => - prevFields.map((field) => - field.name === fieldName ? { ...field, ...updatedField } : field, - ), - ); - }; - - useEffect(() => { - const typefileld = fields.find((item) => item.name === 'type'); - - if (typefileld) { - typefileld.onChange = (value) => { - // setWatchType(value); - handleFieldUpdate('value', { - type: TypeMaps[value as keyof typeof TypeMaps], - }); - const values = formRef.current?.getValues(); - setTimeout(() => { - switch (value) { - case TypesWithArray.Boolean: - setDefaultValues({ ...values, value: false }); - break; - case TypesWithArray.Number: - setDefaultValues({ ...values, value: 0 }); - break; - default: - setDefaultValues({ ...values, value: '' }); - } - }, 0); - }; - } - }, [fields]); - - const { saveGraph, loading } = useSaveGraph(); - - const handleSubmit = async (value: FieldValues) => { - const param = { - ...(data.dsl?.variables || {}), - [value.name]: value, - } as Record; - - const res = await saveGraph(undefined, { - gobalVariables: param, - }); - - if (res.code === 0) { - refetch(); - } - hideAddModal(); - }; - - const handleDeleteGobalVariable = async (key: string) => { + const handleDeleteGlobalVariable = async (key: string) => { const param = { ...(data.dsl?.variables || {}), } as Record; delete param[key]; const res = await saveGraph(undefined, { - gobalVariables: param, + globalVariables: param, }); - console.log('delete gobal variable-->', res); if (res.code === 0) { refetch(); } }; - const handleEditGobalVariable = (item: FieldValues) => { - fields.forEach((field) => { - if (field.name === 'value') { - switch (item.type) { - // [TypesWithArray.String]: FormFieldType.Textarea, - // [TypesWithArray.Number]: FormFieldType.Number, - // [TypesWithArray.Boolean]: FormFieldType.Checkbox, - case TypesWithArray.Boolean: - field.type = FormFieldType.Checkbox; - break; - case TypesWithArray.Number: - field.type = FormFieldType.Number; - break; - default: - field.type = FormFieldType.Textarea; - } + const handleEditGlobalVariable = (item: FieldValues) => { + const newFields = fields.map((field) => { + let newField = field; + newField.render = undefined; + newField.schema = undefined; + newField.customValidate = undefined; + if (newField.name === 'value') { + newField = { + ...newField, + type: TypeMaps[item.type as keyof typeof TypeMaps], + render: handleRender(item.type), + customValidate: handleCustomValidate(item.type), + schema: handleCustomSchema(item.type), + }; } + return newField; }); + setFields(newFields); setDefaultValues(item); showModal(); }; @@ -149,8 +91,8 @@ export const GobalParamSheet = (props: IGobalParamModalProps) => {
{ - setFields(GobalFormFields); - setDefaultValues(GobalVariableFormDefaultValues); + setFields(GlobalFormFields); + setDefaultValues(GlobalVariableFormDefaultValues); showModal(); }} > @@ -167,7 +109,7 @@ export const GobalParamSheet = (props: IGobalParamModalProps) => { key={key} className="flex items-center gap-3 min-h-14 justify-between px-5 py-3 border border-border-default rounded-lg hover:bg-bg-card group" onClick={() => { - handleEditGobalVariable(item); + handleEditGlobalVariable(item); }} >
@@ -177,13 +119,23 @@ export const GobalParamSheet = (props: IGobalParamModalProps) => { {item.type}
-
- {item.value} -
+ {![ + TypesWithArray.Object, + TypesWithArray.ArrayObject, + TypesWithArray.ArrayString, + TypesWithArray.ArrayNumber, + TypesWithArray.ArrayBoolean, + ].includes(item.type as TypesWithArray) && ( +
+ + {item.value} + +
+ )}
handleDeleteGobalVariable(key)} + onOk={() => handleDeleteGlobalVariable(key)} >
- - { - console.log(data); - }} - defaultValues={defaultValues} - onFieldUpdate={handleFieldUpdate} - > -
- { - hideAddModal?.(); - }} - /> - { - handleSubmit(values); - // console.log(values); - // console.log(nodes, edges); - // handleOk(values); - }} - /> -
- - + ); diff --git a/web/src/pages/agent/hooks/use-build-dsl.ts b/web/src/pages/agent/hooks/use-build-dsl.ts index 1a8569636..47ec1c225 100644 --- a/web/src/pages/agent/hooks/use-build-dsl.ts +++ b/web/src/pages/agent/hooks/use-build-dsl.ts @@ -4,7 +4,7 @@ import { RAGFlowNodeType } from '@/interfaces/database/flow'; import { useCallback } from 'react'; import { Operator } from '../constant'; import useGraphStore from '../store'; -import { buildDslComponentsByGraph, buildDslGobalVariables } from '../utils'; +import { buildDslComponentsByGraph, buildDslGlobalVariables } from '../utils'; export const useBuildDslData = () => { const { data } = useFetchAgent(); @@ -13,7 +13,7 @@ export const useBuildDslData = () => { const buildDslData = useCallback( ( currentNodes?: RAGFlowNodeType[], - otherParam?: { gobalVariables: Record }, + otherParam?: { globalVariables: Record }, ) => { const nodesToProcess = currentNodes ?? nodes; @@ -41,13 +41,13 @@ export const useBuildDslData = () => { data.dsl.components, ); - const gobalVariables = buildDslGobalVariables( + const globalVariables = buildDslGlobalVariables( data.dsl, - otherParam?.gobalVariables, + otherParam?.globalVariables, ); return { ...data.dsl, - ...gobalVariables, + ...globalVariables, graph: { nodes: filteredNodes, edges: filteredEdges }, components: dslComponents, }; diff --git a/web/src/pages/agent/hooks/use-save-graph.ts b/web/src/pages/agent/hooks/use-save-graph.ts index e59b99193..500baf716 100644 --- a/web/src/pages/agent/hooks/use-save-graph.ts +++ b/web/src/pages/agent/hooks/use-save-graph.ts @@ -21,7 +21,7 @@ export const useSaveGraph = (showMessage: boolean = true) => { const saveGraph = useCallback( async ( currentNodes?: RAGFlowNodeType[], - otherParam?: { gobalVariables: Record }, + otherParam?: { globalVariables: Record }, ) => { return setAgent({ id, diff --git a/web/src/pages/agent/index.tsx b/web/src/pages/agent/index.tsx index 21ecb22e7..b0d2f6f15 100644 --- a/web/src/pages/agent/index.tsx +++ b/web/src/pages/agent/index.tsx @@ -39,7 +39,7 @@ import { useParams } from 'umi'; import AgentCanvas from './canvas'; import { DropdownProvider } from './canvas/context'; import { Operator } from './constant'; -import { GobalParamSheet } from './gobal-variable-sheet'; +import { GlobalParamSheet } from './gobal-variable-sheet'; import { useCancelCurrentDataflow } from './hooks/use-cancel-dataflow'; import { useHandleExportJsonFile } from './hooks/use-export-json'; import { useFetchDataOnMount } from './hooks/use-fetch-data'; @@ -126,9 +126,9 @@ export default function Agent() { } = useSetModalState(); const { - visible: gobalParamSheetVisible, - showModal: showGobalParamSheet, - hideModal: hideGobalParamSheet, + visible: globalParamSheetVisible, + showModal: showGlobalParamSheet, + hideModal: hideGlobalParamSheet, } = useSetModalState(); const { @@ -216,7 +216,7 @@ export default function Agent() { showGobalParamSheet()} + onClick={() => showGlobalParamSheet()} loading={loading} > {t('flow.conversationVariable')} @@ -314,11 +314,11 @@ export default function Agent() { loading={pipelineRunning} > )} - {gobalParamSheetVisible && ( - + hideModal={hideGlobalParamSheet} + > )} ); diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts index 487067ed8..3312b7236 100644 --- a/web/src/pages/agent/utils.ts +++ b/web/src/pages/agent/utils.ts @@ -348,30 +348,30 @@ export const buildDslComponentsByGraph = ( return components; }; -export const buildDslGobalVariables = ( +export const buildDslGlobalVariables = ( dsl: DSL, - gobalVariables?: Record, + globalVariables?: Record, ) => { - if (!gobalVariables) { + if (!globalVariables) { return { globals: dsl.globals, variables: dsl.variables || {} }; } - let gobalVariablesTemp: Record = {}; - let gobalSystem: Record = {}; + let globalVariablesTemp: Record = {}; + let globalSystem: Record = {}; Object.keys(dsl.globals)?.forEach((key) => { if (key.indexOf('sys') > -1) { - gobalSystem[key] = dsl.globals[key]; + globalSystem[key] = dsl.globals[key]; } }); - Object.keys(gobalVariables).forEach((key) => { - gobalVariablesTemp['env.' + key] = gobalVariables[key].value; + Object.keys(globalVariables).forEach((key) => { + globalVariablesTemp['env.' + key] = globalVariables[key].value; }); - const gobalVariablesResult = { - ...gobalSystem, - ...gobalVariablesTemp, + const globalVariablesResult = { + ...globalSystem, + ...globalVariablesTemp, }; - return { globals: gobalVariablesResult, variables: gobalVariables }; + return { globals: globalVariablesResult, variables: globalVariables }; }; export const receiveMessageError = (res: any) => diff --git a/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx b/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx index c63309c50..c6d18af13 100644 --- a/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx +++ b/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx @@ -7,11 +7,14 @@ import { FormMessage, } from '@/components/ui/form'; import { Radio } from '@/components/ui/radio'; +import { Spin } from '@/components/ui/spin'; import { Switch } from '@/components/ui/switch'; import { useTranslate } from '@/hooks/common-hooks'; import { cn } from '@/lib/utils'; +import { useMemo, useState } from 'react'; import { useFormContext } from 'react-hook-form'; import { + useHandleKbEmbedding, useHasParsedDocument, useSelectChunkMethodList, useSelectEmbeddingModelOptions, @@ -62,11 +65,17 @@ export function ChunkMethodItem(props: IProps) { /> ); } -export function EmbeddingModelItem({ line = 1, isEdit = true }: IProps) { +export function EmbeddingModelItem({ line = 1, isEdit }: IProps) { const { t } = useTranslate('knowledgeConfiguration'); const form = useFormContext(); const embeddingModelOptions = useSelectEmbeddingModelOptions(); + const { handleChange } = useHandleKbEmbedding(); const disabled = useHasParsedDocument(isEdit); + const oldValue = useMemo(() => { + const embdStr = form.getValues('embd_id'); + return embdStr || ''; + }, [form]); + const [loading, setLoading] = useState(false); return ( <> - + + { + field.onChange(value); + if (isEdit && disabled) { + setLoading(true); + const res = await handleChange({ + embed_id: value, + callback: field.onChange, + }); + if (res.code !== 0) { + field.onChange(oldValue); + } + setLoading(false); + } + }} + value={field.value} + options={embeddingModelOptions} + placeholder={t('embeddingModelPlaceholder')} + triggerClassName="!bg-bg-base" + /> + diff --git a/web/src/pages/dataset/dataset-setting/general-form.tsx b/web/src/pages/dataset/dataset-setting/general-form.tsx index b4a7b9635..110c03a3e 100644 --- a/web/src/pages/dataset/dataset-setting/general-form.tsx +++ b/web/src/pages/dataset/dataset-setting/general-form.tsx @@ -88,7 +88,7 @@ export function GeneralForm() { }} /> - + diff --git a/web/src/pages/dataset/dataset-setting/hooks.ts b/web/src/pages/dataset/dataset-setting/hooks.ts index 605f91e4d..f9efe1d08 100644 --- a/web/src/pages/dataset/dataset-setting/hooks.ts +++ b/web/src/pages/dataset/dataset-setting/hooks.ts @@ -4,10 +4,12 @@ import { useSetModalState } from '@/hooks/common-hooks'; import { useSelectLlmOptionsByModelType } from '@/hooks/llm-hooks'; import { useFetchKnowledgeBaseConfiguration } from '@/hooks/use-knowledge-request'; import { useSelectParserList } from '@/hooks/user-setting-hooks'; +import kbService from '@/services/knowledge-service'; import { useIsFetching } from '@tanstack/react-query'; import { pick } from 'lodash'; import { useCallback, useEffect, useState } from 'react'; import { UseFormReturn } from 'react-hook-form'; +import { useParams, useSearchParams } from 'umi'; import { z } from 'zod'; import { formSchema } from './form-schema'; @@ -98,3 +100,22 @@ export const useRenameKnowledgeTag = () => { showTagRenameModal: handleShowTagRenameModal, }; }; + +export const useHandleKbEmbedding = () => { + const { id } = useParams(); + const [searchParams] = useSearchParams(); + const knowledgeBaseId = searchParams.get('id') || id; + const handleChange = useCallback( + async ({ embed_id }: { embed_id: string }) => { + const res = await kbService.checkEmbedding({ + kb_id: knowledgeBaseId, + embd_id: embed_id, + }); + return res.data; + }, + [knowledgeBaseId], + ); + return { + handleChange, + }; +}; diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.ts index 350fa4e2a..01b8da127 100644 --- a/web/src/services/knowledge-service.ts +++ b/web/src/services/knowledge-service.ts @@ -47,6 +47,7 @@ const { traceGraphRag, runRaptor, traceRaptor, + check_embedding, } = api; const methods = { @@ -214,6 +215,11 @@ const methods = { url: api.pipelineRerun, method: 'post', }, + + checkEmbedding: { + url: check_embedding, + method: 'post', + }, }; const kbService = registerServer(methods, request); diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts index 0d97801ac..e0afdbeb3 100644 --- a/web/src/utils/api.ts +++ b/web/src/utils/api.ts @@ -49,6 +49,8 @@ export default { llm_tools: `${api_host}/plugin/llm_tools`, // knowledge base + + check_embedding: `${api_host}/kb/check_embedding`, kb_list: `${api_host}/kb/list`, create_kb: `${api_host}/kb/create`, update_kb: `${api_host}/kb/update`, From 5f59418ababc619aa61244dba6772dca424c507b Mon Sep 17 00:00:00 2001 From: redredrrred <1589289338@qq.com> Date: Fri, 14 Nov 2025 13:59:03 +0800 Subject: [PATCH 06/10] Remove leftover account and password from the code (#11248) Remove legacy accounts and passwords. ### What problem does this PR solve? Remove leftover account and password in agent/templates/sql_assistant.json ### Type of change - [x] Other (please describe): --- agent/templates/sql_assistant.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/agent/templates/sql_assistant.json b/agent/templates/sql_assistant.json index 92804abc6..6e7140196 100644 --- a/agent/templates/sql_assistant.json +++ b/agent/templates/sql_assistant.json @@ -83,10 +83,10 @@ "value": [] } }, - "password": "20010812Yy!", + "password": "", "port": 3306, "sql": "{Agent:WickedGoatsDivide@content}", - "username": "13637682833@163.com" + "username": "" } }, "upstream": [ @@ -527,10 +527,10 @@ "value": [] } }, - "password": "20010812Yy!", + "password": "", "port": 3306, "sql": "{Agent:WickedGoatsDivide@content}", - "username": "13637682833@163.com" + "username": "" }, "label": "ExeSQL", "name": "ExeSQL" From e27ff8d3d42ce726941f8494a1a428ebe76587de Mon Sep 17 00:00:00 2001 From: Billy Bao Date: Fri, 14 Nov 2025 13:59:54 +0800 Subject: [PATCH 07/10] Fix: rerank algorithm (#11266) ### What problem does this PR solve? Fix: rerank algorithm #11234 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- rag/nlp/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rag/nlp/search.py b/rag/nlp/search.py index f8b3d513f..4dbd9945c 100644 --- a/rag/nlp/search.py +++ b/rag/nlp/search.py @@ -347,7 +347,7 @@ class Dealer: ## For rank feature(tag_fea) scores. rank_fea = self._rank_feature_scores(rank_feature, sres) - return tkweight * (np.array(tksim)+rank_fea) + vtweight * vtsim, tksim, vtsim + return tkweight * np.array(tksim) + vtweight * vtsim + rank_fea, tksim, vtsim def hybrid_similarity(self, ans_embd, ins_embd, ans, inst): return self.qryr.hybrid_similarity(ans_embd, From b5f2cf16bcad7b1f9f9f10ff11323352680d02ff Mon Sep 17 00:00:00 2001 From: Lynn Date: Fri, 14 Nov 2025 15:52:28 +0800 Subject: [PATCH 08/10] Fix: check task executor alive and display status (#11270) ### What problem does this PR solve? Correctly check task executor alive and display status. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- admin/client/admin_client.py | 9 ++++++--- api/utils/health_utils.py | 3 ++- rag/utils/redis_conn.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/admin/client/admin_client.py b/admin/client/admin_client.py index b52e67494..0d04cb3b2 100644 --- a/admin/client/admin_client.py +++ b/admin/client/admin_client.py @@ -393,7 +393,9 @@ class AdminCLI(Cmd): print(f"Can't access {self.host}, port: {self.port}") def _format_service_detail_table(self, data): - if not any([isinstance(v, list) for v in data.values()]): + if isinstance(data, list): + return data + if not all([isinstance(v, list) for v in data.values()]): # normal table return data # handle task_executor heartbeats map, for example {'name': [{'done': 2, 'now': timestamp1}, {'done': 3, 'now': timestamp2}] @@ -404,7 +406,7 @@ class AdminCLI(Cmd): task_executor_list.append({ "task_executor_name": k, **heartbeats[0], - }) + } if heartbeats else {"task_executor_name": k}) return task_executor_list def _print_table_simple(self, data): @@ -415,7 +417,8 @@ class AdminCLI(Cmd): # handle single row data data = [data] - columns = list(data[0].keys()) + columns = list(set().union(*(d.keys() for d in data))) + columns.sort() col_widths = {} def get_string_width(text): diff --git a/api/utils/health_utils.py b/api/utils/health_utils.py index 88e5aaebb..0a7ab6e7a 100644 --- a/api/utils/health_utils.py +++ b/api/utils/health_utils.py @@ -173,7 +173,8 @@ def check_task_executor_alive(): heartbeats = [json.loads(heartbeat) for heartbeat in heartbeats] task_executor_heartbeats[task_executor_id] = heartbeats if task_executor_heartbeats: - return {"status": "alive", "message": task_executor_heartbeats} + status = "alive" if any(task_executor_heartbeats.values()) else "timeout" + return {"status": status, "message": task_executor_heartbeats} else: return {"status": "timeout", "message": "Not found any task executor."} except Exception as e: diff --git a/rag/utils/redis_conn.py b/rag/utils/redis_conn.py index 58b0fe15b..a8bc43b57 100644 --- a/rag/utils/redis_conn.py +++ b/rag/utils/redis_conn.py @@ -110,7 +110,7 @@ class RedisDB: info = self.REDIS.info() return { 'redis_version': info["redis_version"], - 'server_mode': info["server_mode"], + 'server_mode': info["server_mode"] if "server_mode" in info else info.get("redis_mode", ""), 'used_memory': info["used_memory_human"], 'total_system_memory': info["total_system_memory_human"], 'mem_fragmentation_ratio': info["mem_fragmentation_ratio"], From 12db62b9c736c8b9efba3ef58fa7151a0c50099b Mon Sep 17 00:00:00 2001 From: Stephen Hu <812791840@qq.com> Date: Fri, 14 Nov 2025 16:32:35 +0800 Subject: [PATCH 09/10] Refactor: improve mineru_parser get property logic (#11268) ### What problem does this PR solve? improve mineru_parser get property logic ### Type of change - [x] Refactoring --- deepdoc/parser/mineru_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py index 3d4c9f149..bb663de0d 100644 --- a/deepdoc/parser/mineru_parser.py +++ b/deepdoc/parser/mineru_parser.py @@ -434,7 +434,7 @@ class MinerUParser(RAGFlowPdfParser): if not section.strip(): section = "FAILED TO PARSE TABLE" case MinerUContentType.IMAGE: - section = "".join(output["image_caption"]) + "\n" + "".join(output["image_footnote"]) + section = "".join(output.get(["image_caption"],[])) + "\n" + "".join(output.get(["image_footnote"],[])) case MinerUContentType.EQUATION: section = output["text"] case MinerUContentType.CODE: From db4fd19c8269a64f8d213a64c37f41e5325f22cd Mon Sep 17 00:00:00 2001 From: buua436 <66937541+buua436@users.noreply.github.com> Date: Fri, 14 Nov 2025 16:33:20 +0800 Subject: [PATCH 10/10] Feat:new component list operations (#11276) ### What problem does this PR solve? issue: https://github.com/infiniflow/ragflow/issues/10427 change: new component list operations ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- agent/component/list_operations.py | 149 ++++++++++++++++++ web/src/constants/agent.tsx | 1 + web/src/locales/en.ts | 15 ++ web/src/locales/zh.ts | 15 ++ web/src/pages/agent/canvas/index.tsx | 2 + .../node/dropdown/accordion-operators.tsx | 1 + .../canvas/node/list-operations-node.tsx | 22 +++ web/src/pages/agent/constant/index.tsx | 31 ++++ .../agent/form-sheet/form-config-map.tsx | 4 + .../agent/form/list-operations-form/index.tsx | 140 ++++++++++++++++ web/src/pages/agent/hooks/use-add-node.ts | 2 + web/src/pages/agent/operator-icon.tsx | 3 +- web/src/pages/agent/utils.ts | 1 - 13 files changed, 384 insertions(+), 2 deletions(-) create mode 100644 agent/component/list_operations.py create mode 100644 web/src/pages/agent/canvas/node/list-operations-node.tsx create mode 100644 web/src/pages/agent/form/list-operations-form/index.tsx diff --git a/agent/component/list_operations.py b/agent/component/list_operations.py new file mode 100644 index 000000000..c29d79ea6 --- /dev/null +++ b/agent/component/list_operations.py @@ -0,0 +1,149 @@ +from abc import ABC +import os +from agent.component.base import ComponentBase, ComponentParamBase +from api.utils.api_utils import timeout + +class ListOperationsParam(ComponentParamBase): + """ + Define the List Operations component parameters. + """ + def __init__(self): + super().__init__() + self.query = "" + self.operations = "topN" + self.n=0 + self.sort_method = "asc" + self.filter = { + "operator": "=", + "value": "" + } + self.outputs = { + "result": { + "value": [], + "type": "Array of ?" + }, + "first": { + "value": "", + "type": "?" + }, + "last": { + "value": "", + "type": "?" + } + } + + def check(self): + self.check_empty(self.query, "query") + self.check_valid_value(self.operations, "Support operations", ["topN","head","tail","filter","sort","drop_duplicates"]) + + def get_input_form(self) -> dict[str, dict]: + return {} + + +class ListOperations(ComponentBase,ABC): + component_name = "ListOperations" + + @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60))) + def _invoke(self, **kwargs): + self.input_objects=[] + inputs = getattr(self._param, "query", None) + self.inputs=self._canvas.get_variable_value(inputs) + self.set_input_value(inputs, self.inputs) + if self._param.operations == "topN": + self._topN() + elif self._param.operations == "head": + self._head() + elif self._param.operations == "tail": + self._tail() + elif self._param.operations == "filter": + self._filter() + elif self._param.operations == "sort": + self._sort() + elif self._param.operations == "drop_duplicates": + self._drop_duplicates() + + + def _coerce_n(self): + try: + return int(getattr(self._param, "n", 0)) + except Exception: + return 0 + + def _set_outputs(self, outputs): + self._param.outputs["result"]["value"] = outputs + self._param.outputs["first"]["value"] = outputs[0] if outputs else None + self._param.outputs["last"]["value"] = outputs[-1] if outputs else None + + def _topN(self): + n = self._coerce_n() + if n < 1: + outputs = [] + else: + n = min(n, len(self.inputs)) + outputs = self.inputs[:n] + self._set_outputs(outputs) + + def _head(self): + n = self._coerce_n() + if 1 <= n <= len(self.inputs): + outputs = [self.inputs[n - 1]] + else: + outputs = [] + self._set_outputs(outputs) + + def _tail(self): + n = self._coerce_n() + if 1 <= n <= len(self.inputs): + outputs = [self.inputs[-n]] + else: + outputs = [] + self._set_outputs(outputs) + + def _filter(self): + self._set_outputs([i for i in self.inputs if self._eval(self._norm(i),self._param.filter["operator"],self._param.filter["value"])]) + + def _norm(self,v): + s = "" if v is None else str(v) + return s + + def _eval(self, v, operator, value): + if operator == "=": + return v == value + elif operator == "≠": + return v != value + elif operator == "contains": + return value in v + elif operator == "start with": + return v.startswith(value) + elif operator == "end with": + return v.endswith(value) + else: + return False + + def _sort(self): + if self._param.sort_method == "asc": + self._set_outputs(sorted(self.inputs)) + elif self._param.sort_method == "desc": + self._set_outputs(sorted(self.inputs, reverse=True)) + + def _drop_duplicates(self): + seen = set() + outs = [] + for item in self.inputs: + k = self._hashable(item) + if k in seen: + continue + seen.add(k) + outs.append(item) + self._set_outputs(outs) + + def _hashable(self,x): + if isinstance(x, dict): + return tuple(sorted((k, self._hashable(v)) for k, v in x.items())) + if isinstance(x, (list, tuple)): + return tuple(self._hashable(v) for v in x) + if isinstance(x, set): + return tuple(sorted(self._hashable(v) for v in x)) + return x + def thoughts(self) -> str: + return "ListOperation in progress" diff --git a/web/src/constants/agent.tsx b/web/src/constants/agent.tsx index 6ee8ab516..3a8411ce3 100644 --- a/web/src/constants/agent.tsx +++ b/web/src/constants/agent.tsx @@ -109,6 +109,7 @@ export enum Operator { SearXNG = 'SearXNG', Placeholder = 'Placeholder', DataOperations = 'DataOperations', + ListOperations = 'ListOperations', VariableAssigner = 'VariableAssigner', VariableAggregator = 'VariableAggregator', File = 'File', // pipeline diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 9a0569ab5..b9f374f7c 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -1591,6 +1591,8 @@ This delimiter is used to split the input text into several text pieces echo of codeDescription: 'It allows developers to write custom Python logic.', dataOperations: 'Data operations', dataOperationsDescription: 'Perform various operations on a Data object.', + listOperations: 'List operations', + listOperationsDescription: 'Perform operations on a list.', variableAssigner: 'Variable assigner', variableAssignerDescription: 'This component performs operations on Data objects, including extracting, filtering, and editing keys and values in the Data.', @@ -1806,6 +1808,19 @@ Important structured information may include: names, dates, locations, events, k removeKeys: 'Remove keys', renameKeys: 'Rename keys', }, + ListOperationsOptions: { + topN: 'Top N', + head: 'Head', + tail: 'Tail', + sort: 'Sort', + filter: 'Filter', + dropDuplicates: 'Drop duplicates', + }, + sortMethod: 'Sort method', + SortMethodOptions: { + asc: 'Ascending', + desc: 'Descending', + }, }, llmTools: { bad_calculator: { diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index c065986f2..ce21c5a30 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -1508,6 +1508,8 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于 codeDescription: '它允许开发人员编写自定义 Python 逻辑。', dataOperations: '数据操作', dataOperationsDescription: '对数据对象执行各种操作。', + listOperations: '列表操作', + listOperationsDescription: '对列表对象执行各种操作。', variableAssigner: '变量赋值器', variableAssignerDescription: '此组件对数据对象执行操作，包括提取、筛选和编辑数据中的键和值。', @@ -1679,6 +1681,19 @@ Tokenizer 会根据所选方式将内容存储为对应的数据结构。`, removeKeys: '删除键', renameKeys: '重命名键', }, + ListOperationsOptions: { + topN: '取前N项', + head: '取前第N项', + tail: '取后第N项', + sort: '排序', + filter: '筛选', + dropDuplicates: '去重', + }, + sortMethod: '排序方式', + SortMethodOptions: { + asc: '升序', + desc: '降序', + }, }, footer: { profile: 'All rights reserved @ React', diff --git a/web/src/pages/agent/canvas/index.tsx b/web/src/pages/agent/canvas/index.tsx index 5f78e8185..f2fc983e2 100644 --- a/web/src/pages/agent/canvas/index.tsx +++ b/web/src/pages/agent/canvas/index.tsx @@ -61,6 +61,7 @@ import { FileNode } from './node/file-node'; import { InvokeNode } from './node/invoke-node'; import { IterationNode, IterationStartNode } from './node/iteration-node'; import { KeywordNode } from './node/keyword-node'; +import { ListOperationsNode } from './node/list-operations-node'; import { MessageNode } from './node/message-node'; import NoteNode from './node/note-node'; import ParserNode from './node/parser-node'; @@ -101,6 +102,7 @@ export const nodeTypes: NodeTypes = { splitterNode: SplitterNode, contextNode: ExtractorNode, dataOperationsNode: DataOperationsNode, + listOperationsNode: ListOperationsNode, variableAssignerNode: VariableAssignerNode, variableAggregatorNode: VariableAggregatorNode, }; diff --git a/web/src/pages/agent/canvas/node/dropdown/accordion-operators.tsx b/web/src/pages/agent/canvas/node/dropdown/accordion-operators.tsx index 232ab78ff..8fd96f55f 100644 --- a/web/src/pages/agent/canvas/node/dropdown/accordion-operators.tsx +++ b/web/src/pages/agent/canvas/node/dropdown/accordion-operators.tsx @@ -79,6 +79,7 @@ export function AccordionOperators({ Operator.Code, Operator.StringTransform, Operator.DataOperations, + Operator.ListOperations, // Operator.VariableAssigner, Operator.VariableAggregator, ]} diff --git a/web/src/pages/agent/canvas/node/list-operations-node.tsx b/web/src/pages/agent/canvas/node/list-operations-node.tsx new file mode 100644 index 000000000..5b2778c92 --- /dev/null +++ b/web/src/pages/agent/canvas/node/list-operations-node.tsx @@ -0,0 +1,22 @@ +import { BaseNode } from '@/interfaces/database/agent'; +import { NodeProps } from '@xyflow/react'; +import { camelCase } from 'lodash'; +import { useTranslation } from 'react-i18next'; +import { RagNode } from '.'; +import { ListOperationsFormSchemaType } from '../../form/list-operations-form'; +import { LabelCard } from './card'; + +export function ListOperationsNode({ + ...props +}: NodeProps>) { + const { data } = props; + const { t } = useTranslation(); + + return ( + + + {t(`flow.ListOperationsOptions.${camelCase(data.form?.operations)}`)} + + + ); +} diff --git a/web/src/pages/agent/constant/index.tsx b/web/src/pages/agent/constant/index.tsx index 45341abf4..7aad5e4a3 100644 --- a/web/src/pages/agent/constant/index.tsx +++ b/web/src/pages/agent/constant/index.tsx @@ -595,6 +595,35 @@ export const initialDataOperationsValues = { }, }, }; +export enum SortMethod { + Asc = 'asc', + Desc = 'desc', +} + +export enum ListOperations { + TopN = 'topN', + Head = 'head', + Tail = 'tail', + Filter = 'filter', + Sort = 'sort', + DropDuplicates = 'drop_duplicates', +} + +export const initialListOperationsValues = { + query: '', + operations: ListOperations.TopN, + outputs: { + result: { + type: 'Array', + }, + first: { + type: '?', + }, + last: { + type: '?', + }, + }, +}; export const initialVariableAssignerValues = {}; @@ -673,6 +702,7 @@ export const RestrictedUpstreamMap = { [Operator.Tool]: [Operator.Begin], [Operator.Placeholder]: [Operator.Begin], [Operator.DataOperations]: [Operator.Begin], + [Operator.ListOperations]: [Operator.Begin], [Operator.Parser]: [Operator.Begin], // pipeline [Operator.Splitter]: [Operator.Begin], [Operator.HierarchicalMerger]: [Operator.Begin], @@ -729,6 +759,7 @@ export const NodeMap = { [Operator.HierarchicalMerger]: 'splitterNode', [Operator.Extractor]: 'contextNode', [Operator.DataOperations]: 'dataOperationsNode', + [Operator.ListOperations]: 'listOperationsNode', [Operator.VariableAssigner]: 'variableAssignerNode', [Operator.VariableAggregator]: 'variableAggregatorNode', }; diff --git a/web/src/pages/agent/form-sheet/form-config-map.tsx b/web/src/pages/agent/form-sheet/form-config-map.tsx index c291e4e05..37ab4cf2f 100644 --- a/web/src/pages/agent/form-sheet/form-config-map.tsx +++ b/web/src/pages/agent/form-sheet/form-config-map.tsx @@ -21,6 +21,7 @@ import IterationForm from '../form/iteration-form'; import IterationStartForm from '../form/iteration-start-from'; import Jin10Form from '../form/jin10-form'; import KeywordExtractForm from '../form/keyword-extract-form'; +import ListOperationsForm from '../form/list-operations-form'; import MessageForm from '../form/message-form'; import ParserForm from '../form/parser-form'; import PubMedForm from '../form/pubmed-form'; @@ -184,6 +185,9 @@ export const FormConfigMap = { [Operator.DataOperations]: { component: DataOperationsForm, }, + [Operator.ListOperations]: { + component: ListOperationsForm, + }, [Operator.VariableAssigner]: { component: VariableAssignerForm, }, diff --git a/web/src/pages/agent/form/list-operations-form/index.tsx b/web/src/pages/agent/form/list-operations-form/index.tsx new file mode 100644 index 000000000..5803fe055 --- /dev/null +++ b/web/src/pages/agent/form/list-operations-form/index.tsx @@ -0,0 +1,140 @@ +import NumberInput from '@/components/originui/number-input'; +import { SelectWithSearch } from '@/components/originui/select-with-search'; +import { RAGFlowFormItem } from '@/components/ragflow-form'; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { Separator } from '@/components/ui/separator'; +import { useBuildSwitchOperatorOptions } from '@/hooks/logic-hooks/use-build-operator-options'; +import { buildOptions } from '@/utils/form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { memo } from 'react'; +import { useForm, useWatch } from 'react-hook-form'; +import { useTranslation } from 'react-i18next'; +import { z } from 'zod'; +import { + DataOperationsOperatorOptions, + JsonSchemaDataType, + ListOperations, + SortMethod, + initialListOperationsValues, +} from '../../constant'; +import { useFormValues } from '../../hooks/use-form-values'; +import { useWatchFormChange } from '../../hooks/use-watch-form-change'; +import { INextOperatorForm } from '../../interface'; +import { buildOutputList } from '../../utils/build-output-list'; +import { FormWrapper } from '../components/form-wrapper'; +import { Output, OutputSchema } from '../components/output'; +import { PromptEditor } from '../components/prompt-editor'; +import { QueryVariable } from '../components/query-variable'; + +export const RetrievalPartialSchema = { + query: z.string(), + operations: z.string(), + n: z.number().int().min(0).optional(), + sort_method: z.string().optional(), + filter: z + .object({ + value: z.string().optional(), + operator: z.string().optional(), + }) + .optional(), + ...OutputSchema, +}; + +export const FormSchema = z.object(RetrievalPartialSchema); + +export type ListOperationsFormSchemaType = z.infer; + +const outputList = buildOutputList(initialListOperationsValues.outputs); + +function ListOperationsForm({ node }: INextOperatorForm) { + const { t } = useTranslation(); + + const defaultValues = useFormValues(initialListOperationsValues, node); + + const form = useForm({ + defaultValues: defaultValues, + mode: 'onChange', + resolver: zodResolver(FormSchema), + shouldUnregister: true, + }); + + const operations = useWatch({ control: form.control, name: 'operations' }); + + const ListOperationsOptions = buildOptions( + ListOperations, + t, + `flow.ListOperationsOptions`, + true, + ); + const SortMethodOptions = buildOptions( + SortMethod, + t, + `flow.SortMethodOptions`, + true, + ); + const operatorOptions = useBuildSwitchOperatorOptions( + DataOperationsOperatorOptions, + ); + useWatchFormChange(node?.id, form, true); + + return ( +
+ + + + + + + {[ + ListOperations.TopN, + ListOperations.Head, + ListOperations.Tail, + ].includes(operations as ListOperations) && ( + ( + + {t('flowNum')} + + + + + + )} + /> + )} + {[ListOperations.Sort].includes(operations as ListOperations) && ( + + + + )} + {[ListOperations.Filter].includes(operations as ListOperations) && ( +
+ + + + + + + +
+ )} + + +
+ ); +} + +export default memo(ListOperationsForm); diff --git a/web/src/pages/agent/hooks/use-add-node.ts b/web/src/pages/agent/hooks/use-add-node.ts index ed092a01b..44091f1b1 100644 --- a/web/src/pages/agent/hooks/use-add-node.ts +++ b/web/src/pages/agent/hooks/use-add-node.ts @@ -31,6 +31,7 @@ import { initialIterationValues, initialJin10Values, initialKeywordExtractValues, + initialListOperationsValues, initialMessageValues, initialNoteValues, initialParserValues, @@ -129,6 +130,7 @@ export const useInitializeOperatorParams = () => { prompts: t('flow.prompts.user.summary'), }, [Operator.DataOperations]: initialDataOperationsValues, + [Operator.ListOperations]: initialListOperationsValues, [Operator.VariableAssigner]: initialVariableAssignerValues, [Operator.VariableAggregator]: initialVariableAggregatorValues, }; diff --git a/web/src/pages/agent/operator-icon.tsx b/web/src/pages/agent/operator-icon.tsx index a7ece8ead..44fe9d01a 100644 --- a/web/src/pages/agent/operator-icon.tsx +++ b/web/src/pages/agent/operator-icon.tsx @@ -14,7 +14,7 @@ import { ReactComponent as YahooFinanceIcon } from '@/assets/svg/yahoo-finance.s import { IconFont } from '@/components/icon-font'; import { cn } from '@/lib/utils'; -import { Equal, FileCode, HousePlus, Variable } from 'lucide-react'; +import { Columns3, Equal, FileCode, HousePlus, Variable } from 'lucide-react'; import { Operator } from './constant'; interface IProps { @@ -57,6 +57,7 @@ export const SVGIconMap = { }; export const LucideIconMap = { [Operator.DataOperations]: FileCode, + [Operator.ListOperations]: Columns3, [Operator.VariableAssigner]: Equal, [Operator.VariableAggregator]: Variable, }; diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts index 3312b7236..a7d4248ff 100644 --- a/web/src/pages/agent/utils.ts +++ b/web/src/pages/agent/utils.ts @@ -328,7 +328,6 @@ export const buildDslComponentsByGraph = ( case Operator.DataOperations: params = transformDataOperationsParams(params); break; - default: break; }