From edb32b130402bd544a3140a8dbf3b57bddc98963 Mon Sep 17 00:00:00 2001 From: chanx <1243304602@qq.com> Date: Tue, 8 Jul 2025 19:19:07 +0800 Subject: [PATCH] fix: Change the data in the dataset page to be obtained using the interface (#8726) ### What problem does this PR solve? Change the data in the dataset page to be obtained using the interface, and change the import to obtain all data every 15 seconds to obtain the data of the current page every 5 seconds when parsing the existing file. [#3221](https://github.com/infiniflow/ragflow/issues/3221) ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- .../list-filter-bar/filter-popover.tsx | 12 ++- web/src/components/list-filter-bar/index.tsx | 8 +- web/src/hooks/use-document-request.ts | 80 ++++++++++++------- web/src/interfaces/database/document.ts | 5 ++ web/src/interfaces/request/knowledge.ts | 2 +- web/src/locales/en.ts | 5 ++ web/src/locales/zh-traditional.ts | 5 ++ web/src/locales/zh.ts | 5 ++ .../components/chunk-card/index.less | 2 + .../components/chunk-card/index.tsx | 64 ++++++++++----- .../chunk-result-bar/checkbox-sets.tsx | 20 ++--- .../components/knowledge-chunk/index.tsx | 4 +- web/src/pages/dataset/dataset/index.tsx | 22 ++--- .../dataset/dataset/use-select-filters.ts | 42 +++++----- web/src/pages/dataset/testing/index.tsx | 5 +- web/src/services/knowledge-service.ts | 7 ++ web/src/utils/api.ts | 1 + 17 files changed, 189 insertions(+), 100 deletions(-) diff --git a/web/src/components/list-filter-bar/filter-popover.tsx b/web/src/components/list-filter-bar/filter-popover.tsx index de5bfb2ca..168a50784 100644 --- a/web/src/components/list-filter-bar/filter-popover.tsx +++ b/web/src/components/list-filter-bar/filter-popover.tsx @@ -24,6 +24,7 @@ export type CheckboxFormMultipleProps = { filters?: FilterCollection[]; value?: FilterValue; onChange?: FilterChange; + onOpenChange?: (open: boolean) => void; setOpen(open: boolean): void; }; @@ -148,12 +149,19 @@ export function FilterPopover({ children, value, onChange, + onOpenChange, filters, }: PropsWithChildren & Omit) { const [open, setOpen] = useState(false); - + const onOpenChangeFun = useCallback( + (e: boolean) => { + onOpenChange?.(e); + setOpen(e); + }, + [onOpenChange], + ); return ( - + {children}
{showFilter && ( - + )} diff --git a/web/src/hooks/use-document-request.ts b/web/src/hooks/use-document-request.ts index 3f0b2baa1..6549fcb2c 100644 --- a/web/src/hooks/use-document-request.ts +++ b/web/src/hooks/use-document-request.ts @@ -1,5 +1,8 @@ import { useHandleFilterSubmit } from '@/components/list-filter-bar/use-handle-filter-submit'; -import { IDocumentInfo } from '@/interfaces/database/document'; +import { + IDocumentInfo, + IDocumentInfoFilter, +} from '@/interfaces/database/document'; import { IChangeParserConfigRequestBody, IDocumentMetaRequestBody, @@ -10,7 +13,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import { useDebounce } from 'ahooks'; import { message } from 'antd'; import { get } from 'lodash'; -import { useCallback } from 'react'; +import { useCallback, useMemo, useState } from 'react'; import { useParams } from 'umi'; import { useGetPaginationWithRouter, @@ -30,7 +33,7 @@ export const enum DocumentApiAction { SaveDocumentName = 'saveDocumentName', SetDocumentParser = 'setDocumentParser', SetDocumentMeta = 'setDocumentMeta', - FetchAllDocumentList = 'fetchAllDocumentList', + FetchDocumentFilter = 'fetchDocumentFilter', CreateDocument = 'createDocument', } @@ -81,6 +84,10 @@ export const useFetchDocumentList = () => { const { id } = useParams(); const debouncedSearchString = useDebounce(searchString, { wait: 500 }); const { filterValue, handleFilterSubmit } = useHandleFilterSubmit(); + const [docs, setDocs] = useState([]); + const isLoop = useMemo(() => { + return docs.some((doc) => doc.run === '1'); + }, [docs]); const { data, isFetching: loading } = useQuery<{ docs: IDocumentInfo[]; @@ -93,7 +100,7 @@ export const useFetchDocumentList = () => { filterValue, ], initialData: { docs: [], total: 0 }, - // refetchInterval: 15000, + refetchInterval: isLoop ? 5000 : false, enabled: !!knowledgeId || !!id, queryFn: async () => { const ret = await listDocument( @@ -104,7 +111,7 @@ export const useFetchDocumentList = () => { page: pagination.current, }, { - types: filterValue.type, + suffix: filterValue.type, run_status: filterValue.run, }, ); @@ -118,7 +125,9 @@ export const useFetchDocumentList = () => { }; }, }); - + useMemo(() => { + setDocs(data.docs); + }, [data.docs]); const onInputChange: React.ChangeEventHandler = useCallback( (e) => { setPagination({ page: 1 }); @@ -139,34 +148,48 @@ export const useFetchDocumentList = () => { }; }; -export function useFetchAllDocumentList() { +// get document filter +export const useGetDocumentFilter = (): { + filter: IDocumentInfoFilter; + onOpenChange: (open: boolean) => void; +} => { + const { knowledgeId } = useGetKnowledgeSearchParams(); + const { searchString } = useHandleSearchChange(); const { id } = useParams(); - const { data, isFetching: loading } = useQuery<{ - docs: IDocumentInfo[]; - total: number; - }>({ - queryKey: [DocumentApiAction.FetchAllDocumentList], - initialData: { docs: [], total: 0 }, - refetchInterval: 15000, - enabled: !!id, + const debouncedSearchString = useDebounce(searchString, { wait: 500 }); + const [open, setOpen] = useState(0); + const { data } = useQuery({ + queryKey: [ + DocumentApiAction.FetchDocumentFilter, + debouncedSearchString, + knowledgeId, + open, + ], queryFn: async () => { - const ret = await listDocument({ - kb_id: id, + const { data } = await kbService.documentFilter({ + kb_id: knowledgeId || id, + keywords: debouncedSearchString, }); - if (ret.data.code === 0) { - return ret.data.data; + if (data.code === 0) { + return data.data; } - - return { - docs: [], - total: 0, - }; }, }); - - return { data, loading }; -} - + const handleOnpenChange = (e: boolean) => { + if (e) { + const currentOpen = open + 1; + setOpen(currentOpen); + } + }; + return { + filter: data?.filter || { + run_status: {}, + suffix: {}, + }, + onOpenChange: handleOnpenChange, + }; +}; +// update document status export const useSetDocumentStatus = () => { const queryClient = useQueryClient(); @@ -200,6 +223,7 @@ export const useSetDocumentStatus = () => { return { setDocumentStatus: mutateAsync, data, loading }; }; +// This hook is used to run a document by its IDs export const useRunDocument = () => { const queryClient = useQueryClient(); diff --git a/web/src/interfaces/database/document.ts b/web/src/interfaces/database/document.ts index 5ccbc44a2..dfc284835 100644 --- a/web/src/interfaces/database/document.ts +++ b/web/src/interfaces/database/document.ts @@ -47,3 +47,8 @@ interface GraphRag { resolution?: boolean; use_graphrag?: boolean; } + +export type IDocumentInfoFilter = { + run_status: Record; + suffix: Record; +}; diff --git a/web/src/interfaces/request/knowledge.ts b/web/src/interfaces/request/knowledge.ts index f18fb35b9..c43e5b2d1 100644 --- a/web/src/interfaces/request/knowledge.ts +++ b/web/src/interfaces/request/knowledge.ts @@ -21,6 +21,6 @@ export interface IFetchKnowledgeListRequestParams { } export interface IFetchDocumentListRequestBody { - types?: string[]; + suffix?: string[]; run_status?: string[]; } diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 9397b73d6..1ad67ea63 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -407,6 +407,11 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s mind: 'Mind map', question: 'Question', questionTip: `If there are given questions, the embedding of the chunk will be based on them.`, + chunkResult: 'Chunk Result', + chunkResultTip: `View the chunked segments used for embedding and retrieval.`, + enable: 'Enable', + disable: 'Disable', + delete: 'Delete', }, chat: { newConversation: 'New conversation', diff --git a/web/src/locales/zh-traditional.ts b/web/src/locales/zh-traditional.ts index 1d2662a26..df29ac4cd 100644 --- a/web/src/locales/zh-traditional.ts +++ b/web/src/locales/zh-traditional.ts @@ -396,6 +396,11 @@ export default { mind: '心智圖', question: '問題', questionTip: `如果存在給定的問題，則區塊的嵌入將基於它們。`, + chunkResult: '切片結果', + chunkResultTip: `查看用於嵌入和召回的切片段落`, + enable: '啟用', + disable: '禁用', + delete: '删除', }, chat: { newConversation: '新會話', diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index eca66c4e4..5dc1dbb51 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -414,6 +414,11 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于 mind: '思维导图', question: '问题', questionTip: `如果有给定的问题，则块的嵌入将基于它们。`, + chunkResult: '切片结果', + chunkResultTip: `查看用于嵌入和召回的切片段落。`, + enable: '启用', + disable: '禁用', + delete: '删除', }, chat: { newConversation: '新会话', diff --git a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.less b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.less index 127a5c790..aac7724af 100644 --- a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.less +++ b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.less @@ -24,11 +24,13 @@ .chunkCard { width: 100%; + padding: 18px 10px; } .cardSelected { background-color: @selectedBackgroundColor; } + .cardSelectedDark { background-color: #ffffff2f; } diff --git a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.tsx b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.tsx index b7e61e06a..69859ebfd 100644 --- a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.tsx +++ b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.tsx @@ -1,11 +1,18 @@ import Image from '@/components/image'; +import { useTheme } from '@/components/theme-provider'; +import { Card } from '@/components/ui/card'; +import { Checkbox } from '@/components/ui/checkbox'; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from '@/components/ui/popover'; +import { Switch } from '@/components/ui/switch'; import { IChunk } from '@/interfaces/database/knowledge'; -import { Card, Checkbox, CheckboxProps, Flex, Popover, Switch } from 'antd'; +import { CheckedState } from '@radix-ui/react-checkbox'; import classNames from 'classnames'; import DOMPurify from 'dompurify'; import { useEffect, useState } from 'react'; - -import { useTheme } from '@/components/theme-provider'; import { ChunkTextMode } from '../../constant'; import styles from './index.less'; @@ -39,8 +46,8 @@ const ChunkCard = ({ switchChunk(available === 0 ? 1 : 0, [item.chunk_id]); }; - const handleCheck: CheckboxProps['onChange'] = (e) => { - handleCheckboxClick(item.chunk_id, e.target.checked); + const handleCheck = (e: CheckedState) => { + handleCheckboxClick(item.chunk_id, e === 'indeterminate' ? false : e); }; const handleContentDoubleClick = () => { @@ -54,7 +61,7 @@ const ChunkCard = ({ useEffect(() => { setEnabled(available === 1); }, [available]); - + const [open, setOpen] = useState(false); return ( - - +
+ {item.image_id && ( - - } - > - + + setOpen(true)} + onMouseLeave={() => setOpen(false)} + > +
+ +
+ + +
+ +
+ )} -
-
- +
- +
); }; diff --git a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-result-bar/checkbox-sets.tsx b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-result-bar/checkbox-sets.tsx index 131d99619..2b71beff2 100644 --- a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-result-bar/checkbox-sets.tsx +++ b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-result-bar/checkbox-sets.tsx @@ -2,6 +2,7 @@ import { Checkbox } from '@/components/ui/checkbox'; import { Label } from '@/components/ui/label'; import { Ban, CircleCheck, Trash2 } from 'lucide-react'; import { useCallback } from 'react'; +import { useTranslation } from 'react-i18next'; type ICheckboxSetProps = { selectAllChunk: (e: any) => void; @@ -11,6 +12,7 @@ type ICheckboxSetProps = { }; export default (props: ICheckboxSetProps) => { const { selectAllChunk, removeChunk, switchChunk, checked } = props; + const { t } = useTranslation(); const handleSelectAllCheck = useCallback( (e: any) => { console.log('eee=', e); @@ -33,35 +35,35 @@ export default (props: ICheckboxSetProps) => { return (
-
+
- +
- Enable + {t('chunk.enable')}
- Disable + {t('chunk.disable')}
- Delete + {t('chunk.delete')}
); diff --git a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/index.tsx b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/index.tsx index 5f4a69c85..3822d691a 100644 --- a/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/index.tsx +++ b/web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/index.tsx @@ -183,9 +183,9 @@ const Chunk = () => {
-

Chunk Result

+

{t('chunk.chunkResult')}

- View the chunked segments used for embedding and retrieval. + {t('chunk.chunkResultTip')}
diff --git a/web/src/pages/dataset/dataset/index.tsx b/web/src/pages/dataset/dataset/index.tsx index 8ceeea173..da8a09dd9 100644 --- a/web/src/pages/dataset/dataset/index.tsx +++ b/web/src/pages/dataset/dataset/index.tsx @@ -12,9 +12,7 @@ import { } from '@/components/ui/dropdown-menu'; import { useRowSelection } from '@/hooks/logic-hooks/use-row-selection'; import { useFetchDocumentList } from '@/hooks/use-document-request'; -import { IDocumentInfo } from '@/interfaces/database/document'; import { Upload } from 'lucide-react'; -import { useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { DatasetTable } from './dataset-table'; import { useBulkOperateDataset } from './use-bulk-operate-dataset'; @@ -42,16 +40,7 @@ export default function Dataset() { handleFilterSubmit, loading, } = useFetchDocumentList(); - const { filters, documents: filteredDocuments } = useSelectDatasetFilters(); - const [datasetInfo, setDatasetInfo] = useState(documents); - - useMemo(() => { - setDatasetInfo(documents); - }, [documents]); - - useMemo(() => { - setDatasetInfo(filteredDocuments); - }, [filteredDocuments]); + const { filters, onOpenChange } = useSelectDatasetFilters(); const { createLoading, @@ -69,7 +58,6 @@ export default function Dataset() { rowSelection, setRowSelection, }); - return (
-
Dataset
+
{t('knowledgeDetails.dataset')}
- Please wait for your files to finish parsing before starting an - AI-powered chat. + {t('knowledgeDetails.datasetDescription')}
} @@ -111,7 +99,7 @@ export default function Dataset() { )} { - return groupListByType(documents, 'type', 'type'); - }, [documents]); - + if (filter.suffix) { + return Object.keys(filter.suffix).map((x) => ({ + id: x, + label: x.toUpperCase(), + count: filter.suffix[x], + })); + } + }, [filter.suffix]); const fileStatus = useMemo(() => { - return groupListByType(documents, 'run', 'run').map((x) => ({ - ...x, - label: t(`knowledgeDetails.runningStatus${x.label}`), - })); - }, [documents, t]); - const filters = useMemo(() => { + if (filter.run_status) { + return Object.keys(filter.run_status).map((x) => ({ + id: x, + label: t(`runningStatus${x}`), + count: filter.run_status[x as unknown as number], + })); + } + }, [filter.run_status, t]); + const filters: FilterCollection[] = useMemo(() => { return [ { field: 'type', label: 'File Type', list: fileTypes }, { field: 'run', label: 'Status', list: fileStatus }, - ]; + ] as FilterCollection[]; }, [fileStatus, fileTypes]); - return { fileTypes, fileStatus, filters, documents }; + return { filters, onOpenChange }; } diff --git a/web/src/pages/dataset/testing/index.tsx b/web/src/pages/dataset/testing/index.tsx index bed7e238f..244457d0b 100644 --- a/web/src/pages/dataset/testing/index.tsx +++ b/web/src/pages/dataset/testing/index.tsx @@ -35,9 +35,8 @@ export default function RetrievalTesting() {
{/* */}
diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.ts index c228f0e9a..f5383b63b 100644 --- a/web/src/services/knowledge-service.ts +++ b/web/src/services/knowledge-service.ts @@ -155,6 +155,10 @@ const methods = { url: listTagByKnowledgeIds, method: 'get', }, + documentFilter: { + url: api.get_dataset_filter, + method: 'post', + }, }; const kbService = registerServer(methods, request); @@ -188,4 +192,7 @@ export const listDocument = ( body?: IFetchDocumentListRequestBody, ) => request.post(api.get_document_list, { data: body || {}, params }); +export const documentFilter = (kb_id: string) => + request.post(api.get_dataset_filter, { kb_id }); + export default kbService; diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts index 2a51a2ad2..e809ff1a3 100644 --- a/web/src/utils/api.ts +++ b/web/src/utils/api.ts @@ -78,6 +78,7 @@ export default { upload_and_parse: `${api_host}/document/upload_and_parse`, parse: `${api_host}/document/parse`, setMeta: `${api_host}/document/set_meta`, + get_dataset_filter: `${api_host}/document/filter`, // chat setDialog: `${api_host}/dialog/set`,