Refactor datasets service toward TanStack Query (#29008)

Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
2025-12-09 13:44:45 +08:00
parent 57d244de69
commit 18601d8b38
12 changed files with 270 additions and 152 deletions
--- a/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx
+++ b/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx
@@ -1,18 +1,18 @@
 'use client'
 import type { FC } from 'react'
-import React, { useRef, useState } from 'react'
-import { useGetState, useInfiniteScroll } from 'ahooks'
+import React, { useEffect, useMemo, useRef, useState } from 'react'
+import { useInfiniteScroll } from 'ahooks'
 import { useTranslation } from 'react-i18next'
 import Link from 'next/link'
 import Modal from '@/app/components/base/modal'
 import type { DataSet } from '@/models/datasets'
 import Button from '@/app/components/base/button'
-import { fetchDatasets } from '@/service/datasets'
 import Loading from '@/app/components/base/loading'
 import Badge from '@/app/components/base/badge'
 import { useKnowledge } from '@/hooks/use-knowledge'
 import cn from '@/utils/classnames'
 import AppIcon from '@/app/components/base/app-icon'
+import { useInfiniteDatasets } from '@/service/knowledge/use-dataset'
 import { ModelFeatureEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import FeatureIcon from '@/app/components/header/account-setting/model-provider-page/model-selector/feature-icon'

@@ -30,51 +30,70 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
  onSelect,
 }) => {
  const { t } = useTranslation()
-  const [selected, setSelected] = React.useState<DataSet[]>([])
-  const [loaded, setLoaded] = React.useState(false)
-  const [datasets, setDataSets] = React.useState<DataSet[] | null>(null)
-  const [hasInitialized, setHasInitialized] = React.useState(false)
-  const hasNoData = !datasets || datasets?.length === 0
+  const [selected, setSelected] = useState<DataSet[]>([])
  const canSelectMulti = true
+  const { formatIndexingTechniqueAndMethod } = useKnowledge()
+  const { data, isLoading, isFetchingNextPage, fetchNextPage, hasNextPage } = useInfiniteDatasets(
+    { page: 1 },
+    { enabled: isShow, staleTime: 0, refetchOnMount: 'always' },
+  )
+  const pages = data?.pages || []
+  const datasets = useMemo(() => {
+    return pages.flatMap(page => page.data.filter(item => item.indexing_technique || item.provider === 'external'))
+  }, [pages])
+  const hasNoData = !isLoading && datasets.length === 0

  const listRef = useRef<HTMLDivElement>(null)
-  const [page, setPage, getPage] = useGetState(1)
-  const [isNoMore, setIsNoMore] = useState(false)
-  const { formatIndexingTechniqueAndMethod } = useKnowledge()
+  const isNoMore = hasNextPage === false

  useInfiniteScroll(
    async () => {
-      if (!isNoMore) {
-        const { data, has_more } = await fetchDatasets({ url: '/datasets', params: { page } })
-        setPage(getPage() + 1)
-        setIsNoMore(!has_more)
-        const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique || item.provider === 'external')]
-        setDataSets(newList)
-        setLoaded(true)
-
-        // Initialize selected datasets based on selectedIds and available datasets
-        if (!hasInitialized) {
-          if (selectedIds.length > 0) {
-            const validSelectedDatasets = selectedIds
-              .map(id => newList.find(item => item.id === id))
-              .filter(Boolean) as DataSet[]
-            setSelected(validSelectedDatasets)
-          }
-          setHasInitialized(true)
-        }
-      }
+      if (!hasNextPage || isFetchingNextPage)
+        return { list: [] }
+      await fetchNextPage()
      return { list: [] }
    },
    {
      target: listRef,
-      isNoMore: () => {
-        return isNoMore
-      },
-      reloadDeps: [isNoMore],
+      isNoMore: () => isNoMore,
+      reloadDeps: [isNoMore, isFetchingNextPage],
    },
  )

+  const prevSelectedIdsRef = useRef<string[]>([])
+  const hasUserModifiedSelectionRef = useRef(false)
+  useEffect(() => {
+    if (isShow)
+      hasUserModifiedSelectionRef.current = false
+  }, [isShow])
+  useEffect(() => {
+    const prevSelectedIds = prevSelectedIdsRef.current
+    const idsChanged = selectedIds.length !== prevSelectedIds.length
+      || selectedIds.some((id, idx) => id !== prevSelectedIds[idx])
+
+    if (!selectedIds.length && (!hasUserModifiedSelectionRef.current || idsChanged)) {
+      setSelected([])
+      prevSelectedIdsRef.current = selectedIds
+      hasUserModifiedSelectionRef.current = false
+      return
+    }
+
+    if (!idsChanged && hasUserModifiedSelectionRef.current)
+      return
+
+    setSelected((prev) => {
+      const prevMap = new Map(prev.map(item => [item.id, item]))
+      const nextSelected = selectedIds
+        .map(id => datasets.find(item => item.id === id) || prevMap.get(id))
+        .filter(Boolean) as DataSet[]
+      return nextSelected
+    })
+    prevSelectedIdsRef.current = selectedIds
+    hasUserModifiedSelectionRef.current = false
+  }, [datasets, selectedIds])
+
  const toggleSelect = (dataSet: DataSet) => {
+    hasUserModifiedSelectionRef.current = true
    const isSelected = selected.some(item => item.id === dataSet.id)
    if (isSelected) {
      setSelected(selected.filter(item => item.id !== dataSet.id))
@@ -98,13 +117,13 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
      className='w-[400px]'
      title={t('appDebug.feature.dataSet.selectTitle')}
    >
-      {!loaded && (
+      {(isLoading && datasets.length === 0) && (
        <div className='flex h-[200px]'>
          <Loading type='area' />
        </div>
      )}

-      {(loaded && hasNoData) && (
+      {hasNoData && (
        <div className='mt-6 flex h-[128px] items-center justify-center space-x-1  rounded-lg border text-[13px]'
          style={{
            background: 'rgba(0, 0, 0, 0.02)',
@@ -116,7 +135,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
        </div>
      )}

-      {datasets && datasets?.length > 0 && (
+      {datasets.length > 0 && (
        <>
          <div ref={listRef} className='mt-7 max-h-[286px] space-y-1 overflow-y-auto'>
            {datasets.map(item => (
@@ -171,7 +190,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
          </div>
        </>
      )}
-      {loaded && (
+      {!isLoading && (
        <div className='mt-8 flex items-center justify-between'>
          <div className='text-sm  font-medium text-text-secondary'>
            {selected.length > 0 && `${selected.length} ${t('appDebug.feature.dataSet.selected')}`}
--- a/web/app/components/datasets/common/document-status-with-action/index-failed.tsx
+++ b/web/app/components/datasets/common/document-status-with-action/index-failed.tsx
@@ -2,11 +2,11 @@
 import type { FC } from 'react'
 import React, { useEffect, useReducer } from 'react'
 import { useTranslation } from 'react-i18next'
-import useSWR from 'swr'
 import StatusWithAction from './status-with-action'
-import { getErrorDocs, retryErrorDocs } from '@/service/datasets'
+import { retryErrorDocs } from '@/service/datasets'
 import type { IndexingStatusResponse } from '@/models/datasets'
 import { noop } from 'lodash-es'
+import { useDatasetErrorDocs } from '@/service/knowledge/use-dataset'

 type Props = {
  datasetId: string
@@ -35,16 +35,19 @@ const indexStateReducer = (state: IIndexState, action: IAction) => {
 const RetryButton: FC<Props> = ({ datasetId }) => {
  const { t } = useTranslation()
  const [indexState, dispatch] = useReducer(indexStateReducer, { value: 'success' })
-  const { data: errorDocs, isLoading } = useSWR({ datasetId }, getErrorDocs)
+  const { data: errorDocs, isLoading, refetch: refetchErrorDocs } = useDatasetErrorDocs(datasetId)

  const onRetryErrorDocs = async () => {
    dispatch({ type: 'retry' })
    const document_ids = errorDocs?.data.map((doc: IndexingStatusResponse) => doc.id) || []
    const res = await retryErrorDocs({ datasetId, document_ids })
-    if (res.result === 'success')
+    if (res.result === 'success') {
+      refetchErrorDocs()
      dispatch({ type: 'success' })
-    else
+    }
+    else {
      dispatch({ type: 'error' })
+    }
  }

  useEffect(() => {
--- a/web/app/components/datasets/create/embedding-process/index.tsx
+++ b/web/app/components/datasets/create/embedding-process/index.tsx
@@ -1,9 +1,7 @@
 import type { FC } from 'react'
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
-import useSWR from 'swr'
 import { useRouter } from 'next/navigation'
 import { useTranslation } from 'react-i18next'
-import { omit } from 'lodash-es'
 import {
  RiArrowRightLine,
  RiCheckboxCircleFill,
@@ -25,7 +23,7 @@ import type {
  LegacyDataSourceInfo,
  ProcessRuleResponse,
 } from '@/models/datasets'
-import { fetchIndexingStatusBatch as doFetchIndexingStatus, fetchProcessRule } from '@/service/datasets'
+import { fetchIndexingStatusBatch as doFetchIndexingStatus } from '@/service/datasets'
 import { DataSourceType, ProcessMode } from '@/models/datasets'
 import NotionIcon from '@/app/components/base/notion-icon'
 import PriorityLabel from '@/app/components/billing/priority-label'
@@ -40,6 +38,7 @@ import { useInvalidDocumentList } from '@/service/knowledge/use-document'
 import Divider from '@/app/components/base/divider'
 import { useDatasetApiAccessUrl } from '@/hooks/use-api-access-url'
 import Link from 'next/link'
+import { useProcessRule } from '@/service/knowledge/use-dataset'

 type Props = {
  datasetId: string
@@ -207,12 +206,7 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
  }, [])

  // get rule
-  const { data: ruleDetail } = useSWR({
-    action: 'fetchProcessRule',
-    params: { documentId: getFirstDocument.id },
-  }, apiParams => fetchProcessRule(omit(apiParams, 'action')), {
-    revalidateOnFocus: false,
-  })
+  const { data: ruleDetail } = useProcessRule(getFirstDocument?.id)

  const router = useRouter()
  const invalidDocumentList = useInvalidDocumentList()
--- a/web/app/components/datasets/create/file-uploader/index.tsx
+++ b/web/app/components/datasets/create/file-uploader/index.tsx
@@ -2,7 +2,6 @@
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
-import useSWR from 'swr'
 import { RiDeleteBinLine, RiUploadCloud2Line } from '@remixicon/react'
 import DocumentFileIcon from '../../common/document-file-icon'
 import cn from '@/utils/classnames'
@@ -11,8 +10,7 @@ import { ToastContext } from '@/app/components/base/toast'
 import SimplePieChart from '@/app/components/base/simple-pie-chart'

 import { upload } from '@/service/base'
-import { fetchFileUploadConfig } from '@/service/common'
-import { fetchSupportFileTypes } from '@/service/datasets'
+import { useFileSupportTypes, useFileUploadConfig } from '@/service/use-common'
 import I18n from '@/context/i18n'
 import { LanguagesSupported } from '@/i18n-config/language'
 import { IS_CE_EDITION } from '@/config'
@@ -48,8 +46,8 @@ const FileUploader = ({
  const fileUploader = useRef<HTMLInputElement>(null)
  const hideUpload = notSupportBatchUpload && fileList.length > 0

-  const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig)
-  const { data: supportFileTypesResponse } = useSWR({ url: '/files/support-type' }, fetchSupportFileTypes)
+  const { data: fileUploadConfigResponse } = useFileUploadConfig()
+  const { data: supportFileTypesResponse } = useFileSupportTypes()
  const supportTypes = supportFileTypesResponse?.allowed_extensions || []
  const supportTypesShowNames = (() => {
    const extensionMap: { [key: string]: string } = {
--- a/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx
@@ -13,11 +13,10 @@ import Button from '@/app/components/base/button'
 import type { FileItem } from '@/models/datasets'
 import { upload } from '@/service/base'
 import { getFileUploadErrorMessage } from '@/app/components/base/file-uploader/utils'
-import useSWR from 'swr'
-import { fetchFileUploadConfig } from '@/service/common'
 import SimplePieChart from '@/app/components/base/simple-pie-chart'
 import { Theme } from '@/types/app'
 import useTheme from '@/hooks/use-theme'
+import { useFileUploadConfig } from '@/service/use-common'

 export type Props = {
  file: FileItem | undefined
@@ -34,7 +33,7 @@ const CSVUploader: FC<Props> = ({
  const dropRef = useRef<HTMLDivElement>(null)
  const dragRef = useRef<HTMLDivElement>(null)
  const fileUploader = useRef<HTMLInputElement>(null)
-  const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig)
+  const { data: fileUploadConfigResponse } = useFileUploadConfig()
  const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
    file_size_limit: 15,
  }, [fileUploadConfigResponse])
--- a/web/app/components/datasets/documents/detail/embedding/index.tsx
+++ b/web/app/components/datasets/documents/detail/embedding/index.tsx
@@ -1,9 +1,7 @@
 import type { FC } from 'react'
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
-import useSWR from 'swr'
 import { useContext } from 'use-context-selector'
 import { useTranslation } from 'react-i18next'
-import { omit } from 'lodash-es'
 import { RiLoader2Line, RiPauseCircleLine, RiPlayCircleLine } from '@remixicon/react'
 import Image from 'next/image'
 import { FieldInfo } from '../metadata'
@@ -21,10 +19,10 @@ import type { CommonResponse } from '@/models/common'
 import { asyncRunSafe, sleep } from '@/utils'
 import {
  fetchIndexingStatus as doFetchIndexingStatus,
-  fetchProcessRule,
  pauseDocIndexing,
  resumeDocIndexing,
 } from '@/service/datasets'
+import { useProcessRule } from '@/service/knowledge/use-dataset'

 type IEmbeddingDetailProps = {
  datasetId?: string
@@ -207,12 +205,7 @@ const EmbeddingDetail: FC<IEmbeddingDetailProps> = ({
    }
  }, [startQueryStatus, stopQueryStatus])

-  const { data: ruleDetail } = useSWR({
-    action: 'fetchProcessRule',
-    params: { documentId: localDocumentId },
-  }, apiParams => fetchProcessRule(omit(apiParams, 'action')), {
-    revalidateOnFocus: false,
-  })
+  const { data: ruleDetail } = useProcessRule(localDocumentId)

  const isEmbedding = useMemo(() => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || ''), [indexingStatusDetail])
  const isEmbeddingCompleted = useMemo(() => ['completed'].includes(indexingStatusDetail?.indexing_status || ''), [indexingStatusDetail])
--- a/web/app/components/datasets/hit-testing/index.tsx
+++ b/web/app/components/datasets/hit-testing/index.tsx
@@ -32,9 +32,8 @@ import Records from './components/records'
 import {
  useExternalKnowledgeBaseHitTesting,
  useHitTesting,
-  useHitTestingRecords,
-  useInvalidateHitTestingRecords,
 } from '@/service/knowledge/use-hit-testing'
+import { useDatasetTestingRecords } from '@/service/knowledge/use-dataset'

 const limit = 10

@@ -48,14 +47,13 @@ const HitTestingPage: FC<Props> = ({ datasetId }: Props) => {
  const media = useBreakpoints()
  const isMobile = media === MediaType.mobile

-  const [hitResult, setHitResult] = useState<HitTestingResponse | undefined>() // 初始化记录为空数组
+  const [hitResult, setHitResult] = useState<HitTestingResponse | undefined>()
  const [externalHitResult, setExternalHitResult] = useState<ExternalKnowledgeBaseHitTestingResponse | undefined>()
  const [queries, setQueries] = useState<Query[]>([])
  const [queryInputKey, setQueryInputKey] = useState(Date.now())

  const [currPage, setCurrPage] = useState<number>(0)
-  const { data: recordsRes, isLoading: isRecordsLoading } = useHitTestingRecords({ datasetId, page: currPage + 1, limit })
-  const invalidateHitTestingRecords = useInvalidateHitTestingRecords(datasetId)
+  const { data: recordsRes, refetch: recordsRefetch, isLoading: isRecordsLoading } = useDatasetTestingRecords(datasetId, { limit, page: currPage + 1 })

  const total = recordsRes?.total || 0

@@ -107,8 +105,7 @@ const HitTestingPage: FC<Props> = ({ datasetId }: Props) => {
  )

  const handleClickRecord = useCallback((record: HitTestingRecord) => {
-    const { queries } = record
-    setQueries(queries)
+    setQueries(record.queries)
    setQueryInputKey(Date.now())
  }, [])

@@ -128,7 +125,7 @@ const HitTestingPage: FC<Props> = ({ datasetId }: Props) => {
          setHitResult={setHitResult}
          setExternalHitResult={setExternalHitResult}
          onSubmit={showRightPanel}
-          onUpdateList={invalidateHitTestingRecords}
+          onUpdateList={recordsRefetch}
          loading={isRetrievalLoading}
          queries={queries}
          setQueries={setQueries}
@@ -140,11 +137,9 @@ const HitTestingPage: FC<Props> = ({ datasetId }: Props) => {
          externalKnowledgeBaseHitTestingMutation={externalKnowledgeBaseHitTestingMutation}
        />
        <div className='mb-3 mt-6 text-base font-semibold text-text-primary'>{t('datasetHitTesting.records')}</div>
-        {isRecordsLoading
-          && (
-            <div className='flex-1'><Loading type='app' /></div>
-          )
-        }
+        {isRecordsLoading && (
+          <div className='flex-1'><Loading type='app' /></div>
+        )}
        {!isRecordsLoading && recordsRes?.data && recordsRes.data.length > 0 && (
          <>
            <Records records={recordsRes?.data} onClickRecord={handleClickRecord}/>
--- a/web/app/components/develop/secret-key/secret-key-modal.tsx
+++ b/web/app/components/develop/secret-key/secret-key-modal.tsx
@@ -5,7 +5,6 @@ import {
 import { useTranslation } from 'react-i18next'
 import { RiDeleteBinLine } from '@remixicon/react'
 import { PlusIcon, XMarkIcon } from '@heroicons/react/20/solid'
-import useSWR from 'swr'
 import SecretKeyGenerateModal from './secret-key-generate'
 import s from './style.module.css'
 import ActionButton from '@/app/components/base/action-button'
@@ -19,7 +18,6 @@ import {
 import {
  createApikey as createDatasetApikey,
  delApikey as delDatasetApikey,
-  fetchApiKeysList as fetchDatasetApiKeysList,
 } from '@/service/datasets'
 import type { CreateApiKeyResponse } from '@/models/app'
 import Loading from '@/app/components/base/loading'
@@ -27,6 +25,7 @@ import Confirm from '@/app/components/base/confirm'
 import useTimestamp from '@/hooks/use-timestamp'
 import { useAppContext } from '@/context/app-context'
 import { useAppApiKeys, useInvalidateAppApiKeys } from '@/service/use-apps'
+import { useDatasetApiKeys, useInvalidateDatasetApiKeys } from '@/service/knowledge/use-dataset'

 type ISecretKeyModalProps = {
  isShow: boolean
@@ -46,11 +45,9 @@ const SecretKeyModal = ({
  const [isVisible, setVisible] = useState(false)
  const [newKey, setNewKey] = useState<CreateApiKeyResponse | undefined>(undefined)
  const invalidateAppApiKeys = useInvalidateAppApiKeys()
+  const invalidateDatasetApiKeys = useInvalidateDatasetApiKeys()
  const { data: appApiKeys, isLoading: isAppApiKeysLoading } = useAppApiKeys(appId, { enabled: !!appId && isShow })
-  const { data: datasetApiKeys, isLoading: isDatasetApiKeysLoading, mutate: mutateDatasetApiKeys } = useSWR(
-    !appId && isShow ? { url: '/datasets/api-keys', params: {} } : null,
-    fetchDatasetApiKeysList,
-  )
+  const { data: datasetApiKeys, isLoading: isDatasetApiKeysLoading } = useDatasetApiKeys({ enabled: !appId && isShow })
  const apiKeysList = appId ? appApiKeys : datasetApiKeys
  const isApiKeysLoading = appId ? isAppApiKeysLoading : isDatasetApiKeysLoading

@@ -69,7 +66,7 @@ const SecretKeyModal = ({
    if (appId)
      invalidateAppApiKeys(appId)
    else
-      mutateDatasetApiKeys()
+      invalidateDatasetApiKeys()
  }

  const onCreate = async () => {
@@ -83,7 +80,7 @@ const SecretKeyModal = ({
    if (appId)
      invalidateAppApiKeys(appId)
    else
-      mutateDatasetApiKeys()
+      invalidateDatasetApiKeys()
  }

  const generateToken = (token: string) => {
--- a/web/app/components/workflow/nodes/document-extractor/panel.tsx
+++ b/web/app/components/workflow/nodes/document-extractor/panel.tsx
@@ -1,6 +1,5 @@
 import type { FC } from 'react'
 import React from 'react'
-import useSWR from 'swr'
 import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
 import VarReferencePicker from '../_base/components/variable/var-reference-picker'
@@ -9,11 +8,11 @@ import Split from '../_base/components/split'
 import { useNodeHelpLink } from '../_base/hooks/use-node-help-link'
 import useConfig from './use-config'
 import type { DocExtractorNodeType } from './types'
-import { fetchSupportFileTypes } from '@/service/datasets'
 import Field from '@/app/components/workflow/nodes/_base/components/field'
 import { BlockEnum, type NodePanelProps } from '@/app/components/workflow/types'
 import I18n from '@/context/i18n'
 import { LanguagesSupported } from '@/i18n-config/language'
+import { useFileSupportTypes } from '@/service/use-common'

 const i18nPrefix = 'workflow.nodes.docExtractor'

@@ -24,7 +23,7 @@ const Panel: FC<NodePanelProps<DocExtractorNodeType>> = ({
  const { t } = useTranslation()
  const { locale } = useContext(I18n)
  const link = useNodeHelpLink(BlockEnum.DocExtractor)
-  const { data: supportFileTypesResponse } = useSWR({ url: '/files/support-type' }, fetchSupportFileTypes)
+  const { data: supportFileTypesResponse } = useFileSupportTypes()
  const supportTypes = supportFileTypesResponse?.allowed_extensions || []
  const supportTypesShowNames = (() => {
    const extensionMap: { [key: string]: string } = {
--- a/web/context/external-knowledge-api-context.tsx
+++ b/web/context/external-knowledge-api-context.tsx
@@ -1,10 +1,9 @@
 'use client'

-import { createContext, useContext, useMemo } from 'react'
+import { createContext, useCallback, useContext, useMemo } from 'react'
 import type { FC, ReactNode } from 'react'
-import useSWR from 'swr'
 import type { ExternalAPIItem, ExternalAPIListResponse } from '@/models/datasets'
-import { fetchExternalAPIList } from '@/service/datasets'
+import { useExternalKnowledgeApiList } from '@/service/knowledge/use-dataset'

 type ExternalKnowledgeApiContextType = {
  externalKnowledgeApiList: ExternalAPIItem[]
@@ -19,10 +18,11 @@ export type ExternalKnowledgeApiProviderProps = {
 }

 export const ExternalKnowledgeApiProvider: FC<ExternalKnowledgeApiProviderProps> = ({ children }) => {
-  const { data, mutate: mutateExternalKnowledgeApis, isLoading } = useSWR<ExternalAPIListResponse>(
-    { url: '/datasets/external-knowledge-api' },
-    fetchExternalAPIList,
-  )
+  const { data, refetch, isLoading } = useExternalKnowledgeApiList()
+
+  const mutateExternalKnowledgeApis = useCallback(() => {
+    return refetch().then(res => res.data)
+  }, [refetch])

  const contextValue = useMemo<ExternalKnowledgeApiContextType>(() => ({
    externalKnowledgeApiList: data?.data || [],
--- a/web/service/datasets.ts
+++ b/web/service/datasets.ts
@@ -1,4 +1,3 @@
-import type { Fetcher } from 'swr'
 import qs from 'qs'
 import { del, get, patch, post, put } from './base'
 import type {
@@ -50,140 +49,143 @@ export type SortType = 'created_at' | 'hit_count' | '-created_at' | '-hit_count'

 export type MetadataType = 'all' | 'only' | 'without'

-export const fetchDatasetDetail: Fetcher<DataSet, string> = (datasetId: string) => {
+export const fetchDatasetDetail = (datasetId: string): Promise<DataSet> => {
  return get<DataSet>(`/datasets/${datasetId}`)
 }

-export const updateDatasetSetting: Fetcher<DataSet, {
+export const updateDatasetSetting = ({
+  datasetId,
+  body,
+}: {
  datasetId: string
  body: Partial<Pick<DataSet,
    'name' | 'description' | 'permission' | 'partial_member_list' | 'indexing_technique' | 'retrieval_model' | 'embedding_model' | 'embedding_model_provider' | 'icon_info' | 'doc_form'
  >>
-}> = ({ datasetId, body }) => {
+}): Promise<DataSet> => {
  return patch<DataSet>(`/datasets/${datasetId}`, { body })
 }

-export const fetchDatasetRelatedApps: Fetcher<RelatedAppResponse, string> = (datasetId: string) => {
+export const fetchDatasetRelatedApps = (datasetId: string): Promise<RelatedAppResponse> => {
  return get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`)
 }

-export const fetchDatasets: Fetcher<DataSetListResponse, FetchDatasetsParams> = ({ url, params }) => {
+export const fetchDatasets = ({ url, params }: FetchDatasetsParams): Promise<DataSetListResponse> => {
  const urlParams = qs.stringify(params, { indices: false })
  return get<DataSetListResponse>(`${url}?${urlParams}`)
 }

-export const createEmptyDataset: Fetcher<DataSet, { name: string }> = ({ name }) => {
+export const createEmptyDataset = ({ name }: { name: string }): Promise<DataSet> => {
  return post<DataSet>('/datasets', { body: { name } })
 }

-export const checkIsUsedInApp: Fetcher<{ is_using: boolean }, string> = (id) => {
+export const checkIsUsedInApp = (id: string): Promise<{ is_using: boolean }> => {
  return get<{ is_using: boolean }>(`/datasets/${id}/use-check`, {}, {
    silent: true,
  })
 }

-export const deleteDataset: Fetcher<DataSet, string> = (datasetID) => {
+export const deleteDataset = (datasetID: string): Promise<DataSet> => {
  return del<DataSet>(`/datasets/${datasetID}`)
 }

-export const fetchExternalAPIList: Fetcher<ExternalAPIListResponse, { url: string }> = ({ url }) => {
+export const fetchExternalAPIList = ({ url }: { url: string }): Promise<ExternalAPIListResponse> => {
  return get<ExternalAPIListResponse>(url)
 }

-export const fetchExternalAPI: Fetcher<ExternalAPIItem, { apiTemplateId: string }> = ({ apiTemplateId }) => {
+export const fetchExternalAPI = ({ apiTemplateId }: { apiTemplateId: string }): Promise<ExternalAPIItem> => {
  return get<ExternalAPIItem>(`/datasets/external-knowledge-api/${apiTemplateId}`)
 }

-export const updateExternalAPI: Fetcher<ExternalAPIItem, { apiTemplateId: string; body: ExternalAPIItem }> = ({ apiTemplateId, body }) => {
+export const updateExternalAPI = ({ apiTemplateId, body }: { apiTemplateId: string; body: ExternalAPIItem }): Promise<ExternalAPIItem> => {
  return patch<ExternalAPIItem>(`/datasets/external-knowledge-api/${apiTemplateId}`, { body })
 }

-export const deleteExternalAPI: Fetcher<ExternalAPIDeleteResponse, { apiTemplateId: string }> = ({ apiTemplateId }) => {
+export const deleteExternalAPI = ({ apiTemplateId }: { apiTemplateId: string }): Promise<ExternalAPIDeleteResponse> => {
  return del<ExternalAPIDeleteResponse>(`/datasets/external-knowledge-api/${apiTemplateId}`)
 }

-export const checkUsageExternalAPI: Fetcher<ExternalAPIUsage, { apiTemplateId: string }> = ({ apiTemplateId }) => {
+export const checkUsageExternalAPI = ({ apiTemplateId }: { apiTemplateId: string }): Promise<ExternalAPIUsage> => {
  return get<ExternalAPIUsage>(`/datasets/external-knowledge-api/${apiTemplateId}/use-check`)
 }

-export const createExternalAPI: Fetcher<ExternalAPIItem, { body: CreateExternalAPIReq }> = ({ body }) => {
+export const createExternalAPI = ({ body }: { body: CreateExternalAPIReq }): Promise<ExternalAPIItem> => {
  return post<ExternalAPIItem>('/datasets/external-knowledge-api', { body })
 }

-export const createExternalKnowledgeBase: Fetcher<ExternalKnowledgeItem, { body: CreateKnowledgeBaseReq }> = ({ body }) => {
+export const createExternalKnowledgeBase = ({ body }: { body: CreateKnowledgeBaseReq }): Promise<ExternalKnowledgeItem> => {
  return post<ExternalKnowledgeItem>('/datasets/external', { body })
 }

-export const fetchDefaultProcessRule: Fetcher<ProcessRuleResponse, { url: string }> = ({ url }) => {
+export const fetchDefaultProcessRule = ({ url }: { url: string }): Promise<ProcessRuleResponse> => {
  return get<ProcessRuleResponse>(url)
 }
-export const fetchProcessRule: Fetcher<ProcessRuleResponse, { params: { documentId: string } }> = ({ params: { documentId } }) => {
+export const fetchProcessRule = ({ params: { documentId } }: { params: { documentId: string } }): Promise<ProcessRuleResponse> => {
  return get<ProcessRuleResponse>('/datasets/process-rule', { params: { document_id: documentId } })
 }

-export const createFirstDocument: Fetcher<createDocumentResponse, { body: CreateDocumentReq }> = ({ body }) => {
+export const createFirstDocument = ({ body }: { body: CreateDocumentReq }): Promise<createDocumentResponse> => {
  return post<createDocumentResponse>('/datasets/init', { body })
 }

-export const createDocument: Fetcher<createDocumentResponse, { datasetId: string; body: CreateDocumentReq }> = ({ datasetId, body }) => {
+export const createDocument = ({ datasetId, body }: { datasetId: string; body: CreateDocumentReq }): Promise<createDocumentResponse> => {
  return post<createDocumentResponse>(`/datasets/${datasetId}/documents`, { body })
 }

-export const fetchIndexingEstimate: Fetcher<IndexingEstimateResponse, CommonDocReq> = ({ datasetId, documentId }) => {
+export const fetchIndexingEstimate = ({ datasetId, documentId }: CommonDocReq): Promise<IndexingEstimateResponse> => {
  return get<IndexingEstimateResponse>(`/datasets/${datasetId}/documents/${documentId}/indexing-estimate`, {})
 }
-export const fetchIndexingEstimateBatch: Fetcher<IndexingEstimateResponse, BatchReq> = ({ datasetId, batchId }) => {
+export const fetchIndexingEstimateBatch = ({ datasetId, batchId }: BatchReq): Promise<IndexingEstimateResponse> => {
  return get<IndexingEstimateResponse>(`/datasets/${datasetId}/batch/${batchId}/indexing-estimate`, {})
 }

-export const fetchIndexingStatus: Fetcher<IndexingStatusResponse, CommonDocReq> = ({ datasetId, documentId }) => {
+export const fetchIndexingStatus = ({ datasetId, documentId }: CommonDocReq): Promise<IndexingStatusResponse> => {
  return get<IndexingStatusResponse>(`/datasets/${datasetId}/documents/${documentId}/indexing-status`, {})
 }

-export const fetchIndexingStatusBatch: Fetcher<IndexingStatusBatchResponse, BatchReq> = ({ datasetId, batchId }) => {
+export const fetchIndexingStatusBatch = ({ datasetId, batchId }: BatchReq): Promise<IndexingStatusBatchResponse> => {
  return get<IndexingStatusBatchResponse>(`/datasets/${datasetId}/batch/${batchId}/indexing-status`, {})
 }

-export const renameDocumentName: Fetcher<CommonResponse, CommonDocReq & { name: string }> = ({ datasetId, documentId, name }) => {
+export const renameDocumentName = ({ datasetId, documentId, name }: CommonDocReq & { name: string }): Promise<CommonResponse> => {
  return post<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/rename`, {
    body: { name },
  })
 }

-export const pauseDocIndexing: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
+export const pauseDocIndexing = ({ datasetId, documentId }: CommonDocReq): Promise<CommonResponse> => {
  return patch<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/processing/pause`)
 }

-export const resumeDocIndexing: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
+export const resumeDocIndexing = ({ datasetId, documentId }: CommonDocReq): Promise<CommonResponse> => {
  return patch<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/processing/resume`)
 }

-export const preImportNotionPages: Fetcher<{ notion_info: DataSourceNotionWorkspace[] }, { url: string; datasetId?: string }> = ({ url, datasetId }) => {
+export const preImportNotionPages = ({ url, datasetId }: { url: string; datasetId?: string }): Promise<{ notion_info: DataSourceNotionWorkspace[] }> => {
  return get<{ notion_info: DataSourceNotionWorkspace[] }>(url, { params: { dataset_id: datasetId } })
 }

-export const modifyDocMetadata: Fetcher<CommonResponse, CommonDocReq & { body: { doc_type: string; doc_metadata: Record<string, any> } }> = ({ datasetId, documentId, body }) => {
+export const modifyDocMetadata = ({ datasetId, documentId, body }: CommonDocReq & { body: { doc_type: string; doc_metadata: Record<string, any> } }): Promise<CommonResponse> => {
  return put<CommonResponse>(`/datasets/${datasetId}/documents/${documentId}/metadata`, { body })
 }

 // hit testing
-export const hitTesting: Fetcher<HitTestingResponse, { datasetId: string; queryText: string; retrieval_model: RetrievalConfig }> = ({ datasetId, queryText, retrieval_model }) => {
+export const hitTesting = ({ datasetId, queryText, retrieval_model }: { datasetId: string; queryText: string; retrieval_model: RetrievalConfig }): Promise<HitTestingResponse> => {
  return post<HitTestingResponse>(`/datasets/${datasetId}/hit-testing`, { body: { query: queryText, retrieval_model } })
 }

-export const externalKnowledgeBaseHitTesting: Fetcher<ExternalKnowledgeBaseHitTestingResponse, { datasetId: string; query: string; external_retrieval_model: { top_k: number; score_threshold: number; score_threshold_enabled: boolean } }> = ({ datasetId, query, external_retrieval_model }) => {
+export const externalKnowledgeBaseHitTesting = ({ datasetId, query, external_retrieval_model }: { datasetId: string; query: string; external_retrieval_model: { top_k: number; score_threshold: number; score_threshold_enabled: boolean } }): Promise<ExternalKnowledgeBaseHitTestingResponse> => {
  return post<ExternalKnowledgeBaseHitTestingResponse>(`/datasets/${datasetId}/external-hit-testing`, { body: { query, external_retrieval_model } })
 }

-export const fetchTestingRecords: Fetcher<HitTestingRecordsResponse, { datasetId: string; params: { page: number; limit: number } }> = ({ datasetId, params }) => {
+export const fetchTestingRecords = ({ datasetId, params }: { datasetId: string; params: { page: number; limit: number } }): Promise<HitTestingRecordsResponse> => {
  return get<HitTestingRecordsResponse>(`/datasets/${datasetId}/queries`, { params })
 }

-export const fetchFileIndexingEstimate: Fetcher<FileIndexingEstimateResponse, IndexingEstimateParams> = (body: IndexingEstimateParams) => {
+export const fetchFileIndexingEstimate = (body: IndexingEstimateParams): Promise<FileIndexingEstimateResponse> => {
  return post<FileIndexingEstimateResponse>('/datasets/indexing-estimate', { body })
 }

-export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID: string; pageID: string; pageType: string; credentialID: string; }> = ({ workspaceID, pageID, pageType, credentialID }) => {
+export const fetchNotionPagePreview = ({ workspaceID, pageID, pageType, credentialID }: { workspaceID: string; pageID: string; pageType: string; credentialID: string }): Promise<{ content: string }> => {
  return get<{ content: string }>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`, {
    params: {
      credential_id: credentialID,
@@ -191,31 +193,31 @@ export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID:
  })
 }

-export const fetchApiKeysList: Fetcher<ApiKeysListResponse, { url: string; params: Record<string, any> }> = ({ url, params }) => {
+export const fetchApiKeysList = ({ url, params }: { url: string; params: Record<string, any> }): Promise<ApiKeysListResponse> => {
  return get<ApiKeysListResponse>(url, params)
 }

-export const delApikey: Fetcher<CommonResponse, { url: string; params: Record<string, any> }> = ({ url, params }) => {
+export const delApikey = ({ url, params }: { url: string; params: Record<string, any> }): Promise<CommonResponse> => {
  return del<CommonResponse>(url, params)
 }

-export const createApikey: Fetcher<CreateApiKeyResponse, { url: string; body: Record<string, any> }> = ({ url, body }) => {
+export const createApikey = ({ url, body }: { url: string; body: Record<string, any> }): Promise<CreateApiKeyResponse> => {
  return post<CreateApiKeyResponse>(url, body)
 }

-export const fetchDataSources = () => {
+export const fetchDataSources = (): Promise<CommonResponse> => {
  return get<CommonResponse>('api-key-auth/data-source')
 }

-export const createDataSourceApiKeyBinding: Fetcher<CommonResponse, Record<string, any>> = (body) => {
+export const createDataSourceApiKeyBinding = (body: Record<string, any>): Promise<CommonResponse> => {
  return post<CommonResponse>('api-key-auth/data-source/binding', { body })
 }

-export const removeDataSourceApiKeyBinding: Fetcher<CommonResponse, string> = (id: string) => {
+export const removeDataSourceApiKeyBinding = (id: string): Promise<CommonResponse> => {
  return del<CommonResponse>(`api-key-auth/data-source/${id}`)
 }

-export const createFirecrawlTask: Fetcher<CommonResponse, Record<string, any>> = (body) => {
+export const createFirecrawlTask = (body: Record<string, any>): Promise<CommonResponse> => {
  return post<CommonResponse>('website/crawl', {
    body: {
      ...body,
@@ -224,7 +226,7 @@ export const createFirecrawlTask: Fetcher<CommonResponse, Record<string, any>> =
  })
 }

-export const checkFirecrawlTaskStatus: Fetcher<CommonResponse, string> = (jobId: string) => {
+export const checkFirecrawlTaskStatus = (jobId: string): Promise<CommonResponse> => {
  return get<CommonResponse>(`website/crawl/status/${jobId}`, {
    params: {
      provider: DataSourceProvider.fireCrawl,
@@ -234,7 +236,7 @@ export const checkFirecrawlTaskStatus: Fetcher<CommonResponse, string> = (jobId:
  })
 }

-export const createJinaReaderTask: Fetcher<CommonResponse, Record<string, any>> = (body) => {
+export const createJinaReaderTask = (body: Record<string, any>): Promise<CommonResponse> => {
  return post<CommonResponse>('website/crawl', {
    body: {
      ...body,
@@ -243,7 +245,7 @@ export const createJinaReaderTask: Fetcher<CommonResponse, Record<string, any>>
  })
 }

-export const checkJinaReaderTaskStatus: Fetcher<CommonResponse, string> = (jobId: string) => {
+export const checkJinaReaderTaskStatus = (jobId: string): Promise<CommonResponse> => {
  return get<CommonResponse>(`website/crawl/status/${jobId}`, {
    params: {
      provider: 'jinareader',
@@ -253,7 +255,7 @@ export const checkJinaReaderTaskStatus: Fetcher<CommonResponse, string> = (jobId
  })
 }

-export const createWatercrawlTask: Fetcher<CommonResponse, Record<string, any>> = (body) => {
+export const createWatercrawlTask = (body: Record<string, any>): Promise<CommonResponse> => {
  return post<CommonResponse>('website/crawl', {
    body: {
      ...body,
@@ -262,7 +264,7 @@ export const createWatercrawlTask: Fetcher<CommonResponse, Record<string, any>>
  })
 }

-export const checkWatercrawlTaskStatus: Fetcher<CommonResponse, string> = (jobId: string) => {
+export const checkWatercrawlTaskStatus = (jobId: string): Promise<CommonResponse> => {
  return get<CommonResponse>(`website/crawl/status/${jobId}`, {
    params: {
      provider: DataSourceProvider.waterCrawl,
@@ -276,14 +278,14 @@ export type FileTypesRes = {
  allowed_extensions: string[]
 }

-export const fetchSupportFileTypes: Fetcher<FileTypesRes, { url: string }> = ({ url }) => {
+export const fetchSupportFileTypes = ({ url }: { url: string }): Promise<FileTypesRes> => {
  return get<FileTypesRes>(url)
 }

-export const getErrorDocs: Fetcher<ErrorDocsResponse, { datasetId: string }> = ({ datasetId }) => {
+export const getErrorDocs = ({ datasetId }: { datasetId: string }): Promise<ErrorDocsResponse> => {
  return get<ErrorDocsResponse>(`/datasets/${datasetId}/error-docs`)
 }

-export const retryErrorDocs: Fetcher<CommonResponse, { datasetId: string; document_ids: string[] }> = ({ datasetId, document_ids }) => {
+export const retryErrorDocs = ({ datasetId, document_ids }: { datasetId: string; document_ids: string[] }): Promise<CommonResponse> => {
  return post<CommonResponse>(`/datasets/${datasetId}/retry`, { body: { document_ids } })
 }
--- a/web/service/knowledge/use-dataset.ts
+++ b/web/service/knowledge/use-dataset.ts
@@ -1,23 +1,86 @@
 import type { MutationOptions } from '@tanstack/react-query'
-import { useInfiniteQuery, useMutation, useQuery } from '@tanstack/react-query'
+import {
+  keepPreviousData,
+  useInfiniteQuery,
+  useMutation,
+  useQuery,
+  useQueryClient,
+} from '@tanstack/react-query'
+import qs from 'qs'
 import type {
  DataSet,
  DataSetListResponse,
  DatasetListRequest,
+  ErrorDocsResponse,
+  ExternalAPIListResponse,
+  FetchDatasetsParams,
+  HitTestingRecordsResponse,
  IndexingStatusBatchRequest,
  IndexingStatusBatchResponse,
  ProcessRuleResponse,
  RelatedAppResponse,
 } from '@/models/datasets'
+import type { ApiKeysListResponse } from '@/models/app'
 import { get, post } from '../base'
 import { useInvalid } from '../use-base'
-import qs from 'qs'
 import type { CommonResponse } from '@/models/common'

 const NAME_SPACE = 'dataset'

 const DatasetListKey = [NAME_SPACE, 'list']

+const normalizeDatasetsParams = (params: Partial<FetchDatasetsParams['params']> = {}) => {
+  const {
+    page = 1,
+    limit,
+    ids,
+    tag_ids,
+    include_all,
+    keyword,
+  } = params
+
+  return {
+    page,
+    ...(limit ? { limit } : {}),
+    ...(ids?.length ? { ids } : {}),
+    ...(tag_ids?.length ? { tag_ids } : {}),
+    ...(include_all !== undefined ? { include_all } : {}),
+    ...(keyword ? { keyword } : {}),
+  }
+}
+
+type UseInfiniteDatasetsOptions = {
+  enabled?: boolean
+  refetchOnMount?: boolean | 'always'
+  staleTime?: number
+  refetchOnReconnect?: boolean
+  refetchOnWindowFocus?: boolean
+}
+
+export const useInfiniteDatasets = (
+  params: Partial<FetchDatasetsParams['params']>,
+  options?: UseInfiniteDatasetsOptions,
+) => {
+  const normalizedParams = normalizeDatasetsParams(params)
+  const buildUrl = (pageParam: number | undefined) => {
+    const queryString = qs.stringify({
+      ...normalizedParams,
+      page: pageParam ?? normalizedParams.page,
+    }, { indices: false })
+    return `/datasets?${queryString}`
+  }
+
+  return useInfiniteQuery<DataSetListResponse>({
+    queryKey: [...DatasetListKey, 'infinite', normalizedParams],
+    queryFn: ({ pageParam = normalizedParams.page }) => get<DataSetListResponse>(buildUrl(pageParam as number | undefined)),
+    getNextPageParam: lastPage => lastPage.has_more ? lastPage.page + 1 : undefined,
+    initialPageParam: normalizedParams.page,
+    staleTime: 0,
+    refetchOnMount: 'always',
+    ...options,
+  })
+}
+
 export const useDatasetList = (params: DatasetListRequest) => {
  const { initialPage, tag_ids, limit, include_all, keyword } = params
  return useInfiniteQuery({
@@ -70,10 +133,12 @@ export const useIndexingStatusBatch = (
  })
 }

-export const useProcessRule = (documentId: string) => {
+export const useProcessRule = (documentId?: string) => {
  return useQuery<ProcessRuleResponse>({
    queryKey: [NAME_SPACE, 'process-rule', documentId],
    queryFn: () => get<ProcessRuleResponse>('/datasets/process-rule', { params: { document_id: documentId } }),
+    enabled: !!documentId,
+    refetchOnWindowFocus: false,
  })
 }

@@ -97,3 +162,57 @@ export const useDisableDatasetServiceApi = () => {
    mutationFn: (datasetId: string) => post<CommonResponse>(`/datasets/${datasetId}/api-keys/disable`),
  })
 }
+
+export const useDatasetApiKeys = (options?: { enabled?: boolean }) => {
+  return useQuery<ApiKeysListResponse>({
+    queryKey: [NAME_SPACE, 'api-keys'],
+    queryFn: () => get<ApiKeysListResponse>('/datasets/api-keys'),
+    enabled: options?.enabled ?? true,
+  })
+}
+
+export const useInvalidateDatasetApiKeys = () => {
+  const queryClient = useQueryClient()
+  return () => {
+    queryClient.invalidateQueries({
+      queryKey: [NAME_SPACE, 'api-keys'],
+    })
+  }
+}
+
+export const useExternalKnowledgeApiList = (options?: { enabled?: boolean }) => {
+  return useQuery<ExternalAPIListResponse>({
+    queryKey: [NAME_SPACE, 'external-knowledge-api'],
+    queryFn: () => get<ExternalAPIListResponse>('/datasets/external-knowledge-api'),
+    enabled: options?.enabled ?? true,
+  })
+}
+
+export const useInvalidateExternalKnowledgeApiList = () => {
+  const queryClient = useQueryClient()
+  return () => {
+    queryClient.invalidateQueries({
+      queryKey: [NAME_SPACE, 'external-knowledge-api'],
+    })
+  }
+}
+
+export const useDatasetTestingRecords = (
+  datasetId?: string,
+  params?: { page: number; limit: number },
+) => {
+  return useQuery<HitTestingRecordsResponse>({
+    queryKey: [NAME_SPACE, 'testing-records', datasetId, params],
+    queryFn: () => get<HitTestingRecordsResponse>(`/datasets/${datasetId}/queries`, { params }),
+    enabled: !!datasetId && !!params,
+    placeholderData: keepPreviousData,
+  })
+}
+
+export const useDatasetErrorDocs = (datasetId?: string) => {
+  return useQuery<ErrorDocsResponse>({
+    queryKey: [NAME_SPACE, 'error-docs', datasetId],
+    queryFn: () => get<ErrorDocsResponse>(`/datasets/${datasetId}/error-docs`),
+    enabled: !!datasetId,
+  })
+}