From 583ba2ded39b58e97d9495ff1be8dfd15ae0117b Mon Sep 17 00:00:00 2001 From: Lucas Oliveira <62335616+lucaseduoli@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:30:23 -0300 Subject: [PATCH] Fixed failed tasks not going away, and processing tasks going away (#393) --- frontend/src/app/knowledge/page.tsx | 730 ++++++++++++------------- frontend/src/contexts/task-context.tsx | 40 +- 2 files changed, 398 insertions(+), 372 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index e544d8f3..62939bad 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -1,11 +1,11 @@ "use client"; import { - themeQuartz, - type CheckboxSelectionCallbackParams, - type ColDef, - type GetRowIdParams, - type ValueFormatterParams, + type CheckboxSelectionCallbackParams, + type ColDef, + type GetRowIdParams, + themeQuartz, + type ValueFormatterParams, } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; import { Cloud, FileIcon, Globe } from "lucide-react"; @@ -21,331 +21,331 @@ import "@/components/AgGrid/registerAgGridModules"; import "@/components/AgGrid/agGridStyles.css"; import { toast } from "sonner"; import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdown"; +import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; import { StatusBadge } from "@/components/ui/status-badge"; import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog"; import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; import GoogleDriveIcon from "../settings/icons/google-drive-icon"; import OneDriveIcon from "../settings/icons/one-drive-icon"; import SharePointIcon from "../settings/icons/share-point-icon"; -import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogHeader, - DialogTitle, - DialogTrigger, -} from "@/components/ui/dialog"; // Function to get the appropriate icon for a connector type function getSourceIcon(connectorType?: string) { - switch (connectorType) { - case "google_drive": - return ( - - ); - case "onedrive": - return ; - case "sharepoint": - return ( - - ); - case "url": - return ; - case "s3": - return ; - default: - return ( - - ); - } + switch (connectorType) { + case "google_drive": + return ( + + ); + case "onedrive": + return ; + case "sharepoint": + return ( + + ); + case "url": + return ; + case "s3": + return ; + default: + return ( + + ); + } } function SearchPage() { - const router = useRouter(); - const { files: taskFiles, refreshTasks } = useTask(); - const { parsedFilterData, queryOverride } = useKnowledgeFilter(); - const [selectedRows, setSelectedRows] = useState([]); - const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); + const router = useRouter(); + const { files: taskFiles, refreshTasks } = useTask(); + const { parsedFilterData, queryOverride } = useKnowledgeFilter(); + const [selectedRows, setSelectedRows] = useState([]); + const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); - const deleteDocumentMutation = useDeleteDocument(); + const deleteDocumentMutation = useDeleteDocument(); - useEffect(() => { - refreshTasks(); - }, [refreshTasks]); + useEffect(() => { + refreshTasks(); + }, [refreshTasks]); - const { data: searchData = [], isFetching } = useGetSearchQuery( - queryOverride, - parsedFilterData - ); - // Convert TaskFiles to File format and merge with backend results - const taskFilesAsFiles: File[] = taskFiles.map(taskFile => { - return { - filename: taskFile.filename, - mimetype: taskFile.mimetype, - source_url: taskFile.source_url, - size: taskFile.size, - connector_type: taskFile.connector_type, - status: taskFile.status, - error: taskFile.error, - embedding_model: taskFile.embedding_model, - embedding_dimensions: taskFile.embedding_dimensions, - }; - }); + const { data: searchData = [], isFetching } = useGetSearchQuery( + queryOverride, + parsedFilterData, + ); + // Convert TaskFiles to File format and merge with backend results + const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { + return { + filename: taskFile.filename, + mimetype: taskFile.mimetype, + source_url: taskFile.source_url, + size: taskFile.size, + connector_type: taskFile.connector_type, + status: taskFile.status, + error: taskFile.error, + embedding_model: taskFile.embedding_model, + embedding_dimensions: taskFile.embedding_dimensions, + }; + }); + // Create a map of task files by filename for quick lookup + const taskFileMap = new Map( + taskFilesAsFiles.map((file) => [file.filename, file]), + ); + // Override backend files with task file status if they exist + const backendFiles = (searchData as File[]) + .map((file) => { + const taskFile = taskFileMap.get(file.filename); + if (taskFile) { + // Override backend file with task file data (includes status) + return { ...file, ...taskFile }; + } + return file; + }) + .filter((file) => { + // Only filter out files that are currently processing AND in taskFiles + const taskFile = taskFileMap.get(file.filename); + return !taskFile || taskFile.status !== "processing"; + }); - // Create a map of task files by filename for quick lookup - const taskFileMap = new Map( - taskFilesAsFiles.map(file => [file.filename, file]) - ); - // Override backend files with task file status if they exist - const backendFiles = (searchData as File[]) - .map(file => { - const taskFile = taskFileMap.get(file.filename); - if (taskFile) { - // Override backend file with task file data (includes status) - return { ...file, ...taskFile }; - } - return file; - }) - .filter(file => { - // Only filter out files that are currently processing AND in taskFiles - const taskFile = taskFileMap.get(file.filename); - return !taskFile || taskFile.status !== "processing"; - }); + const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { + return ( + taskFile.status !== "active" && + !backendFiles.some( + (backendFile) => backendFile.filename === taskFile.filename, + ) + ); + }); + // Combine task files first, then backend files + const fileResults = [...backendFiles, ...filteredTaskFiles]; + const gridRef = useRef(null); - const filteredTaskFiles = taskFilesAsFiles.filter(taskFile => { - return ( - taskFile.status !== "active" && - !backendFiles.some( - backendFile => backendFile.filename === taskFile.filename - ) - ); - }); + const columnDefs: ColDef[] = [ + { + field: "filename", + headerName: "Source", + checkboxSelection: (params: CheckboxSelectionCallbackParams) => + (params?.data?.status || "active") === "active", + headerCheckboxSelection: true, + initialFlex: 2, + minWidth: 220, + cellRenderer: ({ data, value }: CustomCellRendererProps) => { + // Read status directly from data on each render + const status = data?.status || "active"; + const isActive = status === "active"; + return ( +
+
+ +
+ ); + }, + }, + { + field: "size", + headerName: "Size", + valueFormatter: (params: ValueFormatterParams) => + params.value ? `${Math.round(params.value / 1024)} KB` : "-", + }, + { + field: "mimetype", + headerName: "Type", + }, + { + field: "owner", + headerName: "Owner", + valueFormatter: (params: ValueFormatterParams) => + params.data?.owner_name || params.data?.owner_email || "—", + }, + { + field: "chunkCount", + headerName: "Chunks", + valueFormatter: (params: ValueFormatterParams) => + params.data?.chunkCount?.toString() || "-", + }, + { + field: "avgScore", + headerName: "Avg score", + cellRenderer: ({ value }: CustomCellRendererProps) => { + return ( + + {value?.toFixed(2) ?? "-"} + + ); + }, + }, + { + field: "embedding_model", + headerName: "Embedding model", + minWidth: 200, + cellRenderer: ({ data }: CustomCellRendererProps) => ( + + {data?.embedding_model || "—"} + + ), + }, + { + field: "embedding_dimensions", + headerName: "Dimensions", + width: 110, + cellRenderer: ({ data }: CustomCellRendererProps) => ( + + {typeof data?.embedding_dimensions === "number" + ? data.embedding_dimensions.toString() + : "—"} + + ), + }, + { + field: "status", + headerName: "Status", + cellRenderer: ({ data }: CustomCellRendererProps) => { + const status = data?.status || "active"; + const error = + typeof data?.error === "string" && data.error.trim().length > 0 + ? data.error.trim() + : undefined; + if (status === "failed" && error) { + return ( + + + + + + + Ingestion failed + + {data?.filename || "Unknown file"} + + +
+ {error} +
+ + + ); + } + return ; + }, + }, + { + cellRenderer: ({ data }: CustomCellRendererProps) => { + const status = data?.status || "active"; + if (status !== "active") { + return null; + } + return ; + }, + cellStyle: { + alignItems: "center", + display: "flex", + justifyContent: "center", + padding: 0, + }, + colId: "actions", + filter: false, + minWidth: 0, + width: 40, + resizable: false, + sortable: false, + initialFlex: 0, + }, + ]; - // Combine task files first, then backend files - const fileResults = [...backendFiles, ...filteredTaskFiles]; + const defaultColDef: ColDef = { + resizable: false, + suppressMovable: true, + initialFlex: 1, + minWidth: 100, + }; - const gridRef = useRef(null); + const onSelectionChanged = useCallback(() => { + if (gridRef.current) { + const selectedNodes = gridRef.current.api.getSelectedRows(); + setSelectedRows(selectedNodes); + } + }, []); - const columnDefs: ColDef[] = [ - { - field: "filename", - headerName: "Source", - checkboxSelection: (params: CheckboxSelectionCallbackParams) => - (params?.data?.status || "active") === "active", - headerCheckboxSelection: true, - initialFlex: 2, - minWidth: 220, - cellRenderer: ({ data, value }: CustomCellRendererProps) => { - // Read status directly from data on each render - const status = data?.status || "active"; - const isActive = status === "active"; - return ( -
-
- -
- ); - }, - }, - { - field: "size", - headerName: "Size", - valueFormatter: (params: ValueFormatterParams) => - params.value ? `${Math.round(params.value / 1024)} KB` : "-", - }, - { - field: "mimetype", - headerName: "Type", - }, - { - field: "owner", - headerName: "Owner", - valueFormatter: (params: ValueFormatterParams) => - params.data?.owner_name || params.data?.owner_email || "—", - }, - { - field: "chunkCount", - headerName: "Chunks", - valueFormatter: (params: ValueFormatterParams) => - params.data?.chunkCount?.toString() || "-", - }, - { - field: "avgScore", - headerName: "Avg score", - cellRenderer: ({ value }: CustomCellRendererProps) => { - return ( - - {value?.toFixed(2) ?? "-"} - - ); - }, - }, - { - field: "embedding_model", - headerName: "Embedding model", - minWidth: 200, - cellRenderer: ({ data }: CustomCellRendererProps) => ( - - {data?.embedding_model || "—"} - - ), - }, - { - field: "embedding_dimensions", - headerName: "Dimensions", - width: 110, - cellRenderer: ({ data }: CustomCellRendererProps) => ( - - {typeof data?.embedding_dimensions === "number" - ? data.embedding_dimensions.toString() - : "—"} - - ), - }, - { - field: "status", - headerName: "Status", - cellRenderer: ({ data }: CustomCellRendererProps) => { - const status = data?.status || "active"; - const error = - typeof data?.error === "string" && data.error.trim().length > 0 - ? data.error.trim() - : undefined; - if (status === "failed" && error) { - return ( - - - - - - - Ingestion failed - - {data?.filename || "Unknown file"} - - -
- {error} -
- - - ); - } - return ; - }, - }, - { - cellRenderer: ({ data }: CustomCellRendererProps) => { - const status = data?.status || "active"; - if (status !== "active") { - return null; - } - return ; - }, - cellStyle: { - alignItems: "center", - display: "flex", - justifyContent: "center", - padding: 0, - }, - colId: "actions", - filter: false, - minWidth: 0, - width: 40, - resizable: false, - sortable: false, - initialFlex: 0, - }, - ]; + const handleBulkDelete = async () => { + if (selectedRows.length === 0) return; - const defaultColDef: ColDef = { - resizable: false, - suppressMovable: true, - initialFlex: 1, - minWidth: 100, - }; + try { + // Delete each file individually since the API expects one filename at a time + const deletePromises = selectedRows.map((row) => + deleteDocumentMutation.mutateAsync({ filename: row.filename }), + ); - const onSelectionChanged = useCallback(() => { - if (gridRef.current) { - const selectedNodes = gridRef.current.api.getSelectedRows(); - setSelectedRows(selectedNodes); - } - }, []); + await Promise.all(deletePromises); - const handleBulkDelete = async () => { - if (selectedRows.length === 0) return; + toast.success( + `Successfully deleted ${selectedRows.length} document${ + selectedRows.length > 1 ? "s" : "" + }`, + ); + setSelectedRows([]); + setShowBulkDeleteDialog(false); - try { - // Delete each file individually since the API expects one filename at a time - const deletePromises = selectedRows.map(row => - deleteDocumentMutation.mutateAsync({ filename: row.filename }) - ); + // Clear selection in the grid + if (gridRef.current) { + gridRef.current.api.deselectAll(); + } + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Failed to delete some documents", + ); + } + }; - await Promise.all(deletePromises); + return ( + <> +
+
+

Project Knowledge

+
- toast.success( - `Successfully deleted ${selectedRows.length} document${ - selectedRows.length > 1 ? "s" : "" - }` - ); - setSelectedRows([]); - setShowBulkDeleteDialog(false); - - // Clear selection in the grid - if (gridRef.current) { - gridRef.current.api.deselectAll(); - } - } catch (error) { - toast.error( - error instanceof Error - ? error.message - : "Failed to delete some documents" - ); - } - }; - - return ( - <> -
-
-

Project Knowledge

-
- - {/* Search Input Area */} -
- - {/* //TODO: Implement sync button */} - {/* */} - {selectedRows.length > 0 && ( - - )} -
- -
-
- []} - defaultColDef={defaultColDef} - loading={isFetching} - ref={gridRef} - theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} - rowData={fileResults} - rowSelection="multiple" - rowMultiSelectWithClick={false} - suppressRowClickSelection={true} - getRowId={(params: GetRowIdParams) => params.data?.filename} - domLayout="normal" - onSelectionChanged={onSelectionChanged} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> -
+ {selectedRows.length > 0 && ( + + )} +
+ +
+
+ []} + defaultColDef={defaultColDef} + loading={isFetching} + ref={gridRef} + theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} + rowData={fileResults} + rowSelection="multiple" + rowMultiSelectWithClick={false} + suppressRowClickSelection={true} + getRowId={(params: GetRowIdParams) => params.data?.filename} + domLayout="normal" + onSelectionChanged={onSelectionChanged} + noRowsOverlayComponent={() => ( +
+
+ No knowledge +
+
+ Add files from local or your preferred cloud. +
+
+ )} + /> + - {/* Bulk Delete Confirmation Dialog */} - 1 ? "s" : "" - }? This will remove all chunks and data associated with these documents. This action cannot be undone. + {/* Bulk Delete Confirmation Dialog */} + 1 ? "s" : "" + }? This will remove all chunks and data associated with these documents. This action cannot be undone. Documents to be deleted: -${selectedRows.map(row => `• ${row.filename}`).join("\n")}`} - confirmText="Delete All" - onConfirm={handleBulkDelete} - isLoading={deleteDocumentMutation.isPending} - /> - - ); +${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`} + confirmText="Delete All" + onConfirm={handleBulkDelete} + isLoading={deleteDocumentMutation.isPending} + /> + + ); } export default function ProtectedSearchPage() { - return ( - - - - ); + return ( + + + + ); } diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index a8e7747e..6ad1791d 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -140,12 +140,30 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { (prev) => prev.task_id === currentTask.task_id, ); - // Only show toasts if we have previous data and status has changed - if ( + // Check if task is in progress + const isTaskInProgress = + currentTask.status === "pending" || + currentTask.status === "running" || + currentTask.status === "processing"; + + // On initial load, previousTasksRef is empty, so we need to process all in-progress tasks + const isInitialLoad = previousTasksRef.current.length === 0; + + // Process files if: + // 1. Task is in progress (always process to keep files list updated) + // 2. Status has changed + // 3. New task appeared (not on initial load) + const shouldProcessFiles = + isTaskInProgress || (previousTask && previousTask.status !== currentTask.status) || - (!previousTask && previousTasksRef.current.length !== 0) - ) { - // Process files from failed task and add them to files list + (!previousTask && !isInitialLoad); + + // Only show toasts if we have previous data and status has changed + const shouldShowToast = + previousTask && previousTask.status !== currentTask.status; + + if (shouldProcessFiles) { + // Process files from task and add them to files list if (currentTask.files && typeof currentTask.files === "object") { const taskFileEntries = Object.entries(currentTask.files); const now = new Date().toISOString(); @@ -247,6 +265,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { }); } if ( + shouldShowToast && previousTask && previousTask.status !== "completed" && currentTask.status === "completed" @@ -283,13 +302,14 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { setFiles((prevFiles) => prevFiles.filter( (file) => - file.task_id !== currentTask.task_id || + file.status === "active" || file.status === "failed", ), ); refetchSearch(); }, 500); } else if ( + shouldShowToast && previousTask && previousTask.status !== "failed" && previousTask.status !== "error" && @@ -321,7 +341,13 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { ); const refreshTasks = useCallback(async () => { - setFiles([]); + setFiles((prevFiles) => + prevFiles.filter( + (file) => + file.status !== "active" && + file.status !== "failed", + ), + ); await refetchTasks(); }, [refetchTasks]);