Fixed failed tasks not going away, and processing tasks going away (#393)

frontend/src/app/knowledge/page.tsx

@@ -1,10 +1,10 @@
 "use client";
 
 import {
-  themeQuartz,
 	type CheckboxSelectionCallbackParams,
 	type ColDef,
 	type GetRowIdParams,
+	themeQuartz,
 	type ValueFormatterParams,
 } from "ag-grid-community";
 import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react";
@@ -21,12 +21,6 @@ import "@/components/AgGrid/registerAgGridModules";
 import "@/components/AgGrid/agGridStyles.css";
 import { toast } from "sonner";
 import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdown";
-import { StatusBadge } from "@/components/ui/status-badge";
-import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog";
-import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
-import GoogleDriveIcon from "../settings/icons/google-drive-icon";
-import OneDriveIcon from "../settings/icons/one-drive-icon";
-import SharePointIcon from "../settings/icons/share-point-icon";
 import { KnowledgeSearchInput } from "@/components/knowledge-search-input";
 import {
 	Dialog,
@@ -36,6 +30,12 @@ import {
 	DialogTitle,
 	DialogTrigger,
 } from "@/components/ui/dialog";
+import { StatusBadge } from "@/components/ui/status-badge";
+import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog";
+import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
+import GoogleDriveIcon from "../settings/icons/google-drive-icon";
+import OneDriveIcon from "../settings/icons/one-drive-icon";
+import SharePointIcon from "../settings/icons/share-point-icon";
 
 // Function to get the appropriate icon for a connector type
 function getSourceIcon(connectorType?: string) {
@@ -76,10 +76,10 @@ function SearchPage() {
 
 	const { data: searchData = [], isFetching } = useGetSearchQuery(
 		queryOverride,
-    parsedFilterData
+		parsedFilterData,
 	);
 	// Convert TaskFiles to File format and merge with backend results
-  const taskFilesAsFiles: File[] = taskFiles.map(taskFile => {
+	const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => {
 		return {
 			filename: taskFile.filename,
 			mimetype: taskFile.mimetype,
@@ -92,15 +92,13 @@ function SearchPage() {
 			embedding_dimensions: taskFile.embedding_dimensions,
 		};
 	});
-
 	// Create a map of task files by filename for quick lookup
 	const taskFileMap = new Map(
-    taskFilesAsFiles.map(file => [file.filename, file])
+		taskFilesAsFiles.map((file) => [file.filename, file]),
 	);
-
 	// Override backend files with task file status if they exist
 	const backendFiles = (searchData as File[])
-    .map(file => {
+		.map((file) => {
 			const taskFile = taskFileMap.get(file.filename);
 			if (taskFile) {
 				// Override backend file with task file data (includes status)
@@ -108,24 +106,23 @@ function SearchPage() {
 			}
 			return file;
 		})
-    .filter(file => {
+		.filter((file) => {
 			// Only filter out files that are currently processing AND in taskFiles
 			const taskFile = taskFileMap.get(file.filename);
 			return !taskFile || taskFile.status !== "processing";
 		});
 
-  const filteredTaskFiles = taskFilesAsFiles.filter(taskFile => {
+
+	const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {
 		return (
 			taskFile.status !== "active" &&
 			!backendFiles.some(
-        backendFile => backendFile.filename === taskFile.filename
+				(backendFile) => backendFile.filename === taskFile.filename,
 			)
 		);
 	});
-
 	// Combine task files first, then backend files
 	const fileResults = [...backendFiles, ...filteredTaskFiles];
-
 	const gridRef = useRef<AgGridReact>(null);
 
 	const columnDefs: ColDef<File>[] = [
@@ -157,8 +154,8 @@ function SearchPage() {
 								}
 								router.push(
 									`/knowledge/chunks?filename=${encodeURIComponent(
-                    data?.filename ?? ""
+										data?.filename ?? "",
-                  )}`
+									)}`,
 								);
 							}}
 						>
@@ -244,7 +241,10 @@ function SearchPage() {
 									className="inline-flex items-center gap-1 text-red-500 transition hover:text-red-400"
 									aria-label="View ingestion error"
 								>
-                  <StatusBadge status={status} className="pointer-events-none" />
+									<StatusBadge
+										status={status}
+										className="pointer-events-none"
+									/>
 								</button>
 							</DialogTrigger>
 							<DialogContent>
@@ -307,8 +307,8 @@ function SearchPage() {
 
 		try {
 			// Delete each file individually since the API expects one filename at a time
-      const deletePromises = selectedRows.map(row =>
+			const deletePromises = selectedRows.map((row) =>
-        deleteDocumentMutation.mutateAsync({ filename: row.filename })
+				deleteDocumentMutation.mutateAsync({ filename: row.filename }),
 			);
 
 			await Promise.all(deletePromises);
@@ -316,7 +316,7 @@ function SearchPage() {
 			toast.success(
 				`Successfully deleted ${selectedRows.length} document${
 					selectedRows.length > 1 ? "s" : ""
-        }`
+				}`,
 			);
 			setSelectedRows([]);
 			setShowBulkDeleteDialog(false);
@@ -329,7 +329,7 @@ function SearchPage() {
 			toast.error(
 				error instanceof Error
 					? error.message
-          : "Failed to delete some documents"
+					: "Failed to delete some documents",
 			);
 		}
 	};
@@ -406,7 +406,7 @@ function SearchPage() {
 				}? This will remove all chunks and data associated with these documents. This action cannot be undone.
 
 Documents to be deleted:
-${selectedRows.map(row => `• ${row.filename}`).join("\n")}`}
+${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`}
 				confirmText="Delete All"
 				onConfirm={handleBulkDelete}
 				isLoading={deleteDocumentMutation.isPending}

frontend/src/contexts/task-context.tsx

@@ -140,12 +140,30 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
 				(prev) => prev.task_id === currentTask.task_id,
 			);
 
-			// Only show toasts if we have previous data and status has changed
+			// Check if task is in progress
-			if (
+			const isTaskInProgress =
+				currentTask.status === "pending" ||
+				currentTask.status === "running" ||
+				currentTask.status === "processing";
+
+			// On initial load, previousTasksRef is empty, so we need to process all in-progress tasks
+			const isInitialLoad = previousTasksRef.current.length === 0;
+
+			// Process files if:
+			// 1. Task is in progress (always process to keep files list updated)
+			// 2. Status has changed
+			// 3. New task appeared (not on initial load)
+			const shouldProcessFiles =
+				isTaskInProgress ||
 				(previousTask && previousTask.status !== currentTask.status) ||
-				(!previousTask && previousTasksRef.current.length !== 0)
+				(!previousTask && !isInitialLoad);
-			) {
+
-				// Process files from failed task and add them to files list
+			// Only show toasts if we have previous data and status has changed
+			const shouldShowToast =
+				previousTask && previousTask.status !== currentTask.status;
+
+			if (shouldProcessFiles) {
+				// Process files from task and add them to files list
 				if (currentTask.files && typeof currentTask.files === "object") {
 					const taskFileEntries = Object.entries(currentTask.files);
 					const now = new Date().toISOString();
@@ -247,6 +265,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
 					});
 				}
 				if (
+					shouldShowToast &&
 					previousTask &&
 					previousTask.status !== "completed" &&
 					currentTask.status === "completed"
@@ -283,13 +302,14 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
 						setFiles((prevFiles) =>
 							prevFiles.filter(
 								(file) =>
-									file.task_id !== currentTask.task_id ||
+									file.status === "active" ||
 									file.status === "failed",
 							),
 						);
 						refetchSearch();
 					}, 500);
 				} else if (
+					shouldShowToast &&
 					previousTask &&
 					previousTask.status !== "failed" &&
 					previousTask.status !== "error" &&
@@ -321,7 +341,13 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
 	);
 
 	const refreshTasks = useCallback(async () => {
-		setFiles([]);
+		setFiles((prevFiles) =>
+			prevFiles.filter(
+				(file) =>
+					file.status !== "active" &&
+					file.status !== "failed",
+			),
+		);
 		await refetchTasks();
 	}, [refetchTasks]);