Fixed failed tasks not going away, and processing tasks going away (#393)

This commit is contained in:
Lucas Oliveira 2025-11-12 16:30:23 -03:00 committed by GitHub
parent 7ecd6a23ea
commit 583ba2ded3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 398 additions and 372 deletions

View file

@ -1,11 +1,11 @@
"use client";
import {
themeQuartz,
type CheckboxSelectionCallbackParams,
type ColDef,
type GetRowIdParams,
type ValueFormatterParams,
type CheckboxSelectionCallbackParams,
type ColDef,
type GetRowIdParams,
themeQuartz,
type ValueFormatterParams,
} from "ag-grid-community";
import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react";
import { Cloud, FileIcon, Globe } from "lucide-react";
@ -21,331 +21,331 @@ import "@/components/AgGrid/registerAgGridModules";
import "@/components/AgGrid/agGridStyles.css";
import { toast } from "sonner";
import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdown";
import { KnowledgeSearchInput } from "@/components/knowledge-search-input";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
import { StatusBadge } from "@/components/ui/status-badge";
import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog";
import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
import GoogleDriveIcon from "../settings/icons/google-drive-icon";
import OneDriveIcon from "../settings/icons/one-drive-icon";
import SharePointIcon from "../settings/icons/share-point-icon";
import { KnowledgeSearchInput } from "@/components/knowledge-search-input";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog";
// Function to get the appropriate icon for a connector type
function getSourceIcon(connectorType?: string) {
switch (connectorType) {
case "google_drive":
return (
<GoogleDriveIcon className="h-4 w-4 text-foreground flex-shrink-0" />
);
case "onedrive":
return <OneDriveIcon className="h-4 w-4 text-foreground flex-shrink-0" />;
case "sharepoint":
return (
<SharePointIcon className="h-4 w-4 text-foreground flex-shrink-0" />
);
case "url":
return <Globe className="h-4 w-4 text-muted-foreground flex-shrink-0" />;
case "s3":
return <Cloud className="h-4 w-4 text-foreground flex-shrink-0" />;
default:
return (
<FileIcon className="h-4 w-4 text-muted-foreground flex-shrink-0" />
);
}
switch (connectorType) {
case "google_drive":
return (
<GoogleDriveIcon className="h-4 w-4 text-foreground flex-shrink-0" />
);
case "onedrive":
return <OneDriveIcon className="h-4 w-4 text-foreground flex-shrink-0" />;
case "sharepoint":
return (
<SharePointIcon className="h-4 w-4 text-foreground flex-shrink-0" />
);
case "url":
return <Globe className="h-4 w-4 text-muted-foreground flex-shrink-0" />;
case "s3":
return <Cloud className="h-4 w-4 text-foreground flex-shrink-0" />;
default:
return (
<FileIcon className="h-4 w-4 text-muted-foreground flex-shrink-0" />
);
}
}
function SearchPage() {
const router = useRouter();
const { files: taskFiles, refreshTasks } = useTask();
const { parsedFilterData, queryOverride } = useKnowledgeFilter();
const [selectedRows, setSelectedRows] = useState<File[]>([]);
const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false);
const router = useRouter();
const { files: taskFiles, refreshTasks } = useTask();
const { parsedFilterData, queryOverride } = useKnowledgeFilter();
const [selectedRows, setSelectedRows] = useState<File[]>([]);
const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false);
const deleteDocumentMutation = useDeleteDocument();
const deleteDocumentMutation = useDeleteDocument();
useEffect(() => {
refreshTasks();
}, [refreshTasks]);
useEffect(() => {
refreshTasks();
}, [refreshTasks]);
const { data: searchData = [], isFetching } = useGetSearchQuery(
queryOverride,
parsedFilterData
);
// Convert TaskFiles to File format and merge with backend results
const taskFilesAsFiles: File[] = taskFiles.map(taskFile => {
return {
filename: taskFile.filename,
mimetype: taskFile.mimetype,
source_url: taskFile.source_url,
size: taskFile.size,
connector_type: taskFile.connector_type,
status: taskFile.status,
error: taskFile.error,
embedding_model: taskFile.embedding_model,
embedding_dimensions: taskFile.embedding_dimensions,
};
});
const { data: searchData = [], isFetching } = useGetSearchQuery(
queryOverride,
parsedFilterData,
);
// Convert TaskFiles to File format and merge with backend results
const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => {
return {
filename: taskFile.filename,
mimetype: taskFile.mimetype,
source_url: taskFile.source_url,
size: taskFile.size,
connector_type: taskFile.connector_type,
status: taskFile.status,
error: taskFile.error,
embedding_model: taskFile.embedding_model,
embedding_dimensions: taskFile.embedding_dimensions,
};
});
// Create a map of task files by filename for quick lookup
const taskFileMap = new Map(
taskFilesAsFiles.map((file) => [file.filename, file]),
);
// Override backend files with task file status if they exist
const backendFiles = (searchData as File[])
.map((file) => {
const taskFile = taskFileMap.get(file.filename);
if (taskFile) {
// Override backend file with task file data (includes status)
return { ...file, ...taskFile };
}
return file;
})
.filter((file) => {
// Only filter out files that are currently processing AND in taskFiles
const taskFile = taskFileMap.get(file.filename);
return !taskFile || taskFile.status !== "processing";
});
// Create a map of task files by filename for quick lookup
const taskFileMap = new Map(
taskFilesAsFiles.map(file => [file.filename, file])
);
// Override backend files with task file status if they exist
const backendFiles = (searchData as File[])
.map(file => {
const taskFile = taskFileMap.get(file.filename);
if (taskFile) {
// Override backend file with task file data (includes status)
return { ...file, ...taskFile };
}
return file;
})
.filter(file => {
// Only filter out files that are currently processing AND in taskFiles
const taskFile = taskFileMap.get(file.filename);
return !taskFile || taskFile.status !== "processing";
});
const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {
return (
taskFile.status !== "active" &&
!backendFiles.some(
(backendFile) => backendFile.filename === taskFile.filename,
)
);
});
// Combine task files first, then backend files
const fileResults = [...backendFiles, ...filteredTaskFiles];
const gridRef = useRef<AgGridReact>(null);
const filteredTaskFiles = taskFilesAsFiles.filter(taskFile => {
return (
taskFile.status !== "active" &&
!backendFiles.some(
backendFile => backendFile.filename === taskFile.filename
)
);
});
const columnDefs: ColDef<File>[] = [
{
field: "filename",
headerName: "Source",
checkboxSelection: (params: CheckboxSelectionCallbackParams<File>) =>
(params?.data?.status || "active") === "active",
headerCheckboxSelection: true,
initialFlex: 2,
minWidth: 220,
cellRenderer: ({ data, value }: CustomCellRendererProps<File>) => {
// Read status directly from data on each render
const status = data?.status || "active";
const isActive = status === "active";
return (
<div className="flex items-center overflow-hidden w-full">
<div
className={`transition-opacity duration-200 ${
isActive ? "w-0" : "w-7"
}`}
></div>
<button
type="button"
className="flex items-center gap-2 cursor-pointer hover:text-blue-600 transition-colors text-left flex-1 overflow-hidden"
onClick={() => {
if (!isActive) {
return;
}
router.push(
`/knowledge/chunks?filename=${encodeURIComponent(
data?.filename ?? "",
)}`,
);
}}
>
{getSourceIcon(data?.connector_type)}
<span className="font-medium text-foreground truncate">
{value}
</span>
</button>
</div>
);
},
},
{
field: "size",
headerName: "Size",
valueFormatter: (params: ValueFormatterParams<File>) =>
params.value ? `${Math.round(params.value / 1024)} KB` : "-",
},
{
field: "mimetype",
headerName: "Type",
},
{
field: "owner",
headerName: "Owner",
valueFormatter: (params: ValueFormatterParams<File>) =>
params.data?.owner_name || params.data?.owner_email || "—",
},
{
field: "chunkCount",
headerName: "Chunks",
valueFormatter: (params: ValueFormatterParams<File>) =>
params.data?.chunkCount?.toString() || "-",
},
{
field: "avgScore",
headerName: "Avg score",
cellRenderer: ({ value }: CustomCellRendererProps<File>) => {
return (
<span className="text-xs text-accent-emerald-foreground bg-accent-emerald px-2 py-1 rounded">
{value?.toFixed(2) ?? "-"}
</span>
);
},
},
{
field: "embedding_model",
headerName: "Embedding model",
minWidth: 200,
cellRenderer: ({ data }: CustomCellRendererProps<File>) => (
<span className="text-xs text-muted-foreground">
{data?.embedding_model || "—"}
</span>
),
},
{
field: "embedding_dimensions",
headerName: "Dimensions",
width: 110,
cellRenderer: ({ data }: CustomCellRendererProps<File>) => (
<span className="text-xs text-muted-foreground">
{typeof data?.embedding_dimensions === "number"
? data.embedding_dimensions.toString()
: "—"}
</span>
),
},
{
field: "status",
headerName: "Status",
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
const status = data?.status || "active";
const error =
typeof data?.error === "string" && data.error.trim().length > 0
? data.error.trim()
: undefined;
if (status === "failed" && error) {
return (
<Dialog>
<DialogTrigger asChild>
<button
type="button"
className="inline-flex items-center gap-1 text-red-500 transition hover:text-red-400"
aria-label="View ingestion error"
>
<StatusBadge
status={status}
className="pointer-events-none"
/>
</button>
</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>Ingestion failed</DialogTitle>
<DialogDescription className="text-sm text-muted-foreground">
{data?.filename || "Unknown file"}
</DialogDescription>
</DialogHeader>
<div className="rounded-md border border-destructive/20 bg-destructive/10 p-4 text-sm text-destructive">
{error}
</div>
</DialogContent>
</Dialog>
);
}
return <StatusBadge status={status} />;
},
},
{
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
const status = data?.status || "active";
if (status !== "active") {
return null;
}
return <KnowledgeActionsDropdown filename={data?.filename || ""} />;
},
cellStyle: {
alignItems: "center",
display: "flex",
justifyContent: "center",
padding: 0,
},
colId: "actions",
filter: false,
minWidth: 0,
width: 40,
resizable: false,
sortable: false,
initialFlex: 0,
},
];
// Combine task files first, then backend files
const fileResults = [...backendFiles, ...filteredTaskFiles];
const defaultColDef: ColDef<File> = {
resizable: false,
suppressMovable: true,
initialFlex: 1,
minWidth: 100,
};
const gridRef = useRef<AgGridReact>(null);
const onSelectionChanged = useCallback(() => {
if (gridRef.current) {
const selectedNodes = gridRef.current.api.getSelectedRows();
setSelectedRows(selectedNodes);
}
}, []);
const columnDefs: ColDef<File>[] = [
{
field: "filename",
headerName: "Source",
checkboxSelection: (params: CheckboxSelectionCallbackParams<File>) =>
(params?.data?.status || "active") === "active",
headerCheckboxSelection: true,
initialFlex: 2,
minWidth: 220,
cellRenderer: ({ data, value }: CustomCellRendererProps<File>) => {
// Read status directly from data on each render
const status = data?.status || "active";
const isActive = status === "active";
return (
<div className="flex items-center overflow-hidden w-full">
<div
className={`transition-opacity duration-200 ${
isActive ? "w-0" : "w-7"
}`}
></div>
<button
type="button"
className="flex items-center gap-2 cursor-pointer hover:text-blue-600 transition-colors text-left flex-1 overflow-hidden"
onClick={() => {
if (!isActive) {
return;
}
router.push(
`/knowledge/chunks?filename=${encodeURIComponent(
data?.filename ?? ""
)}`
);
}}
>
{getSourceIcon(data?.connector_type)}
<span className="font-medium text-foreground truncate">
{value}
</span>
</button>
</div>
);
},
},
{
field: "size",
headerName: "Size",
valueFormatter: (params: ValueFormatterParams<File>) =>
params.value ? `${Math.round(params.value / 1024)} KB` : "-",
},
{
field: "mimetype",
headerName: "Type",
},
{
field: "owner",
headerName: "Owner",
valueFormatter: (params: ValueFormatterParams<File>) =>
params.data?.owner_name || params.data?.owner_email || "—",
},
{
field: "chunkCount",
headerName: "Chunks",
valueFormatter: (params: ValueFormatterParams<File>) =>
params.data?.chunkCount?.toString() || "-",
},
{
field: "avgScore",
headerName: "Avg score",
cellRenderer: ({ value }: CustomCellRendererProps<File>) => {
return (
<span className="text-xs text-accent-emerald-foreground bg-accent-emerald px-2 py-1 rounded">
{value?.toFixed(2) ?? "-"}
</span>
);
},
},
{
field: "embedding_model",
headerName: "Embedding model",
minWidth: 200,
cellRenderer: ({ data }: CustomCellRendererProps<File>) => (
<span className="text-xs text-muted-foreground">
{data?.embedding_model || "—"}
</span>
),
},
{
field: "embedding_dimensions",
headerName: "Dimensions",
width: 110,
cellRenderer: ({ data }: CustomCellRendererProps<File>) => (
<span className="text-xs text-muted-foreground">
{typeof data?.embedding_dimensions === "number"
? data.embedding_dimensions.toString()
: "—"}
</span>
),
},
{
field: "status",
headerName: "Status",
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
const status = data?.status || "active";
const error =
typeof data?.error === "string" && data.error.trim().length > 0
? data.error.trim()
: undefined;
if (status === "failed" && error) {
return (
<Dialog>
<DialogTrigger asChild>
<button
type="button"
className="inline-flex items-center gap-1 text-red-500 transition hover:text-red-400"
aria-label="View ingestion error"
>
<StatusBadge status={status} className="pointer-events-none" />
</button>
</DialogTrigger>
<DialogContent>
<DialogHeader>
<DialogTitle>Ingestion failed</DialogTitle>
<DialogDescription className="text-sm text-muted-foreground">
{data?.filename || "Unknown file"}
</DialogDescription>
</DialogHeader>
<div className="rounded-md border border-destructive/20 bg-destructive/10 p-4 text-sm text-destructive">
{error}
</div>
</DialogContent>
</Dialog>
);
}
return <StatusBadge status={status} />;
},
},
{
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
const status = data?.status || "active";
if (status !== "active") {
return null;
}
return <KnowledgeActionsDropdown filename={data?.filename || ""} />;
},
cellStyle: {
alignItems: "center",
display: "flex",
justifyContent: "center",
padding: 0,
},
colId: "actions",
filter: false,
minWidth: 0,
width: 40,
resizable: false,
sortable: false,
initialFlex: 0,
},
];
const handleBulkDelete = async () => {
if (selectedRows.length === 0) return;
const defaultColDef: ColDef<File> = {
resizable: false,
suppressMovable: true,
initialFlex: 1,
minWidth: 100,
};
try {
// Delete each file individually since the API expects one filename at a time
const deletePromises = selectedRows.map((row) =>
deleteDocumentMutation.mutateAsync({ filename: row.filename }),
);
const onSelectionChanged = useCallback(() => {
if (gridRef.current) {
const selectedNodes = gridRef.current.api.getSelectedRows();
setSelectedRows(selectedNodes);
}
}, []);
await Promise.all(deletePromises);
const handleBulkDelete = async () => {
if (selectedRows.length === 0) return;
toast.success(
`Successfully deleted ${selectedRows.length} document${
selectedRows.length > 1 ? "s" : ""
}`,
);
setSelectedRows([]);
setShowBulkDeleteDialog(false);
try {
// Delete each file individually since the API expects one filename at a time
const deletePromises = selectedRows.map(row =>
deleteDocumentMutation.mutateAsync({ filename: row.filename })
);
// Clear selection in the grid
if (gridRef.current) {
gridRef.current.api.deselectAll();
}
} catch (error) {
toast.error(
error instanceof Error
? error.message
: "Failed to delete some documents",
);
}
};
await Promise.all(deletePromises);
return (
<>
<div className="flex flex-col h-full">
<div className="flex items-center justify-between mb-6">
<h2 className="text-lg font-semibold">Project Knowledge</h2>
</div>
toast.success(
`Successfully deleted ${selectedRows.length} document${
selectedRows.length > 1 ? "s" : ""
}`
);
setSelectedRows([]);
setShowBulkDeleteDialog(false);
// Clear selection in the grid
if (gridRef.current) {
gridRef.current.api.deselectAll();
}
} catch (error) {
toast.error(
error instanceof Error
? error.message
: "Failed to delete some documents"
);
}
};
return (
<>
<div className="flex flex-col h-full">
<div className="flex items-center justify-between mb-6">
<h2 className="text-lg font-semibold">Project Knowledge</h2>
</div>
{/* Search Input Area */}
<div className="flex-1 flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6">
<KnowledgeSearchInput />
{/* //TODO: Implement sync button */}
{/* <Button
{/* Search Input Area */}
<div className="flex-1 flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6">
<KnowledgeSearchInput />
{/* //TODO: Implement sync button */}
{/* <Button
type="button"
variant="outline"
className="rounded-lg flex-shrink-0"
@ -353,72 +353,72 @@ function SearchPage() {
>
Sync
</Button> */}
{selectedRows.length > 0 && (
<Button
type="button"
variant="destructive"
className="rounded-lg flex-shrink-0"
onClick={() => setShowBulkDeleteDialog(true)}
>
Delete
</Button>
)}
<div className="ml-auto">
<KnowledgeDropdown />
</div>
</div>
<AgGridReact
className="w-full overflow-auto"
columnDefs={columnDefs as ColDef<File>[]}
defaultColDef={defaultColDef}
loading={isFetching}
ref={gridRef}
theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}
rowData={fileResults}
rowSelection="multiple"
rowMultiSelectWithClick={false}
suppressRowClickSelection={true}
getRowId={(params: GetRowIdParams<File>) => params.data?.filename}
domLayout="normal"
onSelectionChanged={onSelectionChanged}
noRowsOverlayComponent={() => (
<div className="text-center pb-[45px]">
<div className="text-lg text-primary font-semibold">
No knowledge
</div>
<div className="text-sm mt-1 text-muted-foreground">
Add files from local or your preferred cloud.
</div>
</div>
)}
/>
</div>
{selectedRows.length > 0 && (
<Button
type="button"
variant="destructive"
className="rounded-lg flex-shrink-0"
onClick={() => setShowBulkDeleteDialog(true)}
>
Delete
</Button>
)}
<div className="ml-auto">
<KnowledgeDropdown />
</div>
</div>
<AgGridReact
className="w-full overflow-auto"
columnDefs={columnDefs as ColDef<File>[]}
defaultColDef={defaultColDef}
loading={isFetching}
ref={gridRef}
theme={themeQuartz.withParams({ browserColorScheme: "inherit" })}
rowData={fileResults}
rowSelection="multiple"
rowMultiSelectWithClick={false}
suppressRowClickSelection={true}
getRowId={(params: GetRowIdParams<File>) => params.data?.filename}
domLayout="normal"
onSelectionChanged={onSelectionChanged}
noRowsOverlayComponent={() => (
<div className="text-center pb-[45px]">
<div className="text-lg text-primary font-semibold">
No knowledge
</div>
<div className="text-sm mt-1 text-muted-foreground">
Add files from local or your preferred cloud.
</div>
</div>
)}
/>
</div>
{/* Bulk Delete Confirmation Dialog */}
<DeleteConfirmationDialog
open={showBulkDeleteDialog}
onOpenChange={setShowBulkDeleteDialog}
title="Delete Documents"
description={`Are you sure you want to delete ${
selectedRows.length
} document${
selectedRows.length > 1 ? "s" : ""
}? This will remove all chunks and data associated with these documents. This action cannot be undone.
{/* Bulk Delete Confirmation Dialog */}
<DeleteConfirmationDialog
open={showBulkDeleteDialog}
onOpenChange={setShowBulkDeleteDialog}
title="Delete Documents"
description={`Are you sure you want to delete ${
selectedRows.length
} document${
selectedRows.length > 1 ? "s" : ""
}? This will remove all chunks and data associated with these documents. This action cannot be undone.
Documents to be deleted:
${selectedRows.map(row => `${row.filename}`).join("\n")}`}
confirmText="Delete All"
onConfirm={handleBulkDelete}
isLoading={deleteDocumentMutation.isPending}
/>
</>
);
${selectedRows.map((row) => `${row.filename}`).join("\n")}`}
confirmText="Delete All"
onConfirm={handleBulkDelete}
isLoading={deleteDocumentMutation.isPending}
/>
</>
);
}
export default function ProtectedSearchPage() {
return (
<ProtectedRoute>
<SearchPage />
</ProtectedRoute>
);
return (
<ProtectedRoute>
<SearchPage />
</ProtectedRoute>
);
}

View file

@ -140,12 +140,30 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
(prev) => prev.task_id === currentTask.task_id,
);
// Only show toasts if we have previous data and status has changed
if (
// Check if task is in progress
const isTaskInProgress =
currentTask.status === "pending" ||
currentTask.status === "running" ||
currentTask.status === "processing";
// On initial load, previousTasksRef is empty, so we need to process all in-progress tasks
const isInitialLoad = previousTasksRef.current.length === 0;
// Process files if:
// 1. Task is in progress (always process to keep files list updated)
// 2. Status has changed
// 3. New task appeared (not on initial load)
const shouldProcessFiles =
isTaskInProgress ||
(previousTask && previousTask.status !== currentTask.status) ||
(!previousTask && previousTasksRef.current.length !== 0)
) {
// Process files from failed task and add them to files list
(!previousTask && !isInitialLoad);
// Only show toasts if we have previous data and status has changed
const shouldShowToast =
previousTask && previousTask.status !== currentTask.status;
if (shouldProcessFiles) {
// Process files from task and add them to files list
if (currentTask.files && typeof currentTask.files === "object") {
const taskFileEntries = Object.entries(currentTask.files);
const now = new Date().toISOString();
@ -247,6 +265,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
});
}
if (
shouldShowToast &&
previousTask &&
previousTask.status !== "completed" &&
currentTask.status === "completed"
@ -283,13 +302,14 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
setFiles((prevFiles) =>
prevFiles.filter(
(file) =>
file.task_id !== currentTask.task_id ||
file.status === "active" ||
file.status === "failed",
),
);
refetchSearch();
}, 500);
} else if (
shouldShowToast &&
previousTask &&
previousTask.status !== "failed" &&
previousTask.status !== "error" &&
@ -321,7 +341,13 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
);
const refreshTasks = useCallback(async () => {
setFiles([]);
setFiles((prevFiles) =>
prevFiles.filter(
(file) =>
file.status !== "active" &&
file.status !== "failed",
),
);
await refetchTasks();
}, [refetchTasks]);