gmakstutis/openrag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

made files and toast appear just once, use new queries

Browse source
This commit is contained in:
Lucas Oliveira 2025-10-02 16:50:47 -03:00
parent 41e4ecefdb
commit aa61ba265c
2 changed files with 733 additions and 811 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
frontend/components/knowledge-dropdown.tsx
View file

@ -12,6 +12,7 @@ import {
import { useRouter } from "next/navigation"; import { useRouter } from "next/navigation";
import { useEffect, useRef, useState } from "react"; import { useEffect, useRef, useState } from "react";
import { toast } from "sonner"; import { toast } from "sonner";
import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { import {
Dialog, Dialog,
@ -35,6 +36,7 @@ export function KnowledgeDropdown({
variant = "navigation", variant = "navigation",
}: KnowledgeDropdownProps) { }: KnowledgeDropdownProps) {
const { addTask } = useTask(); const { addTask } = useTask();
const { refetch: refetchTasks } = useGetTasksQuery();
const router = useRouter(); const router = useRouter();
const [isOpen, setIsOpen] = useState(false); const [isOpen, setIsOpen] = useState(false);
const [showFolderDialog, setShowFolderDialog] = useState(false); const [showFolderDialog, setShowFolderDialog] = useState(false);
@ -103,7 +105,7 @@ export function KnowledgeDropdown({
const connections = statusData.connections || []; const connections = statusData.connections || [];
const activeConnection = connections.find( const activeConnection = connections.find(
(conn: { is_active: boolean; connection_id: string }) => (conn: { is_active: boolean; connection_id: string }) =>
conn.is_active conn.is_active,
); );
const isConnected = activeConnection !== undefined; const isConnected = activeConnection !== undefined;
@ -113,7 +115,7 @@ export function KnowledgeDropdown({
// Check token availability // Check token availability
try { try {
const tokenRes = await fetch( const tokenRes = await fetch(
`/api/connectors/${type}/token?connection_id=${activeConnection.connection_id}` `/api/connectors/${type}/token?connection_id=${activeConnection.connection_id}`,
); );
if (tokenRes.ok) { if (tokenRes.ok) {
const tokenData = await tokenRes.json(); const tokenData = await tokenRes.json();
@ -174,7 +176,7 @@ export function KnowledgeDropdown({
window.dispatchEvent( window.dispatchEvent(
new CustomEvent("fileUploadStart", { new CustomEvent("fileUploadStart", {
detail: { filename: files[0].name }, detail: { filename: files[0].name },
}) }),
); );
try { try {
@ -191,23 +193,22 @@ export function KnowledgeDropdown({
if (!uploadIngestRes.ok) { if (!uploadIngestRes.ok) {
throw new Error( throw new Error(
uploadIngestJson?.error || "Upload and ingest failed" uploadIngestJson?.error || "Upload and ingest failed",
); );
} }
// Extract results from the response - handle both unified and simple formats // Extract results from the response - handle both unified and simple formats
const fileId = uploadIngestJson?.upload?.id || uploadIngestJson?.id; const fileId = uploadIngestJson?.upload?.id || uploadIngestJson?.id || uploadIngestJson?.task_id;
const filePath = const filePath =
uploadIngestJson?.upload?.path || uploadIngestJson?.upload?.path ||
uploadIngestJson?.path || uploadIngestJson?.path ||
"uploaded"; "uploaded";
const runJson = uploadIngestJson?.ingestion; const runJson = uploadIngestJson?.ingestion;
const deleteResult = uploadIngestJson?.deletion; const deleteResult = uploadIngestJson?.deletion;
console.log("c", uploadIngestJson )
if (!fileId) { if (!fileId) {
throw new Error("Upload successful but no file id returned"); throw new Error("Upload successful but no file id returned");
} }
// Check if ingestion actually succeeded // Check if ingestion actually succeeded
if ( if (
runJson && runJson &&
@ -216,25 +217,23 @@ export function KnowledgeDropdown({
) { ) {
const errorMsg = runJson.error || "Ingestion pipeline failed"; const errorMsg = runJson.error || "Ingestion pipeline failed";
throw new Error( throw new Error(
`Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.` `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.`,
); );
} }
// Log deletion status if provided // Log deletion status if provided
if (deleteResult) { if (deleteResult) {
if (deleteResult.status === "deleted") { if (deleteResult.status === "deleted") {
console.log( console.log(
"File successfully cleaned up from Langflow:", "File successfully cleaned up from Langflow:",
deleteResult.file_id deleteResult.file_id,
); );
} else if (deleteResult.status === "delete_failed") { } else if (deleteResult.status === "delete_failed") {
console.warn( console.warn(
"Failed to cleanup file from Langflow:", "Failed to cleanup file from Langflow:",
deleteResult.error deleteResult.error,
); );
} }
} }
// Notify UI // Notify UI
window.dispatchEvent( window.dispatchEvent(
new CustomEvent("fileUploaded", { new CustomEvent("fileUploaded", {
@ -248,11 +247,10 @@ export function KnowledgeDropdown({
unified: true, unified: true,
}, },
}, },
}) }),
); );
// Trigger search refresh after successful ingestion refetchTasks();
window.dispatchEvent(new CustomEvent("knowledgeUpdated"));
} catch (error) { } catch (error) {
window.dispatchEvent( window.dispatchEvent(
new CustomEvent("fileUploadError", { new CustomEvent("fileUploadError", {
@ -260,12 +258,11 @@ export function KnowledgeDropdown({
filename: files[0].name, filename: files[0].name,
error: error instanceof Error ? error.message : "Upload failed", error: error instanceof Error ? error.message : "Upload failed",
}, },
}) }),
); );
} finally { } finally {
window.dispatchEvent(new CustomEvent("fileUploadComplete")); window.dispatchEvent(new CustomEvent("fileUploadComplete"));
setFileUploading(false); setFileUploading(false);
// Don't call refetchSearch() here - the knowledgeUpdated event will handle it
} }
} }
@ -301,17 +298,12 @@ export function KnowledgeDropdown({
addTask(taskId); addTask(taskId);
setFolderPath(""); setFolderPath("");
// Trigger search refresh after successful folder processing starts // Refetch tasks to show the new task
console.log( refetchTasks();
"Folder upload successful, dispatching knowledgeUpdated event"
);
window.dispatchEvent(new CustomEvent("knowledgeUpdated"));
} else if (response.ok) { } else if (response.ok) {
setFolderPath(""); setFolderPath("");
console.log( // Refetch tasks even for direct uploads in case tasks were created
"Folder upload successful (direct), dispatching knowledgeUpdated event" refetchTasks();
);
window.dispatchEvent(new CustomEvent("knowledgeUpdated"));
} else { } else {
console.error("Folder upload failed:", result.error); console.error("Folder upload failed:", result.error);
if (response.status === 400) { if (response.status === 400) {
@ -324,7 +316,6 @@ export function KnowledgeDropdown({
console.error("Folder upload error:", error); console.error("Folder upload error:", error);
} finally { } finally {
setFolderLoading(false); setFolderLoading(false);
// Don't call refetchSearch() here - the knowledgeUpdated event will handle it
} }
}; };
@ -354,9 +345,8 @@ export function KnowledgeDropdown({
addTask(taskId); addTask(taskId);
setBucketUrl("s3://"); setBucketUrl("s3://");
// Trigger search refresh after successful S3 processing starts // Refetch tasks to show the new task
console.log("S3 upload successful, dispatching knowledgeUpdated event"); refetchTasks();
window.dispatchEvent(new CustomEvent("knowledgeUpdated"));
} else { } else {
console.error("S3 upload failed:", result.error); console.error("S3 upload failed:", result.error);
if (response.status === 400) { if (response.status === 400) {
@ -369,7 +359,6 @@ export function KnowledgeDropdown({
console.error("S3 upload error:", error); console.error("S3 upload error:", error);
} finally { } finally {
setS3Loading(false); setS3Loading(false);
// Don't call refetchSearch() here - the knowledgeUpdated event will handle it
} }
}; };
@ -438,6 +427,7 @@ export function KnowledgeDropdown({
<> <>
<div ref={dropdownRef} className="relative"> <div ref={dropdownRef} className="relative">
<button <button
type="button"
onClick={() => !isLoading && setIsOpen(!isOpen)} onClick={() => !isLoading && setIsOpen(!isOpen)}
disabled={isLoading} disabled={isLoading}
className={cn( className={cn(
@ -448,7 +438,7 @@ export function KnowledgeDropdown({
? "bg-accent text-accent-foreground shadow-sm" ? "bg-accent text-accent-foreground shadow-sm"
: variant === "navigation" : variant === "navigation"
? "text-foreground hover:text-accent-foreground" ? "text-foreground hover:text-accent-foreground"
: "" : "",
)} )}
> >
{variant === "button" ? ( {variant === "button" ? (
@ -475,7 +465,7 @@ export function KnowledgeDropdown({
<ChevronDown <ChevronDown
className={cn( className={cn(
"h-4 w-4 transition-transform", "h-4 w-4 transition-transform",
isOpen && "rotate-180" isOpen && "rotate-180",
)} )}
/> />
)} )}
@ -491,7 +481,7 @@ export function KnowledgeDropdown({
"h-4 w-4 mr-3 shrink-0", "h-4 w-4 mr-3 shrink-0",
active active
? "text-accent-foreground" ? "text-accent-foreground"
: "text-muted-foreground group-hover:text-foreground" : "text-muted-foreground group-hover:text-foreground",
)} )}
/> />
)} )}
@ -501,7 +491,7 @@ export function KnowledgeDropdown({
<ChevronDown <ChevronDown
className={cn( className={cn(
"h-4 w-4 transition-transform", "h-4 w-4 transition-transform",
isOpen && "rotate-180" isOpen && "rotate-180",
)} )}
/> />
)} )}
@ -514,7 +504,8 @@ export function KnowledgeDropdown({
<div className="py-1"> <div className="py-1">
{menuItems.map((item, index) => ( {menuItems.map((item, index) => (
<button <button
key={index} key={`${item.label}-${index}`}
type="button"
onClick={item.onClick} onClick={item.onClick}
disabled={"disabled" in item ? item.disabled : false} disabled={"disabled" in item ? item.disabled : false}
title={"tooltip" in item ? item.tooltip : undefined} title={"tooltip" in item ? item.tooltip : undefined}
@ -522,7 +513,7 @@ export function KnowledgeDropdown({
"w-full px-3 py-2 text-left text-sm hover:bg-accent hover:text-accent-foreground", "w-full px-3 py-2 text-left text-sm hover:bg-accent hover:text-accent-foreground",
"disabled" in item && "disabled" in item &&
item.disabled && item.disabled &&
"opacity-50 cursor-not-allowed hover:bg-transparent hover:text-current" "opacity-50 cursor-not-allowed hover:bg-transparent hover:text-current",
)} )}
> >
{item.label} {item.label}
@ -561,7 +552,7 @@ export function KnowledgeDropdown({
type="text" type="text"
placeholder="/path/to/documents" placeholder="/path/to/documents"
value={folderPath} value={folderPath}
onChange={e => setFolderPath(e.target.value)} onChange={(e) => setFolderPath(e.target.value)}
/> />
</div> </div>
<div className="flex justify-end gap-2"> <div className="flex justify-end gap-2">
@ -603,7 +594,7 @@ export function KnowledgeDropdown({
type="text" type="text"
placeholder="s3://bucket/path" placeholder="s3://bucket/path"
value={bucketUrl} value={bucketUrl}
onChange={e => setBucketUrl(e.target.value)} onChange={(e) => setBucketUrl(e.target.value)}
/> />
</div> </div>
<div className="flex justify-end gap-2"> <div className="flex justify-end gap-2">

263
frontend/src/contexts/task-context.tsx
View file

@ -7,33 +7,18 @@ import {
useCallback, useCallback,
useContext, useContext,
useEffect, useEffect,
useRef,
useState, useState,
} from "react"; } from "react";
import { toast } from "sonner"; import { toast } from "sonner";
import { useCancelTaskMutation } from "@/app/api/mutations/useCancelTaskMutation";
import {
type Task,
useGetTasksQuery,
} from "@/app/api/queries/useGetTasksQuery";
import { useAuth } from "@/contexts/auth-context"; import { useAuth } from "@/contexts/auth-context";
export interface Task { // Task interface is now imported from useGetTasksQuery
task_id: string;
status:
| "pending"
| "running"
| "processing"
| "completed"
| "failed"
| "error";
total_files?: number;
processed_files?: number;
successful_files?: number;
failed_files?: number;
running_files?: number;
pending_files?: number;
created_at: string;
updated_at: string;
duration_seconds?: number;
result?: Record<string, unknown>;
error?: string;
files?: Record<string, Record<string, unknown>>;
}
export interface TaskFile { export interface TaskFile {
filename: string; filename: string;
@ -58,20 +43,45 @@ interface TaskContextType {
isFetching: boolean; isFetching: boolean;
isMenuOpen: boolean; isMenuOpen: boolean;
toggleMenu: () => void; toggleMenu: () => void;
// React Query states
isLoading: boolean;
error: Error | null;
} }
const TaskContext = createContext<TaskContextType | undefined>(undefined); const TaskContext = createContext<TaskContextType | undefined>(undefined);
export function TaskProvider({ children }: { children: React.ReactNode }) { export function TaskProvider({ children }: { children: React.ReactNode }) {
const [tasks, setTasks] = useState<Task[]>([]);
const [files, setFiles] = useState<TaskFile[]>([]); const [files, setFiles] = useState<TaskFile[]>([]);
const [isPolling, setIsPolling] = useState(false);
const [isFetching, setIsFetching] = useState(false);
const [isMenuOpen, setIsMenuOpen] = useState(false); const [isMenuOpen, setIsMenuOpen] = useState(false);
const previousTasksRef = useRef<Task[]>([]);
const { isAuthenticated, isNoAuthMode } = useAuth(); const { isAuthenticated, isNoAuthMode } = useAuth();
const queryClient = useQueryClient(); const queryClient = useQueryClient();
// Use React Query hooks
const {
data: tasks = [],
isLoading,
error,
refetch: refetchTasks,
isFetching,
} = useGetTasksQuery({
enabled: isAuthenticated || isNoAuthMode,
});
const cancelTaskMutation = useCancelTaskMutation({
onSuccess: () => {
toast.success("Task cancelled", {
description: "Task has been cancelled successfully",
});
},
onError: (error) => {
toast.error("Failed to cancel task", {
description: error.message,
});
},
});
const refetchSearch = useCallback(() => { const refetchSearch = useCallback(() => {
queryClient.invalidateQueries({ queryClient.invalidateQueries({
queryKey: ["search"], queryKey: ["search"],
@ -99,28 +109,28 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
[], [],
); );
const fetchTasks = useCallback(async () => { // Handle task status changes and file updates
if (!isAuthenticated && !isNoAuthMode) return; useEffect(() => {
if (tasks.length === 0) {
// Store current tasks as previous for next comparison
previousTasksRef.current = tasks;
return;
}
console.log(tasks, previousTasksRef.current);
setIsFetching(true); // Check for task status changes by comparing with previous tasks
try { tasks.forEach((currentTask) => {
const response = await fetch("/api/tasks"); const previousTask = previousTasksRef.current.find(
if (response.ok) { (prev) => prev.task_id === currentTask.task_id,
const data = await response.json();
const newTasks = data.tasks || [];
// Update tasks and check for status changes in the same state update
setTasks((prevTasks) => {
// Check for newly completed tasks to show toasts
if (prevTasks.length > 0) {
newTasks.forEach((newTask: Task) => {
const oldTask = prevTasks.find(
(t) => t.task_id === newTask.task_id,
); );
// Update or add files from task.files if available // Only show toasts if we have previous data and status has changed
if (newTask.files && typeof newTask.files === "object") { if (((previousTask && previousTask.status !== currentTask.status) || (!previousTask && previousTasksRef.current.length !== 0))) {
const taskFileEntries = Object.entries(newTask.files); console.log("task status changed", currentTask.status);
// Process files from failed task and add them to files list
if (currentTask.files && typeof currentTask.files === "object") {
console.log("processing files", currentTask.files);
const taskFileEntries = Object.entries(currentTask.files);
const now = new Date().toISOString(); const now = new Date().toISOString();
taskFileEntries.forEach(([filePath, fileInfo]) => { taskFileEntries.forEach(([filePath, fileInfo]) => {
@ -149,7 +159,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
const existingFileIndex = prevFiles.findIndex( const existingFileIndex = prevFiles.findIndex(
(f) => (f) =>
f.source_url === filePath && f.source_url === filePath &&
f.task_id === newTask.task_id, f.task_id === currentTask.task_id,
); );
// Detect connector type based on file path or other indicators // Detect connector type based on file path or other indicators
@ -166,7 +176,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
size: 0, // We don't have this info from the task size: 0, // We don't have this info from the task
connector_type: connectorType, connector_type: connectorType,
status: mappedStatus, status: mappedStatus,
task_id: newTask.task_id, task_id: currentTask.task_id,
created_at: created_at:
typeof fileInfo.created_at === "string" typeof fileInfo.created_at === "string"
? fileInfo.created_at ? fileInfo.created_at
@ -190,161 +200,80 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
} }
}); });
} }
if ( if (
oldTask && previousTask && previousTask.status !== "completed" &&
oldTask.status !== "completed" && currentTask.status === "completed"
newTask.status === "completed"
) { ) {
// Task just completed - show success toast // Task just completed - show success toast
toast.success("Task completed successfully", { toast.success("Task completed successfully", {
description: `Task ${newTask.task_id} has finished processing.`, description: `Task ${currentTask.task_id} has finished processing.`,
action: { action: {
label: "View", label: "View",
onClick: () => console.log("View task", newTask.task_id), onClick: () => console.log("View task", currentTask.task_id),
}, },
}); });
refetchSearch(); refetchSearch();
// Dispatch knowledge updated event for all knowledge-related pages
console.log(
"Task completed successfully, dispatching knowledgeUpdated event",
);
window.dispatchEvent(new CustomEvent("knowledgeUpdated"));
// Remove files for this completed task from the files list // Remove files for this completed task from the files list
setFiles((prevFiles) => // setFiles((prevFiles) =>
prevFiles.filter((file) => file.task_id !== newTask.task_id), // prevFiles.filter((file) => file.task_id !== currentTask.task_id),
); // );
} else if ( } else if (
oldTask && previousTask && previousTask.status !== "failed" &&
oldTask.status !== "failed" && previousTask.status !== "error" &&
oldTask.status !== "error" && (currentTask.status === "failed" || currentTask.status === "error")
(newTask.status === "failed" || newTask.status === "error")
) { ) {
// Task just failed - show error toast // Task just failed - show error toast
toast.error("Task failed", { toast.error("Task failed", {
description: `Task ${newTask.task_id} failed: ${ description: `Task ${currentTask.task_id} failed: ${
newTask.error || "Unknown error" currentTask.error || "Unknown error"
}`, }`,
}); });
}
// Files will be updated to failed status by the file parsing logic above
} }
}); });
}
return newTasks; // Store current tasks as previous for next comparison
}); previousTasksRef.current = tasks;
} }, [tasks, refetchSearch]);
} catch (error) {
console.error("Failed to fetch tasks:", error);
} finally {
setIsFetching(false);
}
}, [isAuthenticated, isNoAuthMode, refetchSearch]); // Removed 'tasks' from dependencies to prevent infinite loop!
const addTask = useCallback((taskId: string) => { const addTask = useCallback(
// Immediately start aggressive polling for the new task (_taskId: string) => {
let pollAttempts = 0; // React Query will automatically handle polling when tasks are active
const maxPollAttempts = 30; // Poll for up to 30 seconds // Just trigger a refetch to get the latest data
refetchTasks();
const aggressivePoll = async () => { },
try { [refetchTasks],
const response = await fetch("/api/tasks");
if (response.ok) {
const data = await response.json();
const newTasks = data.tasks || [];
const foundTask = newTasks.find(
(task: Task) => task.task_id === taskId,
); );
if (foundTask) {
// Task found! Update the tasks state
setTasks((prevTasks) => {
// Check if task is already in the list
const exists = prevTasks.some((t) => t.task_id === taskId);
if (!exists) {
return [...prevTasks, foundTask];
}
// Update existing task
return prevTasks.map((t) =>
t.task_id === taskId ? foundTask : t,
);
});
return; // Stop polling, we found it
}
}
} catch (error) {
console.error("Aggressive polling failed:", error);
}
pollAttempts++;
if (pollAttempts < maxPollAttempts) {
// Continue polling every 1 second for new tasks
setTimeout(aggressivePoll, 1000);
}
};
// Start aggressive polling after a short delay to allow backend to process
setTimeout(aggressivePoll, 500);
}, []);
const refreshTasks = useCallback(async () => { const refreshTasks = useCallback(async () => {
await fetchTasks(); await refetchTasks();
}, [fetchTasks]); }, [refetchTasks]);
const removeTask = useCallback((taskId: string) => { const removeTask = useCallback((_taskId: string) => {
setTasks((prev) => prev.filter((task) => task.task_id !== taskId)); // This is now handled by React Query automatically
// Tasks will be removed from the list when they're no longer returned by the API
}, []); }, []);
const cancelTask = useCallback( const cancelTask = useCallback(
async (taskId: string) => { async (taskId: string) => {
try { cancelTaskMutation.mutate({ taskId });
const response = await fetch(`/api/tasks/${taskId}/cancel`, {
method: "POST",
});
if (response.ok) {
// Immediately refresh tasks to show the updated status
await fetchTasks();
toast.success("Task cancelled", {
description: `Task ${taskId.substring(0, 8)}... has been cancelled`,
});
} else {
const errorData = await response.json().catch(() => ({}));
throw new Error(errorData.error || "Failed to cancel task");
}
} catch (error) {
console.error("Failed to cancel task:", error);
toast.error("Failed to cancel task", {
description: error instanceof Error ? error.message : "Unknown error",
});
}
}, },
[fetchTasks], [cancelTaskMutation],
); );
const toggleMenu = useCallback(() => { const toggleMenu = useCallback(() => {
setIsMenuOpen((prev) => !prev); setIsMenuOpen((prev) => !prev);
}, []); }, []);
// Periodic polling for task updates // Determine if we're polling based on React Query's refetch interval
useEffect(() => { const isPolling =
if (!isAuthenticated && !isNoAuthMode) return; isFetching &&
tasks.some(
setIsPolling(true); (task) =>
task.status === "pending" ||
// Initial fetch task.status === "running" ||
fetchTasks(); task.status === "processing",
);
// Set up polling interval - every 3 seconds (more responsive for active tasks)
const interval = setInterval(fetchTasks, 3000);
return () => {
clearInterval(interval);
setIsPolling(false);
};
}, [isAuthenticated, isNoAuthMode, fetchTasks]);
const value: TaskContextType = { const value: TaskContextType = {
tasks, tasks,
@ -358,6 +287,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
isFetching, isFetching,
isMenuOpen, isMenuOpen,
toggleMenu, toggleMenu,
isLoading,
error,
}; };
return <TaskContext.Provider value={value}>{children}</TaskContext.Provider>; return <TaskContext.Provider value={value}>{children}</TaskContext.Provider>;