gmakstutis/openrag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/fix/watsonx_fixes' into all-merges

Browse source
This commit is contained in:
Lucas Oliveira 2025-12-05 17:50:41 -03:00
commit 8a456bc605
2 changed files with 366 additions and 233 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
frontend/app/chat/page.tsx
View file

@ -73,8 +73,6 @@ function ChatPage() {
const lastLoadedConversationRef = useRef<string | null>(null); const lastLoadedConversationRef = useRef<string | null>(null);
const { addTask } = useTask(); const { addTask } = useTask();
console.log(endpoint, refreshTrigger);
// Check if chat history is loading // Check if chat history is loading
const { isLoading: isConversationsLoading } = useGetConversationsQuery( const { isLoading: isConversationsLoading } = useGetConversationsQuery(
endpoint, endpoint,

597
frontend/app/onboarding/_components/onboarding-upload.tsx
View file

@ -1,3 +1,4 @@
import { X } from "lucide-react";
import { AnimatePresence, motion } from "motion/react"; import { AnimatePresence, motion } from "motion/react";
import { type ChangeEvent, useEffect, useRef, useState } from "react"; import { type ChangeEvent, useEffect, useRef, useState } from "react";
import { toast } from "sonner"; import { toast } from "sonner";
@ -7,271 +8,405 @@ import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery";
import { AnimatedProviderSteps } from "@/app/onboarding/_components/animated-provider-steps"; import { AnimatedProviderSteps } from "@/app/onboarding/_components/animated-provider-steps";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { import {
ONBOARDING_UPLOAD_STEPS_KEY, ONBOARDING_UPLOAD_STEPS_KEY,
ONBOARDING_USER_DOC_FILTER_ID_KEY, ONBOARDING_USER_DOC_FILTER_ID_KEY,
} from "@/lib/constants"; } from "@/lib/constants";
import { uploadFile } from "@/lib/upload-utils"; import { uploadFile } from "@/lib/upload-utils";
interface OnboardingUploadProps { interface OnboardingUploadProps {
onComplete: () => void; onComplete: () => void;
} }
const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => { const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
const fileInputRef = useRef<HTMLInputElement>(null); const fileInputRef = useRef<HTMLInputElement>(null);
const [isUploading, setIsUploading] = useState(false); const [isUploading, setIsUploading] = useState(false);
const [currentStep, setCurrentStep] = useState<number | null>(null); const [currentStep, setCurrentStep] = useState<number | null>(null);
const [uploadedFilename, setUploadedFilename] = useState<string | null>(null); const [uploadedFilename, setUploadedFilename] = useState<string | null>(null);
const [uploadedTaskId, setUploadedTaskId] = useState<string | null>(null); const [uploadedTaskId, setUploadedTaskId] = useState<string | null>(null);
const [shouldCreateFilter, setShouldCreateFilter] = useState(false); const [shouldCreateFilter, setShouldCreateFilter] = useState(false);
const [isCreatingFilter, setIsCreatingFilter] = useState(false); const [isCreatingFilter, setIsCreatingFilter] = useState(false);
const [error, setError] = useState<string | null>(null);
const createFilterMutation = useCreateFilter(); // Track which tasks we've already handled to prevent infinite loops
const handledFailedTasksRef = useRef<Set<string>>(new Set());
const STEP_LIST = [ const createFilterMutation = useCreateFilter();
"Uploading your document",
"Generating embeddings",
"Ingesting document",
"Processing your document",
];
// Query tasks to track completion const STEP_LIST = [
const { data: tasks } = useGetTasksQuery({ "Uploading your document",
enabled: currentStep !== null, // Only poll when upload has started "Generating embeddings",
refetchInterval: currentStep !== null ? 1000 : false, // Poll every 1 second during upload "Ingesting document",
}); "Processing your document",
];
const { refetch: refetchNudges } = useGetNudgesQuery(null); // Query tasks to track completion
const { data: tasks } = useGetTasksQuery({
enabled: currentStep !== null, // Only poll when upload has started
refetchInterval: currentStep !== null ? 1000 : false, // Poll every 1 second during upload
});
// Monitor tasks and call onComplete when file processing is done // Monitor tasks and call onComplete when file processing is done
useEffect(() => { useEffect(() => {
if (currentStep === null || !tasks || !uploadedTaskId) { if (currentStep === null || !tasks || !uploadedTaskId) {
return; return;
} }
// Find the task by task ID from the upload response // Find the task by task ID from the upload response
const matchingTask = tasks.find((task) => task.task_id === uploadedTaskId); const matchingTask = tasks.find((task) => task.task_id === uploadedTaskId);
// If no matching task found, wait for it to appear // If no matching task found, wait for it to appear
if (!matchingTask) { if (!matchingTask) {
return; return;
} }
// Check if the matching task is still active (pending, running, or processing) // Skip if this task was already handled as a failed task (from a previous failed upload)
const isTaskActive = // This prevents processing old failed tasks when a new upload starts
matchingTask.status === "pending" || if (handledFailedTasksRef.current.has(matchingTask.task_id)) {
matchingTask.status === "running" || // Check if it's a failed task that we've already handled
matchingTask.status === "processing"; const hasFailedFile =
matchingTask.files &&
Object.values(matchingTask.files).some(
(file) => file.status === "failed" || file.status === "error",
);
if (hasFailedFile) {
// This is an old failed task that we've already handled, ignore it
console.log(
"Skipping already-handled failed task:",
matchingTask.task_id,
);
return;
}
// If it's not a failed task, remove it from handled list (it might have succeeded on retry)
handledFailedTasksRef.current.delete(matchingTask.task_id);
}
// If task is completed or has processed files, complete the onboarding step // Check if any file failed in the matching task
if (!isTaskActive || (matchingTask.processed_files ?? 0) > 0) { const hasFailedFile = (() => {
// Set to final step to show "Done" // Must have files object
setCurrentStep(STEP_LIST.length); if (!matchingTask.files || typeof matchingTask.files !== "object") {
return false;
}
// Create knowledge filter for uploaded document if requested const fileEntries = Object.values(matchingTask.files);
// Guard against race condition: only create if not already creating
if (shouldCreateFilter && uploadedFilename && !isCreatingFilter) {
// Reset flags immediately (synchronously) to prevent duplicate creation
setShouldCreateFilter(false);
const filename = uploadedFilename;
setUploadedFilename(null);
setIsCreatingFilter(true);
// Get display name from filename (remove extension for cleaner name) // Must have at least one file
const displayName = filename.includes(".") if (fileEntries.length === 0) {
? filename.substring(0, filename.lastIndexOf(".")) return false;
: filename; }
const queryData = JSON.stringify({ // Check if any file has failed status
query: "", return fileEntries.some(
filters: { (file) => file.status === "failed" || file.status === "error",
data_sources: [filename], );
document_types: ["*"], })();
owners: ["*"],
connector_types: ["*"],
},
limit: 10,
scoreThreshold: 0,
color: "green",
icon: "file",
});
// Wait for filter creation to complete before proceeding // If any file failed, show error and jump back one step (like onboarding-card.tsx)
createFilterMutation // Only handle if we haven't already handled this task
.mutateAsync({ if (
name: displayName, hasFailedFile &&
description: `Filter for ${filename}`, !isCreatingFilter &&
queryData: queryData, !handledFailedTasksRef.current.has(matchingTask.task_id)
}) ) {
.then((result) => { console.error("File failed in task, jumping back one step", matchingTask);
if (result.filter?.id && typeof window !== "undefined") {
localStorage.setItem(
ONBOARDING_USER_DOC_FILTER_ID_KEY,
result.filter.id,
);
console.log(
"Created knowledge filter for uploaded document",
result.filter.id,
);
}
})
.catch((error) => {
console.error("Failed to create knowledge filter:", error);
})
.finally(() => {
setIsCreatingFilter(false);
// Refetch nudges to get new ones
refetchNudges();
// Wait a bit before completing (after filter is created) // Mark this task as handled to prevent infinite loops
setTimeout(() => { handledFailedTasksRef.current.add(matchingTask.task_id);
onComplete();
}, 1000);
});
} else {
// No filter to create, just complete
// Refetch nudges to get new ones
refetchNudges();
// Wait a bit before completing // Extract error messages from failed files
setTimeout(() => { const errorMessages: string[] = [];
onComplete(); if (matchingTask.files) {
}, 1000); Object.values(matchingTask.files).forEach((file) => {
} if (
} (file.status === "failed" || file.status === "error") &&
}, [ file.error
tasks, ) {
currentStep, errorMessages.push(file.error);
onComplete, }
refetchNudges, });
shouldCreateFilter, }
uploadedFilename,
uploadedTaskId,
createFilterMutation,
isCreatingFilter,
]);
const resetFileInput = () => { // Also check task-level error
if (fileInputRef.current) { if (matchingTask.error) {
fileInputRef.current.value = ""; errorMessages.push(matchingTask.error);
} }
};
const handleUploadClick = () => { // Use the first error message, or a generic message if no errors found
fileInputRef.current?.click(); const errorMessage =
}; errorMessages.length > 0
? errorMessages[0]
: "Document failed to ingest. Please try again with a different file.";
const performUpload = async (file: File) => { // Set error message and jump back one step
setIsUploading(true); setError(errorMessage);
try { setCurrentStep(STEP_LIST.length);
setCurrentStep(0);
const result = await uploadFile(file, true, true); // Pass createFilter=true
console.log("Document upload task started successfully");
// Store task ID to track the specific upload task // Clear filter creation flags since ingestion failed
if (result.taskId) { setShouldCreateFilter(false);
setUploadedTaskId(result.taskId); setUploadedFilename(null);
}
// Store filename and createFilter flag in state to create filter after ingestion succeeds // Jump back one step after 1 second (go back to upload step)
if (result.createFilter && result.filename) { setTimeout(() => {
setUploadedFilename(result.filename); setCurrentStep(null);
setShouldCreateFilter(true); }, 1000);
} return;
}
// Move to processing step - task monitoring will handle completion // Check if the matching task is still active (pending, running, or processing)
setTimeout(() => { const isTaskActive =
setCurrentStep(1); matchingTask.status === "pending" ||
}, 1500); matchingTask.status === "running" ||
} catch (error) { matchingTask.status === "processing";
const errorMessage = error instanceof Error ? error.message : "Upload failed";
console.error("Upload failed", errorMessage);
// Dispatch event that chat context can listen to // If task is completed successfully (no failures) and has processed files, complete the onboarding step
// This avoids circular dependency issues if (
if (typeof window !== "undefined") { (!isTaskActive || (matchingTask.processed_files ?? 0) > 0) &&
window.dispatchEvent( !hasFailedFile
new CustomEvent("ingestionFailed", { ) {
detail: { source: "onboarding" }, // Set to final step to show "Done"
}), setCurrentStep(STEP_LIST.length);
);
}
// Show error toast notification // Create knowledge filter for uploaded document if requested
toast.error("Document upload failed", { // Guard against race condition: only create if not already creating
description: errorMessage, if (shouldCreateFilter && uploadedFilename && !isCreatingFilter) {
duration: 5000, // Reset flags immediately (synchronously) to prevent duplicate creation
}); setShouldCreateFilter(false);
const filename = uploadedFilename;
setUploadedFilename(null);
setIsCreatingFilter(true);
// Reset on error // Get display name from filename (remove extension for cleaner name)
setCurrentStep(null); const displayName = filename.includes(".")
setUploadedTaskId(null); ? filename.substring(0, filename.lastIndexOf("."))
} finally { : filename;
setIsUploading(false);
}
};
const handleFileChange = async (event: ChangeEvent<HTMLInputElement>) => { const queryData = JSON.stringify({
const selectedFile = event.target.files?.[0]; query: "",
if (!selectedFile) { filters: {
resetFileInput(); data_sources: [filename],
return; document_types: ["*"],
} owners: ["*"],
connector_types: ["*"],
},
limit: 10,
scoreThreshold: 0,
color: "green",
icon: "file",
});
try { // Wait for filter creation to complete before proceeding
await performUpload(selectedFile); createFilterMutation
} catch (error) { .mutateAsync({
console.error( name: displayName,
"Unable to prepare file for upload", description: `Filter for ${filename}`,
(error as Error).message, queryData: queryData,
); })
} finally { .then((result) => {
resetFileInput(); if (result.filter?.id && typeof window !== "undefined") {
} localStorage.setItem(
}; ONBOARDING_USER_DOC_FILTER_ID_KEY,
result.filter.id,
);
console.log(
"Created knowledge filter for uploaded document",
result.filter.id,
);
}
})
.catch((error) => {
console.error("Failed to create knowledge filter:", error);
})
.finally(() => {
setIsCreatingFilter(false);
// Refetch nudges to get new ones
refetchNudges();
return ( // Wait a bit before completing (after filter is created)
<AnimatePresence mode="wait"> setTimeout(() => {
{currentStep === null ? ( onComplete();
<motion.div }, 1000);
key="user-ingest" });
initial={{ opacity: 1, y: 0 }} } else {
exit={{ opacity: 0, y: -24 }} // No filter to create, just complete
transition={{ duration: 0.4, ease: "easeInOut" }} // Refetch nudges to get new ones
> refetchNudges();
<Button
size="sm" // Wait a bit before completing
variant="outline" setTimeout(() => {
onClick={handleUploadClick} onComplete();
disabled={isUploading} }, 1000);
> }
<div>{isUploading ? "Uploading..." : "Add a document"}</div> }
</Button> }, [
<input tasks,
ref={fileInputRef} currentStep,
type="file" onComplete,
onChange={handleFileChange} refetchNudges,
className="hidden" shouldCreateFilter,
accept=".pdf,.doc,.docx,.txt,.md,.rtf,.odt" uploadedFilename,
/> uploadedTaskId,
</motion.div> createFilterMutation,
) : ( isCreatingFilter,
<motion.div ]);
key="ingest-steps"
initial={{ opacity: 0, y: 24 }} const resetFileInput = () => {
animate={{ opacity: 1, y: 0 }} if (fileInputRef.current) {
transition={{ duration: 0.4, ease: "easeInOut" }} fileInputRef.current.value = "";
> }
<AnimatedProviderSteps };
currentStep={currentStep}
setCurrentStep={setCurrentStep} const handleUploadClick = () => {
isCompleted={false} // Clear any previous error when user clicks to upload again
steps={STEP_LIST} setError(null);
storageKey={ONBOARDING_UPLOAD_STEPS_KEY} fileInputRef.current?.click();
/> };
</motion.div>
)} const performUpload = async (file: File) => {
</AnimatePresence> setIsUploading(true);
); // Clear any previous error when starting a new upload
setError(null);
// Clear handled tasks ref to allow retry
handledFailedTasksRef.current.clear();
// Reset task ID to prevent matching old failed tasks
setUploadedTaskId(null);
// Clear filter creation flags
setShouldCreateFilter(false);
setUploadedFilename(null);
try {
setCurrentStep(0);
const result = await uploadFile(file, true, true); // Pass createFilter=true
console.log("Document upload task started successfully");
// Store task ID to track the specific upload task
if (result.taskId) {
setUploadedTaskId(result.taskId);
}
// Store filename and createFilter flag in state to create filter after ingestion succeeds
if (result.createFilter && result.filename) {
setUploadedFilename(result.filename);
setShouldCreateFilter(true);
}
// Move to processing step - task monitoring will handle completion
setTimeout(() => {
setCurrentStep(1);
}, 1500);
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Upload failed";
console.error("Upload failed", errorMessage);
// Dispatch event that chat context can listen to
// This avoids circular dependency issues
if (typeof window !== "undefined") {
window.dispatchEvent(
new CustomEvent("ingestionFailed", {
detail: { source: "onboarding" },
}),
);
}
// Show error toast notification
toast.error("Document upload failed", {
description: errorMessage,
duration: 5000,
});
// Reset on error
setCurrentStep(null);
setUploadedTaskId(null);
setError(errorMessage);
setShouldCreateFilter(false);
setUploadedFilename(null);
} finally {
setIsUploading(false);
}
};
const handleFileChange = async (event: ChangeEvent<HTMLInputElement>) => {
const selectedFile = event.target.files?.[0];
if (!selectedFile) {
resetFileInput();
return;
}
try {
await performUpload(selectedFile);
} catch (error) {
console.error(
"Unable to prepare file for upload",
(error as Error).message,
);
} finally {
resetFileInput();
}
};
return (
<AnimatePresence mode="wait">
{currentStep === null ? (
<motion.div
key="user-ingest"
initial={{ opacity: 1, y: 0 }}
exit={{ opacity: 0, y: -24 }}
transition={{ duration: 0.4, ease: "easeInOut" }}
>
<div className="w-full flex flex-col gap-4">
<AnimatePresence mode="wait">
{error && (
<motion.div
key="error"
initial={{ opacity: 1, y: 0, height: "auto" }}
exit={{ opacity: 0, y: -10, height: 0 }}
>
<div className="pb-2 flex items-center gap-4">
<X className="w-4 h-4 text-destructive shrink-0" />
<span className="text-sm text-muted-foreground">
{error}
</span>
</div>
</motion.div>
)}
</AnimatePresence>
<div>
<Button
size="sm"
variant="outline"
onClick={handleUploadClick}
disabled={isUploading}
>
<div>{isUploading ? "Uploading..." : "Add a document"}</div>
</Button>
</div>
<input
ref={fileInputRef}
type="file"
onChange={handleFileChange}
className="hidden"
accept=".pdf,.doc,.docx,.txt,.md,.rtf,.odt"
/>
</div>
</motion.div>
) : (
<motion.div
key="ingest-steps"
initial={{ opacity: 0, y: 24 }}
animate={{ opacity: 1, y: 0 }}
transition={{ duration: 0.4, ease: "easeInOut" }}
>
<AnimatedProviderSteps
currentStep={currentStep}
setCurrentStep={setCurrentStep}
isCompleted={false}
steps={STEP_LIST}
storageKey={ONBOARDING_UPLOAD_STEPS_KEY}
hasError={!!error}
/>
</motion.div>
)}
</AnimatePresence>
);
}; };
export default OnboardingUpload; export default OnboardingUpload;