From 7dfe90c8a6cfb9236dc66bdd908af1345ef21050 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 15:49:21 -0300 Subject: [PATCH 01/55] fixed ibm logo --- frontend/components/logo/ibm-logo.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/components/logo/ibm-logo.tsx b/frontend/components/logo/ibm-logo.tsx index 158ffa3b..e37adec1 100644 --- a/frontend/components/logo/ibm-logo.tsx +++ b/frontend/components/logo/ibm-logo.tsx @@ -9,7 +9,7 @@ export default function IBMLogo(props: React.SVGProps) { {...props} > IBM watsonx.ai Logo - + Date: Thu, 2 Oct 2025 16:50:34 -0300 Subject: [PATCH 02/55] implement tasks fetching and cancelling on useQuery --- .../api/mutations/useCancelTaskMutation.ts | 47 +++++++++++ .../app/api/queries/useGetTaskStatusQuery.ts | 80 +++++++++++++++++++ .../src/app/api/queries/useGetTasksQuery.ts | 79 ++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 frontend/src/app/api/mutations/useCancelTaskMutation.ts create mode 100644 frontend/src/app/api/queries/useGetTaskStatusQuery.ts create mode 100644 frontend/src/app/api/queries/useGetTasksQuery.ts diff --git a/frontend/src/app/api/mutations/useCancelTaskMutation.ts b/frontend/src/app/api/mutations/useCancelTaskMutation.ts new file mode 100644 index 00000000..1bf2faed --- /dev/null +++ b/frontend/src/app/api/mutations/useCancelTaskMutation.ts @@ -0,0 +1,47 @@ +import { + type UseMutationOptions, + useMutation, + useQueryClient, +} from "@tanstack/react-query"; + +export interface CancelTaskRequest { + taskId: string; +} + +export interface CancelTaskResponse { + status: string; + task_id: string; +} + +export const useCancelTaskMutation = ( + options?: Omit< + UseMutationOptions, + "mutationFn" + > +) => { + const queryClient = useQueryClient(); + + async function cancelTask( + variables: CancelTaskRequest, + ): Promise { + const response = await fetch(`/api/tasks/${variables.taskId}/cancel`, { + method: "POST", + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error || "Failed to cancel task"); + } + + return response.json(); + } + + return useMutation({ + mutationFn: cancelTask, + onSuccess: () => { + // Invalidate tasks query to refresh the list + queryClient.invalidateQueries({ queryKey: ["tasks"] }); + }, + ...options, + }); +}; diff --git a/frontend/src/app/api/queries/useGetTaskStatusQuery.ts b/frontend/src/app/api/queries/useGetTaskStatusQuery.ts new file mode 100644 index 00000000..17cd2d16 --- /dev/null +++ b/frontend/src/app/api/queries/useGetTaskStatusQuery.ts @@ -0,0 +1,80 @@ +import { + type UseQueryOptions, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; + +export interface TaskStatus { + task_id: string; + status: + | "pending" + | "running" + | "processing" + | "completed" + | "failed" + | "error"; + total_files?: number; + processed_files?: number; + successful_files?: number; + failed_files?: number; + running_files?: number; + pending_files?: number; + created_at: string; + updated_at: string; + duration_seconds?: number; + result?: Record; + error?: string; + files?: Record>; +} + +export const useGetTaskStatusQuery = ( + taskId: string, + options?: Omit, "queryKey" | "queryFn"> +) => { + const queryClient = useQueryClient(); + + async function getTaskStatus(): Promise { + if (!taskId) { + return null; + } + + const response = await fetch(`/api/tasks/${taskId}`); + + if (!response.ok) { + if (response.status === 404) { + return null; // Task not found + } + throw new Error("Failed to fetch task status"); + } + + return response.json(); + } + + const queryResult = useQuery( + { + queryKey: ["task-status", taskId], + queryFn: getTaskStatus, + refetchInterval: (data) => { + // Only poll if the task is still active + if (!data) { + return false; // Stop polling if no data + } + + const isActive = + data.status === "pending" || + data.status === "running" || + data.status === "processing"; + + return isActive ? 3000 : false; // Poll every 3 seconds if active + }, + refetchIntervalInBackground: true, + staleTime: 0, // Always consider data stale to ensure fresh updates + gcTime: 5 * 60 * 1000, // Keep in cache for 5 minutes + enabled: !!taskId, // Only run if taskId is provided + ...options, + }, + queryClient, + ); + + return queryResult; +}; diff --git a/frontend/src/app/api/queries/useGetTasksQuery.ts b/frontend/src/app/api/queries/useGetTasksQuery.ts new file mode 100644 index 00000000..1ea59d26 --- /dev/null +++ b/frontend/src/app/api/queries/useGetTasksQuery.ts @@ -0,0 +1,79 @@ +import { + type UseQueryOptions, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; + +export interface Task { + task_id: string; + status: + | "pending" + | "running" + | "processing" + | "completed" + | "failed" + | "error"; + total_files?: number; + processed_files?: number; + successful_files?: number; + failed_files?: number; + running_files?: number; + pending_files?: number; + created_at: string; + updated_at: string; + duration_seconds?: number; + result?: Record; + error?: string; + files?: Record>; +} + +export interface TasksResponse { + tasks: Task[]; +} + +export const useGetTasksQuery = ( + options?: Omit, "queryKey" | "queryFn"> +) => { + const queryClient = useQueryClient(); + + async function getTasks(): Promise { + const response = await fetch("/api/tasks"); + + if (!response.ok) { + throw new Error("Failed to fetch tasks"); + } + + const data: TasksResponse = await response.json(); + return data.tasks || []; + } + + const queryResult = useQuery( + { + queryKey: ["tasks"], + queryFn: getTasks, + refetchInterval: (query) => { + // Only poll if there are tasks with pending or running status + const data = query.state.data; + if (!data || data.length === 0) { + return false; // Stop polling if no tasks + } + + const hasActiveTasks = data.some( + (task: Task) => + task.status === "pending" || + task.status === "running" || + task.status === "processing" + ); + + return hasActiveTasks ? 3000 : false; // Poll every 3 seconds if active tasks exist + }, + refetchIntervalInBackground: true, + staleTime: 0, // Always consider data stale to ensure fresh updates + gcTime: 5 * 60 * 1000, // Keep in cache for 5 minutes + ...options, + }, + queryClient, + ); + + return queryResult; +}; From aa61ba265c3a9744873c817c75779a15910fe9e7 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 16:50:47 -0300 Subject: [PATCH 03/55] made files and toast appear just once, use new queries --- frontend/components/knowledge-dropdown.tsx | 1117 ++++++++++---------- frontend/src/contexts/task-context.tsx | 427 ++++---- 2 files changed, 733 insertions(+), 811 deletions(-) diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index ee49fc3a..9b71ee81 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -1,24 +1,25 @@ "use client"; import { - ChevronDown, - Cloud, - FolderOpen, - Loader2, - PlugZap, - Plus, - Upload, + ChevronDown, + Cloud, + FolderOpen, + Loader2, + PlugZap, + Plus, + Upload, } from "lucide-react"; import { useRouter } from "next/navigation"; import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; +import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery"; import { Button } from "@/components/ui/button"; import { - Dialog, - DialogContent, - DialogDescription, - DialogHeader, - DialogTitle, + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; @@ -26,600 +27,590 @@ import { useTask } from "@/contexts/task-context"; import { cn } from "@/lib/utils"; interface KnowledgeDropdownProps { - active?: boolean; - variant?: "navigation" | "button"; + active?: boolean; + variant?: "navigation" | "button"; } export function KnowledgeDropdown({ - active, - variant = "navigation", + active, + variant = "navigation", }: KnowledgeDropdownProps) { - const { addTask } = useTask(); - const router = useRouter(); - const [isOpen, setIsOpen] = useState(false); - const [showFolderDialog, setShowFolderDialog] = useState(false); - const [showS3Dialog, setShowS3Dialog] = useState(false); - const [awsEnabled, setAwsEnabled] = useState(false); - const [folderPath, setFolderPath] = useState("/app/documents/"); - const [bucketUrl, setBucketUrl] = useState("s3://"); - const [folderLoading, setFolderLoading] = useState(false); - const [s3Loading, setS3Loading] = useState(false); - const [fileUploading, setFileUploading] = useState(false); - const [isNavigatingToCloud, setIsNavigatingToCloud] = useState(false); - const [cloudConnectors, setCloudConnectors] = useState<{ - [key: string]: { - name: string; - available: boolean; - connected: boolean; - hasToken: boolean; - }; - }>({}); - const fileInputRef = useRef(null); - const dropdownRef = useRef(null); + const { addTask } = useTask(); + const { refetch: refetchTasks } = useGetTasksQuery(); + const router = useRouter(); + const [isOpen, setIsOpen] = useState(false); + const [showFolderDialog, setShowFolderDialog] = useState(false); + const [showS3Dialog, setShowS3Dialog] = useState(false); + const [awsEnabled, setAwsEnabled] = useState(false); + const [folderPath, setFolderPath] = useState("/app/documents/"); + const [bucketUrl, setBucketUrl] = useState("s3://"); + const [folderLoading, setFolderLoading] = useState(false); + const [s3Loading, setS3Loading] = useState(false); + const [fileUploading, setFileUploading] = useState(false); + const [isNavigatingToCloud, setIsNavigatingToCloud] = useState(false); + const [cloudConnectors, setCloudConnectors] = useState<{ + [key: string]: { + name: string; + available: boolean; + connected: boolean; + hasToken: boolean; + }; + }>({}); + const fileInputRef = useRef(null); + const dropdownRef = useRef(null); - // Check AWS availability and cloud connectors on mount - useEffect(() => { - const checkAvailability = async () => { - try { - // Check AWS - const awsRes = await fetch("/api/upload_options"); - if (awsRes.ok) { - const awsData = await awsRes.json(); - setAwsEnabled(Boolean(awsData.aws)); - } + // Check AWS availability and cloud connectors on mount + useEffect(() => { + const checkAvailability = async () => { + try { + // Check AWS + const awsRes = await fetch("/api/upload_options"); + if (awsRes.ok) { + const awsData = await awsRes.json(); + setAwsEnabled(Boolean(awsData.aws)); + } - // Check cloud connectors - const connectorsRes = await fetch("/api/connectors"); - if (connectorsRes.ok) { - const connectorsResult = await connectorsRes.json(); - const cloudConnectorTypes = [ - "google_drive", - "onedrive", - "sharepoint", - ]; - const connectorInfo: { - [key: string]: { - name: string; - available: boolean; - connected: boolean; - hasToken: boolean; - }; - } = {}; + // Check cloud connectors + const connectorsRes = await fetch("/api/connectors"); + if (connectorsRes.ok) { + const connectorsResult = await connectorsRes.json(); + const cloudConnectorTypes = [ + "google_drive", + "onedrive", + "sharepoint", + ]; + const connectorInfo: { + [key: string]: { + name: string; + available: boolean; + connected: boolean; + hasToken: boolean; + }; + } = {}; - for (const type of cloudConnectorTypes) { - if (connectorsResult.connectors[type]) { - connectorInfo[type] = { - name: connectorsResult.connectors[type].name, - available: connectorsResult.connectors[type].available, - connected: false, - hasToken: false, - }; + for (const type of cloudConnectorTypes) { + if (connectorsResult.connectors[type]) { + connectorInfo[type] = { + name: connectorsResult.connectors[type].name, + available: connectorsResult.connectors[type].available, + connected: false, + hasToken: false, + }; - // Check connection status - try { - const statusRes = await fetch(`/api/connectors/${type}/status`); - if (statusRes.ok) { - const statusData = await statusRes.json(); - const connections = statusData.connections || []; - const activeConnection = connections.find( - (conn: { is_active: boolean; connection_id: string }) => - conn.is_active - ); - const isConnected = activeConnection !== undefined; + // Check connection status + try { + const statusRes = await fetch(`/api/connectors/${type}/status`); + if (statusRes.ok) { + const statusData = await statusRes.json(); + const connections = statusData.connections || []; + const activeConnection = connections.find( + (conn: { is_active: boolean; connection_id: string }) => + conn.is_active, + ); + const isConnected = activeConnection !== undefined; - if (isConnected && activeConnection) { - connectorInfo[type].connected = true; + if (isConnected && activeConnection) { + connectorInfo[type].connected = true; - // Check token availability - try { - const tokenRes = await fetch( - `/api/connectors/${type}/token?connection_id=${activeConnection.connection_id}` - ); - if (tokenRes.ok) { - const tokenData = await tokenRes.json(); - if (tokenData.access_token) { - connectorInfo[type].hasToken = true; - } - } - } catch { - // Token check failed - } - } - } - } catch { - // Status check failed - } - } - } + // Check token availability + try { + const tokenRes = await fetch( + `/api/connectors/${type}/token?connection_id=${activeConnection.connection_id}`, + ); + if (tokenRes.ok) { + const tokenData = await tokenRes.json(); + if (tokenData.access_token) { + connectorInfo[type].hasToken = true; + } + } + } catch { + // Token check failed + } + } + } + } catch { + // Status check failed + } + } + } - setCloudConnectors(connectorInfo); - } - } catch (err) { - console.error("Failed to check availability", err); - } - }; - checkAvailability(); - }, []); + setCloudConnectors(connectorInfo); + } + } catch (err) { + console.error("Failed to check availability", err); + } + }; + checkAvailability(); + }, []); - // Handle click outside to close dropdown - useEffect(() => { - const handleClickOutside = (event: MouseEvent) => { - if ( - dropdownRef.current && - !dropdownRef.current.contains(event.target as Node) - ) { - setIsOpen(false); - } - }; + // Handle click outside to close dropdown + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + dropdownRef.current && + !dropdownRef.current.contains(event.target as Node) + ) { + setIsOpen(false); + } + }; - if (isOpen) { - document.addEventListener("mousedown", handleClickOutside); - return () => - document.removeEventListener("mousedown", handleClickOutside); - } - }, [isOpen]); + if (isOpen) { + document.addEventListener("mousedown", handleClickOutside); + return () => + document.removeEventListener("mousedown", handleClickOutside); + } + }, [isOpen]); - const handleFileUpload = () => { - fileInputRef.current?.click(); - }; + const handleFileUpload = () => { + fileInputRef.current?.click(); + }; - const handleFileChange = async (e: React.ChangeEvent) => { - const files = e.target.files; - if (files && files.length > 0) { - // Close dropdown and disable button immediately after file selection - setIsOpen(false); - setFileUploading(true); + const handleFileChange = async (e: React.ChangeEvent) => { + const files = e.target.files; + if (files && files.length > 0) { + // Close dropdown and disable button immediately after file selection + setIsOpen(false); + setFileUploading(true); - // Trigger the same file upload event as the chat page - window.dispatchEvent( - new CustomEvent("fileUploadStart", { - detail: { filename: files[0].name }, - }) - ); + // Trigger the same file upload event as the chat page + window.dispatchEvent( + new CustomEvent("fileUploadStart", { + detail: { filename: files[0].name }, + }), + ); - try { - const formData = new FormData(); - formData.append("file", files[0]); + try { + const formData = new FormData(); + formData.append("file", files[0]); - // Use router upload and ingest endpoint (automatically routes based on configuration) - const uploadIngestRes = await fetch("/api/router/upload_ingest", { - method: "POST", - body: formData, - }); + // Use router upload and ingest endpoint (automatically routes based on configuration) + const uploadIngestRes = await fetch("/api/router/upload_ingest", { + method: "POST", + body: formData, + }); - const uploadIngestJson = await uploadIngestRes.json(); + const uploadIngestJson = await uploadIngestRes.json(); - if (!uploadIngestRes.ok) { - throw new Error( - uploadIngestJson?.error || "Upload and ingest failed" - ); - } + if (!uploadIngestRes.ok) { + throw new Error( + uploadIngestJson?.error || "Upload and ingest failed", + ); + } - // Extract results from the response - handle both unified and simple formats - const fileId = uploadIngestJson?.upload?.id || uploadIngestJson?.id; - const filePath = - uploadIngestJson?.upload?.path || - uploadIngestJson?.path || - "uploaded"; - const runJson = uploadIngestJson?.ingestion; - const deleteResult = uploadIngestJson?.deletion; + // Extract results from the response - handle both unified and simple formats + const fileId = uploadIngestJson?.upload?.id || uploadIngestJson?.id || uploadIngestJson?.task_id; + const filePath = + uploadIngestJson?.upload?.path || + uploadIngestJson?.path || + "uploaded"; + const runJson = uploadIngestJson?.ingestion; + const deleteResult = uploadIngestJson?.deletion; + console.log("c", uploadIngestJson ) + if (!fileId) { + throw new Error("Upload successful but no file id returned"); + } + // Check if ingestion actually succeeded + if ( + runJson && + runJson.status !== "COMPLETED" && + runJson.status !== "SUCCESS" + ) { + const errorMsg = runJson.error || "Ingestion pipeline failed"; + throw new Error( + `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.`, + ); + } + // Log deletion status if provided + if (deleteResult) { + if (deleteResult.status === "deleted") { + console.log( + "File successfully cleaned up from Langflow:", + deleteResult.file_id, + ); + } else if (deleteResult.status === "delete_failed") { + console.warn( + "Failed to cleanup file from Langflow:", + deleteResult.error, + ); + } + } + // Notify UI + window.dispatchEvent( + new CustomEvent("fileUploaded", { + detail: { + file: files[0], + result: { + file_id: fileId, + file_path: filePath, + run: runJson, + deletion: deleteResult, + unified: true, + }, + }, + }), + ); - if (!fileId) { - throw new Error("Upload successful but no file id returned"); - } + refetchTasks(); + } catch (error) { + window.dispatchEvent( + new CustomEvent("fileUploadError", { + detail: { + filename: files[0].name, + error: error instanceof Error ? error.message : "Upload failed", + }, + }), + ); + } finally { + window.dispatchEvent(new CustomEvent("fileUploadComplete")); + setFileUploading(false); + } + } - // Check if ingestion actually succeeded - if ( - runJson && - runJson.status !== "COMPLETED" && - runJson.status !== "SUCCESS" - ) { - const errorMsg = runJson.error || "Ingestion pipeline failed"; - throw new Error( - `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.` - ); - } + // Reset file input + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } + }; - // Log deletion status if provided - if (deleteResult) { - if (deleteResult.status === "deleted") { - console.log( - "File successfully cleaned up from Langflow:", - deleteResult.file_id - ); - } else if (deleteResult.status === "delete_failed") { - console.warn( - "Failed to cleanup file from Langflow:", - deleteResult.error - ); - } - } + const handleFolderUpload = async () => { + if (!folderPath.trim()) return; - // Notify UI - window.dispatchEvent( - new CustomEvent("fileUploaded", { - detail: { - file: files[0], - result: { - file_id: fileId, - file_path: filePath, - run: runJson, - deletion: deleteResult, - unified: true, - }, - }, - }) - ); + setFolderLoading(true); + setShowFolderDialog(false); - // Trigger search refresh after successful ingestion - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - } catch (error) { - window.dispatchEvent( - new CustomEvent("fileUploadError", { - detail: { - filename: files[0].name, - error: error instanceof Error ? error.message : "Upload failed", - }, - }) - ); - } finally { - window.dispatchEvent(new CustomEvent("fileUploadComplete")); - setFileUploading(false); - // Don't call refetchSearch() here - the knowledgeUpdated event will handle it - } - } + try { + const response = await fetch("/api/upload_path", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ path: folderPath }), + }); - // Reset file input - if (fileInputRef.current) { - fileInputRef.current.value = ""; - } - }; + const result = await response.json(); - const handleFolderUpload = async () => { - if (!folderPath.trim()) return; + if (response.status === 201) { + const taskId = result.task_id || result.id; - setFolderLoading(true); - setShowFolderDialog(false); + if (!taskId) { + throw new Error("No task ID received from server"); + } - try { - const response = await fetch("/api/upload_path", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ path: folderPath }), - }); + addTask(taskId); + setFolderPath(""); + // Refetch tasks to show the new task + refetchTasks(); + } else if (response.ok) { + setFolderPath(""); + // Refetch tasks even for direct uploads in case tasks were created + refetchTasks(); + } else { + console.error("Folder upload failed:", result.error); + if (response.status === 400) { + toast.error("Upload failed", { + description: result.error || "Bad request", + }); + } + } + } catch (error) { + console.error("Folder upload error:", error); + } finally { + setFolderLoading(false); + } + }; - const result = await response.json(); + const handleS3Upload = async () => { + if (!bucketUrl.trim()) return; - if (response.status === 201) { - const taskId = result.task_id || result.id; + setS3Loading(true); + setShowS3Dialog(false); - if (!taskId) { - throw new Error("No task ID received from server"); - } + try { + const response = await fetch("/api/upload_bucket", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ s3_url: bucketUrl }), + }); - addTask(taskId); - setFolderPath(""); - // Trigger search refresh after successful folder processing starts - console.log( - "Folder upload successful, dispatching knowledgeUpdated event" - ); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - } else if (response.ok) { - setFolderPath(""); - console.log( - "Folder upload successful (direct), dispatching knowledgeUpdated event" - ); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - } else { - console.error("Folder upload failed:", result.error); - if (response.status === 400) { - toast.error("Upload failed", { - description: result.error || "Bad request", - }); - } - } - } catch (error) { - console.error("Folder upload error:", error); - } finally { - setFolderLoading(false); - // Don't call refetchSearch() here - the knowledgeUpdated event will handle it - } - }; + const result = await response.json(); - const handleS3Upload = async () => { - if (!bucketUrl.trim()) return; + if (response.status === 201) { + const taskId = result.task_id || result.id; - setS3Loading(true); - setShowS3Dialog(false); + if (!taskId) { + throw new Error("No task ID received from server"); + } - try { - const response = await fetch("/api/upload_bucket", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ s3_url: bucketUrl }), - }); + addTask(taskId); + setBucketUrl("s3://"); + // Refetch tasks to show the new task + refetchTasks(); + } else { + console.error("S3 upload failed:", result.error); + if (response.status === 400) { + toast.error("Upload failed", { + description: result.error || "Bad request", + }); + } + } + } catch (error) { + console.error("S3 upload error:", error); + } finally { + setS3Loading(false); + } + }; - const result = await response.json(); + const cloudConnectorItems = Object.entries(cloudConnectors) + .filter(([, info]) => info.available) + .map(([type, info]) => ({ + label: info.name, + icon: PlugZap, + onClick: async () => { + setIsOpen(false); + if (info.connected && info.hasToken) { + setIsNavigatingToCloud(true); + try { + router.push(`/upload/${type}`); + // Keep loading state for a short time to show feedback + setTimeout(() => setIsNavigatingToCloud(false), 1000); + } catch { + setIsNavigatingToCloud(false); + } + } else { + router.push("/settings"); + } + }, + disabled: !info.connected || !info.hasToken, + tooltip: !info.connected + ? `Connect ${info.name} in Settings first` + : !info.hasToken + ? `Reconnect ${info.name} - access token required` + : undefined, + })); - if (response.status === 201) { - const taskId = result.task_id || result.id; + const menuItems = [ + { + label: "Add File", + icon: Upload, + onClick: handleFileUpload, + }, + { + label: "Process Folder", + icon: FolderOpen, + onClick: () => { + setIsOpen(false); + setShowFolderDialog(true); + }, + }, + ...(awsEnabled + ? [ + { + label: "Process S3 Bucket", + icon: Cloud, + onClick: () => { + setIsOpen(false); + setShowS3Dialog(true); + }, + }, + ] + : []), + ...cloudConnectorItems, + ]; - if (!taskId) { - throw new Error("No task ID received from server"); - } + // Comprehensive loading state + const isLoading = + fileUploading || folderLoading || s3Loading || isNavigatingToCloud; - addTask(taskId); - setBucketUrl("s3://"); - // Trigger search refresh after successful S3 processing starts - console.log("S3 upload successful, dispatching knowledgeUpdated event"); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - } else { - console.error("S3 upload failed:", result.error); - if (response.status === 400) { - toast.error("Upload failed", { - description: result.error || "Bad request", - }); - } - } - } catch (error) { - console.error("S3 upload error:", error); - } finally { - setS3Loading(false); - // Don't call refetchSearch() here - the knowledgeUpdated event will handle it - } - }; + return ( + <> +
+ - const cloudConnectorItems = Object.entries(cloudConnectors) - .filter(([, info]) => info.available) - .map(([type, info]) => ({ - label: info.name, - icon: PlugZap, - onClick: async () => { - setIsOpen(false); - if (info.connected && info.hasToken) { - setIsNavigatingToCloud(true); - try { - router.push(`/upload/${type}`); - // Keep loading state for a short time to show feedback - setTimeout(() => setIsNavigatingToCloud(false), 1000); - } catch { - setIsNavigatingToCloud(false); - } - } else { - router.push("/settings"); - } - }, - disabled: !info.connected || !info.hasToken, - tooltip: !info.connected - ? `Connect ${info.name} in Settings first` - : !info.hasToken - ? `Reconnect ${info.name} - access token required` - : undefined, - })); + {isOpen && !isLoading && ( +
+
+ {menuItems.map((item, index) => ( + + ))} +
+
+ )} - const menuItems = [ - { - label: "Add File", - icon: Upload, - onClick: handleFileUpload, - }, - { - label: "Process Folder", - icon: FolderOpen, - onClick: () => { - setIsOpen(false); - setShowFolderDialog(true); - }, - }, - ...(awsEnabled - ? [ - { - label: "Process S3 Bucket", - icon: Cloud, - onClick: () => { - setIsOpen(false); - setShowS3Dialog(true); - }, - }, - ] - : []), - ...cloudConnectorItems, - ]; + +
- // Comprehensive loading state - const isLoading = - fileUploading || folderLoading || s3Loading || isNavigatingToCloud; + {/* Process Folder Dialog */} + + + + + + Process Folder + + + Process all documents in a folder path + + +
+
+ + setFolderPath(e.target.value)} + /> +
+
+ + +
+
+ + - return ( - <> -
- - - {isOpen && !isLoading && ( -
-
- {menuItems.map((item, index) => ( - - ))} -
-
- )} - - -
- - {/* Process Folder Dialog */} - - - - - - Process Folder - - - Process all documents in a folder path - - -
-
- - setFolderPath(e.target.value)} - /> -
-
- - -
-
- - - - {/* Process S3 Bucket Dialog */} - - - - - - Process S3 Bucket - - - Process all documents from an S3 bucket. AWS credentials must be - configured. - - -
-
- - setBucketUrl(e.target.value)} - /> -
-
- - -
-
- - - - ); + {/* Process S3 Bucket Dialog */} + + + + + + Process S3 Bucket + + + Process all documents from an S3 bucket. AWS credentials must be + configured. + + +
+
+ + setBucketUrl(e.target.value)} + /> +
+
+ + +
+
+ + + + ); } diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index 5eb10ea9..b3275422 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -7,33 +7,18 @@ import { useCallback, useContext, useEffect, + useRef, useState, } from "react"; import { toast } from "sonner"; +import { useCancelTaskMutation } from "@/app/api/mutations/useCancelTaskMutation"; +import { + type Task, + useGetTasksQuery, +} from "@/app/api/queries/useGetTasksQuery"; import { useAuth } from "@/contexts/auth-context"; -export interface Task { - task_id: string; - status: - | "pending" - | "running" - | "processing" - | "completed" - | "failed" - | "error"; - total_files?: number; - processed_files?: number; - successful_files?: number; - failed_files?: number; - running_files?: number; - pending_files?: number; - created_at: string; - updated_at: string; - duration_seconds?: number; - result?: Record; - error?: string; - files?: Record>; -} +// Task interface is now imported from useGetTasksQuery export interface TaskFile { filename: string; @@ -58,20 +43,45 @@ interface TaskContextType { isFetching: boolean; isMenuOpen: boolean; toggleMenu: () => void; + // React Query states + isLoading: boolean; + error: Error | null; } const TaskContext = createContext(undefined); export function TaskProvider({ children }: { children: React.ReactNode }) { - const [tasks, setTasks] = useState([]); const [files, setFiles] = useState([]); - const [isPolling, setIsPolling] = useState(false); - const [isFetching, setIsFetching] = useState(false); const [isMenuOpen, setIsMenuOpen] = useState(false); + const previousTasksRef = useRef([]); const { isAuthenticated, isNoAuthMode } = useAuth(); const queryClient = useQueryClient(); + // Use React Query hooks + const { + data: tasks = [], + isLoading, + error, + refetch: refetchTasks, + isFetching, + } = useGetTasksQuery({ + enabled: isAuthenticated || isNoAuthMode, + }); + + const cancelTaskMutation = useCancelTaskMutation({ + onSuccess: () => { + toast.success("Task cancelled", { + description: "Task has been cancelled successfully", + }); + }, + onError: (error) => { + toast.error("Failed to cancel task", { + description: error.message, + }); + }, + }); + const refetchSearch = useCallback(() => { queryClient.invalidateQueries({ queryKey: ["search"], @@ -99,252 +109,171 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { [], ); - const fetchTasks = useCallback(async () => { - if (!isAuthenticated && !isNoAuthMode) return; - - setIsFetching(true); - try { - const response = await fetch("/api/tasks"); - if (response.ok) { - const data = await response.json(); - const newTasks = data.tasks || []; - - // Update tasks and check for status changes in the same state update - setTasks((prevTasks) => { - // Check for newly completed tasks to show toasts - if (prevTasks.length > 0) { - newTasks.forEach((newTask: Task) => { - const oldTask = prevTasks.find( - (t) => t.task_id === newTask.task_id, - ); - - // Update or add files from task.files if available - if (newTask.files && typeof newTask.files === "object") { - const taskFileEntries = Object.entries(newTask.files); - const now = new Date().toISOString(); - - taskFileEntries.forEach(([filePath, fileInfo]) => { - if (typeof fileInfo === "object" && fileInfo) { - const fileName = filePath.split("/").pop() || filePath; - const fileStatus = fileInfo.status as string; - - // Map backend file status to our TaskFile status - let mappedStatus: TaskFile["status"]; - switch (fileStatus) { - case "pending": - case "running": - mappedStatus = "processing"; - break; - case "completed": - mappedStatus = "active"; - break; - case "failed": - mappedStatus = "failed"; - break; - default: - mappedStatus = "processing"; - } - - setFiles((prevFiles) => { - const existingFileIndex = prevFiles.findIndex( - (f) => - f.source_url === filePath && - f.task_id === newTask.task_id, - ); - - // Detect connector type based on file path or other indicators - let connectorType = "local"; - if (filePath.includes("/") && !filePath.startsWith("/")) { - // Likely S3 key format (bucket/path/file.ext) - connectorType = "s3"; - } - - const fileEntry: TaskFile = { - filename: fileName, - mimetype: "", // We don't have this info from the task - source_url: filePath, - size: 0, // We don't have this info from the task - connector_type: connectorType, - status: mappedStatus, - task_id: newTask.task_id, - created_at: - typeof fileInfo.created_at === "string" - ? fileInfo.created_at - : now, - updated_at: - typeof fileInfo.updated_at === "string" - ? fileInfo.updated_at - : now, - }; - - if (existingFileIndex >= 0) { - // Update existing file - const updatedFiles = [...prevFiles]; - updatedFiles[existingFileIndex] = fileEntry; - return updatedFiles; - } else { - // Add new file - return [...prevFiles, fileEntry]; - } - }); - } - }); - } - - if ( - oldTask && - oldTask.status !== "completed" && - newTask.status === "completed" - ) { - // Task just completed - show success toast - toast.success("Task completed successfully", { - description: `Task ${newTask.task_id} has finished processing.`, - action: { - label: "View", - onClick: () => console.log("View task", newTask.task_id), - }, - }); - refetchSearch(); - // Dispatch knowledge updated event for all knowledge-related pages - console.log( - "Task completed successfully, dispatching knowledgeUpdated event", - ); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - - // Remove files for this completed task from the files list - setFiles((prevFiles) => - prevFiles.filter((file) => file.task_id !== newTask.task_id), - ); - } else if ( - oldTask && - oldTask.status !== "failed" && - oldTask.status !== "error" && - (newTask.status === "failed" || newTask.status === "error") - ) { - // Task just failed - show error toast - toast.error("Task failed", { - description: `Task ${newTask.task_id} failed: ${ - newTask.error || "Unknown error" - }`, - }); - - // Files will be updated to failed status by the file parsing logic above - } - }); - } - - return newTasks; - }); - } - } catch (error) { - console.error("Failed to fetch tasks:", error); - } finally { - setIsFetching(false); + // Handle task status changes and file updates + useEffect(() => { + if (tasks.length === 0) { + // Store current tasks as previous for next comparison + previousTasksRef.current = tasks; + return; } - }, [isAuthenticated, isNoAuthMode, refetchSearch]); // Removed 'tasks' from dependencies to prevent infinite loop! + console.log(tasks, previousTasksRef.current); - const addTask = useCallback((taskId: string) => { - // Immediately start aggressive polling for the new task - let pollAttempts = 0; - const maxPollAttempts = 30; // Poll for up to 30 seconds + // Check for task status changes by comparing with previous tasks + tasks.forEach((currentTask) => { + const previousTask = previousTasksRef.current.find( + (prev) => prev.task_id === currentTask.task_id, + ); - const aggressivePoll = async () => { - try { - const response = await fetch("/api/tasks"); - if (response.ok) { - const data = await response.json(); - const newTasks = data.tasks || []; - const foundTask = newTasks.find( - (task: Task) => task.task_id === taskId, - ); + // Only show toasts if we have previous data and status has changed + if (((previousTask && previousTask.status !== currentTask.status) || (!previousTask && previousTasksRef.current.length !== 0))) { + console.log("task status changed", currentTask.status); + // Process files from failed task and add them to files list + if (currentTask.files && typeof currentTask.files === "object") { + console.log("processing files", currentTask.files); + const taskFileEntries = Object.entries(currentTask.files); + const now = new Date().toISOString(); - if (foundTask) { - // Task found! Update the tasks state - setTasks((prevTasks) => { - // Check if task is already in the list - const exists = prevTasks.some((t) => t.task_id === taskId); - if (!exists) { - return [...prevTasks, foundTask]; + taskFileEntries.forEach(([filePath, fileInfo]) => { + if (typeof fileInfo === "object" && fileInfo) { + const fileName = filePath.split("/").pop() || filePath; + const fileStatus = fileInfo.status as string; + + // Map backend file status to our TaskFile status + let mappedStatus: TaskFile["status"]; + switch (fileStatus) { + case "pending": + case "running": + mappedStatus = "processing"; + break; + case "completed": + mappedStatus = "active"; + break; + case "failed": + mappedStatus = "failed"; + break; + default: + mappedStatus = "processing"; } - // Update existing task - return prevTasks.map((t) => - t.task_id === taskId ? foundTask : t, - ); - }); - return; // Stop polling, we found it - } + + setFiles((prevFiles) => { + const existingFileIndex = prevFiles.findIndex( + (f) => + f.source_url === filePath && + f.task_id === currentTask.task_id, + ); + + // Detect connector type based on file path or other indicators + let connectorType = "local"; + if (filePath.includes("/") && !filePath.startsWith("/")) { + // Likely S3 key format (bucket/path/file.ext) + connectorType = "s3"; + } + + const fileEntry: TaskFile = { + filename: fileName, + mimetype: "", // We don't have this info from the task + source_url: filePath, + size: 0, // We don't have this info from the task + connector_type: connectorType, + status: mappedStatus, + task_id: currentTask.task_id, + created_at: + typeof fileInfo.created_at === "string" + ? fileInfo.created_at + : now, + updated_at: + typeof fileInfo.updated_at === "string" + ? fileInfo.updated_at + : now, + }; + + if (existingFileIndex >= 0) { + // Update existing file + const updatedFiles = [...prevFiles]; + updatedFiles[existingFileIndex] = fileEntry; + return updatedFiles; + } else { + // Add new file + return [...prevFiles, fileEntry]; + } + }); + } + }); + } + if ( + previousTask && previousTask.status !== "completed" && + currentTask.status === "completed" + ) { + // Task just completed - show success toast + toast.success("Task completed successfully", { + description: `Task ${currentTask.task_id} has finished processing.`, + action: { + label: "View", + onClick: () => console.log("View task", currentTask.task_id), + }, + }); + refetchSearch(); + // Remove files for this completed task from the files list + // setFiles((prevFiles) => + // prevFiles.filter((file) => file.task_id !== currentTask.task_id), + // ); + } else if ( + previousTask && previousTask.status !== "failed" && + previousTask.status !== "error" && + (currentTask.status === "failed" || currentTask.status === "error") + ) { + // Task just failed - show error toast + toast.error("Task failed", { + description: `Task ${currentTask.task_id} failed: ${ + currentTask.error || "Unknown error" + }`, + }); } - } catch (error) { - console.error("Aggressive polling failed:", error); } + }); - pollAttempts++; - if (pollAttempts < maxPollAttempts) { - // Continue polling every 1 second for new tasks - setTimeout(aggressivePoll, 1000); - } - }; + // Store current tasks as previous for next comparison + previousTasksRef.current = tasks; + }, [tasks, refetchSearch]); - // Start aggressive polling after a short delay to allow backend to process - setTimeout(aggressivePoll, 500); - }, []); + const addTask = useCallback( + (_taskId: string) => { + // React Query will automatically handle polling when tasks are active + // Just trigger a refetch to get the latest data + refetchTasks(); + }, + [refetchTasks], + ); const refreshTasks = useCallback(async () => { - await fetchTasks(); - }, [fetchTasks]); + await refetchTasks(); + }, [refetchTasks]); - const removeTask = useCallback((taskId: string) => { - setTasks((prev) => prev.filter((task) => task.task_id !== taskId)); + const removeTask = useCallback((_taskId: string) => { + // This is now handled by React Query automatically + // Tasks will be removed from the list when they're no longer returned by the API }, []); const cancelTask = useCallback( async (taskId: string) => { - try { - const response = await fetch(`/api/tasks/${taskId}/cancel`, { - method: "POST", - }); - - if (response.ok) { - // Immediately refresh tasks to show the updated status - await fetchTasks(); - toast.success("Task cancelled", { - description: `Task ${taskId.substring(0, 8)}... has been cancelled`, - }); - } else { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.error || "Failed to cancel task"); - } - } catch (error) { - console.error("Failed to cancel task:", error); - toast.error("Failed to cancel task", { - description: error instanceof Error ? error.message : "Unknown error", - }); - } + cancelTaskMutation.mutate({ taskId }); }, - [fetchTasks], + [cancelTaskMutation], ); const toggleMenu = useCallback(() => { setIsMenuOpen((prev) => !prev); }, []); - // Periodic polling for task updates - useEffect(() => { - if (!isAuthenticated && !isNoAuthMode) return; - - setIsPolling(true); - - // Initial fetch - fetchTasks(); - - // Set up polling interval - every 3 seconds (more responsive for active tasks) - const interval = setInterval(fetchTasks, 3000); - - return () => { - clearInterval(interval); - setIsPolling(false); - }; - }, [isAuthenticated, isNoAuthMode, fetchTasks]); + // Determine if we're polling based on React Query's refetch interval + const isPolling = + isFetching && + tasks.some( + (task) => + task.status === "pending" || + task.status === "running" || + task.status === "processing", + ); const value: TaskContextType = { tasks, @@ -358,6 +287,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { isFetching, isMenuOpen, toggleMenu, + isLoading, + error, }; return {children}; From bf04d7ea600cc554a8b2151ce7e8848e824355e4 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Thu, 2 Oct 2025 14:51:33 -0600 Subject: [PATCH 04/55] [design sweep]: update no auth cloud connector warning message --- frontend/src/app/settings/page.tsx | 46 +++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index 6017ab5b..aca2d293 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -628,25 +628,49 @@ function KnowledgeSourcesPage() { {/* Conditional Sync Settings or No-Auth Message */} { - isNoAuthMode ? ( - + true ? ( + - - Cloud connectors are only available with auth mode enabled + + Cloud connectors require authentication - Please provide the following environment variables and - restart: + Add the Google OAuth variables below to your .env{" "} + then restart the OpenRAG containers.
-
- # make here - https://console.cloud.google.com/apis/credentials +
+
+ + 18 + + # Google OAuth +
+
+ + 19 + + # Create credentials here: +
+
+ + 20 + + + # https://console.cloud.google.com/apis/credentials + +
+
+
+ 21 + GOOGLE_OAUTH_CLIENT_ID= +
+
+ 22 + GOOGLE_OAUTH_CLIENT_SECRET=
-
GOOGLE_OAUTH_CLIENT_ID=
-
GOOGLE_OAUTH_CLIENT_SECRET=
From 892dcb3d74561671924c452d8439b8e89699e6df Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Thu, 2 Oct 2025 14:53:00 -0600 Subject: [PATCH 05/55] revert --- frontend/src/app/settings/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index aca2d293..71c0c8e4 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -628,7 +628,7 @@ function KnowledgeSourcesPage() { {/* Conditional Sync Settings or No-Auth Message */} { - true ? ( + isNoAuthMode ? ( From db715c122bf42dfd9a049607eda247fd5f18ff25 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 18:00:09 -0300 Subject: [PATCH 06/55] made failed and processing files not accessible and deletable --- frontend/src/app/knowledge/page.tsx | 604 ++++++++++++++-------------- 1 file changed, 307 insertions(+), 297 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 1b8b60ef..41485289 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -17,270 +17,280 @@ import "@/components/AgGrid/registerAgGridModules"; import "@/components/AgGrid/agGridStyles.css"; import { toast } from "sonner"; import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdown"; +import { filterAccentClasses } from "@/components/knowledge-filter-panel"; import { StatusBadge } from "@/components/ui/status-badge"; import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog"; import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; -import { filterAccentClasses } from "@/components/knowledge-filter-panel"; // Function to get the appropriate icon for a connector type function getSourceIcon(connectorType?: string) { - switch (connectorType) { - case "google_drive": - return ( - - ); - case "onedrive": - return ( - - ); - case "sharepoint": - return ; - case "s3": - return ; - default: - return ( - - ); - } + switch (connectorType) { + case "google_drive": + return ( + + ); + case "onedrive": + return ( + + ); + case "sharepoint": + return ; + case "s3": + return ; + default: + return ( + + ); + } } function SearchPage() { - const router = useRouter(); - const { isMenuOpen, files: taskFiles } = useTask(); - const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = - useKnowledgeFilter(); - const [selectedRows, setSelectedRows] = useState([]); - const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); + const router = useRouter(); + const { isMenuOpen, files: taskFiles } = useTask(); + const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = + useKnowledgeFilter(); + const [selectedRows, setSelectedRows] = useState([]); + const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); - const deleteDocumentMutation = useDeleteDocument(); + const deleteDocumentMutation = useDeleteDocument(); - const { data = [], isFetching } = useGetSearchQuery( - parsedFilterData?.query || "*", - parsedFilterData - ); + const { data = [], isFetching } = useGetSearchQuery( + parsedFilterData?.query || "*", + parsedFilterData, + ); - const handleTableSearch = (e: ChangeEvent) => { - gridRef.current?.api.setGridOption("quickFilterText", e.target.value); - }; + const handleTableSearch = (e: ChangeEvent) => { + gridRef.current?.api.setGridOption("quickFilterText", e.target.value); + }; - // Convert TaskFiles to File format and merge with backend results - const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { - return { - filename: taskFile.filename, - mimetype: taskFile.mimetype, - source_url: taskFile.source_url, - size: taskFile.size, - connector_type: taskFile.connector_type, - status: taskFile.status, - }; - }); + // Convert TaskFiles to File format and merge with backend results + const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { + return { + filename: taskFile.filename, + mimetype: taskFile.mimetype, + source_url: taskFile.source_url, + size: taskFile.size, + connector_type: taskFile.connector_type, + status: taskFile.status, + }; + }); - const backendFiles = data as File[]; + const backendFiles = data as File[]; - const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { - return ( - taskFile.status !== "active" && - !backendFiles.some( - (backendFile) => backendFile.filename === taskFile.filename - ) - ); - }); + const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { + return ( + taskFile.status !== "active" && + !backendFiles.some( + (backendFile) => backendFile.filename === taskFile.filename, + ) + ); + }); - // Combine task files first, then backend files - const fileResults = [...backendFiles, ...filteredTaskFiles]; + // Combine task files first, then backend files + const fileResults = [...backendFiles, ...filteredTaskFiles]; - const gridRef = useRef(null); + const gridRef = useRef(null); - const [columnDefs] = useState[]>([ - { - field: "filename", - headerName: "Source", - checkboxSelection: true, - headerCheckboxSelection: true, - initialFlex: 2, - minWidth: 220, - cellRenderer: ({ data, value }: CustomCellRendererProps) => { - return ( - - ); - }, - }, - { - field: "size", - headerName: "Size", - valueFormatter: (params) => - params.value ? `${Math.round(params.value / 1024)} KB` : "-", - }, - { - field: "mimetype", - headerName: "Type", - }, - { - field: "owner", - headerName: "Owner", - valueFormatter: (params) => - params.data?.owner_name || params.data?.owner_email || "—", - }, - { - field: "chunkCount", - headerName: "Chunks", - valueFormatter: (params) => params.data?.chunkCount?.toString() || "-", - }, - { - field: "avgScore", - headerName: "Avg score", - initialFlex: 0.5, - cellRenderer: ({ value }: CustomCellRendererProps) => { - return ( - - {value?.toFixed(2) ?? "-"} - - ); - }, - }, - { - field: "status", - headerName: "Status", - cellRenderer: ({ data }: CustomCellRendererProps) => { - // Default to 'active' status if no status is provided - const status = data?.status || "active"; - return ; - }, - }, - { - cellRenderer: ({ data }: CustomCellRendererProps) => { - return ; - }, - cellStyle: { - alignItems: "center", - display: "flex", - justifyContent: "center", - padding: 0, - }, - colId: "actions", - filter: false, - minWidth: 0, - width: 40, - resizable: false, - sortable: false, - initialFlex: 0, - }, - ]); + const [columnDefs] = useState[]>([ + { + field: "filename", + headerName: "Source", + checkboxSelection: (data) => (data?.data?.status || "active") === "active", + headerCheckboxSelection: true, + initialFlex: 2, + minWidth: 220, + cellRenderer: ({ data, value }: CustomCellRendererProps) => { + return ( +
{((data?.status || "active") !== "active") && +
+ } +
+ ); + }, + }, + { + field: "size", + headerName: "Size", + valueFormatter: (params) => + params.value ? `${Math.round(params.value / 1024)} KB` : "-", + }, + { + field: "mimetype", + headerName: "Type", + }, + { + field: "owner", + headerName: "Owner", + valueFormatter: (params) => + params.data?.owner_name || params.data?.owner_email || "—", + }, + { + field: "chunkCount", + headerName: "Chunks", + valueFormatter: (params) => params.data?.chunkCount?.toString() || "-", + }, + { + field: "avgScore", + headerName: "Avg score", + initialFlex: 0.5, + cellRenderer: ({ value }: CustomCellRendererProps) => { + return ( + + {value?.toFixed(2) ?? "-"} + + ); + }, + }, + { + field: "status", + headerName: "Status", + cellRenderer: ({ data }: CustomCellRendererProps) => { + // Default to 'active' status if no status is provided + const status = data?.status || "active"; + return ; + }, + }, + { + cellRenderer: ({ data }: CustomCellRendererProps) => { + const status = data?.status || "active"; + if (status !== "active") { + return null; + } + return ; + }, + cellStyle: { + alignItems: "center", + display: "flex", + justifyContent: "center", + padding: 0, + }, + colId: "actions", + filter: false, + minWidth: 0, + width: 40, + resizable: false, + sortable: false, + initialFlex: 0, + }, + ]); - const defaultColDef: ColDef = { - resizable: false, - suppressMovable: true, - initialFlex: 1, - minWidth: 100, - }; + const defaultColDef: ColDef = { + resizable: false, + suppressMovable: true, + initialFlex: 1, + minWidth: 100, + }; - const onSelectionChanged = useCallback(() => { - if (gridRef.current) { - const selectedNodes = gridRef.current.api.getSelectedRows(); - setSelectedRows(selectedNodes); - } - }, []); + const onSelectionChanged = useCallback(() => { + if (gridRef.current) { + const selectedNodes = gridRef.current.api.getSelectedRows(); + setSelectedRows(selectedNodes); + } + }, []); - const handleBulkDelete = async () => { - if (selectedRows.length === 0) return; + const handleBulkDelete = async () => { + if (selectedRows.length === 0) return; - try { - // Delete each file individually since the API expects one filename at a time - const deletePromises = selectedRows.map((row) => - deleteDocumentMutation.mutateAsync({ filename: row.filename }) - ); + try { + // Delete each file individually since the API expects one filename at a time + const deletePromises = selectedRows.map((row) => + deleteDocumentMutation.mutateAsync({ filename: row.filename }), + ); - await Promise.all(deletePromises); + await Promise.all(deletePromises); - toast.success( - `Successfully deleted ${selectedRows.length} document${ - selectedRows.length > 1 ? "s" : "" - }` - ); - setSelectedRows([]); - setShowBulkDeleteDialog(false); + toast.success( + `Successfully deleted ${selectedRows.length} document${ + selectedRows.length > 1 ? "s" : "" + }`, + ); + setSelectedRows([]); + setShowBulkDeleteDialog(false); - // Clear selection in the grid - if (gridRef.current) { - gridRef.current.api.deselectAll(); - } - } catch (error) { - toast.error( - error instanceof Error - ? error.message - : "Failed to delete some documents" - ); - } - }; + // Clear selection in the grid + if (gridRef.current) { + gridRef.current.api.deselectAll(); + } + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Failed to delete some documents", + ); + } + }; - return ( -
-
-
-

Project Knowledge

- -
+ return ( +
+
+
+

Project Knowledge

+ +
- {/* Search Input Area */} -
-
-
- {selectedFilter?.name && ( -
- {selectedFilter?.name} - setSelectedFilter(null)} - /> -
- )} - - -
- {/* */} - {/* //TODO: Implement sync button */} - {/* */} - {selectedRows.length > 0 && ( - - )} -
-
- params.data.filename} - domLayout="normal" - onSelectionChanged={onSelectionChanged} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> -
+ {selectedRows.length > 0 && ( + + )} + +
+ params.data.filename} + domLayout="normal" + onSelectionChanged={onSelectionChanged} + noRowsOverlayComponent={() => ( +
+
+ No knowledge +
+
+ Add files from local or your preferred cloud. +
+
+ )} + /> +
- {/* Bulk Delete Confirmation Dialog */} - 1 ? "s" : "" - }? This will remove all chunks and data associated with these documents. This action cannot be undone. + {/* Bulk Delete Confirmation Dialog */} + 1 ? "s" : "" + }? This will remove all chunks and data associated with these documents. This action cannot be undone. Documents to be deleted: ${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`} - confirmText="Delete All" - onConfirm={handleBulkDelete} - isLoading={deleteDocumentMutation.isPending} - /> -
- ); + confirmText="Delete All" + onConfirm={handleBulkDelete} + isLoading={deleteDocumentMutation.isPending} + /> +
+ ); } export default function ProtectedSearchPage() { - return ( - - - - ); + return ( + + + + ); } From 581879f5f6ffc5807096f8d93c481e13c4787ee3 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 18:04:45 -0300 Subject: [PATCH 07/55] fixed animated processing icon --- .../ui/animated-processing-icon.tsx | 52 +++++++------------ frontend/src/components/ui/status-badge.tsx | 2 +- 2 files changed, 21 insertions(+), 33 deletions(-) diff --git a/frontend/src/components/ui/animated-processing-icon.tsx b/frontend/src/components/ui/animated-processing-icon.tsx index eb36b2ab..51815414 100644 --- a/frontend/src/components/ui/animated-processing-icon.tsx +++ b/frontend/src/components/ui/animated-processing-icon.tsx @@ -1,26 +1,16 @@ -interface AnimatedProcessingIconProps { - className?: string; - size?: number; -} +import type { SVGProps } from "react"; -export const AnimatedProcessingIcon = ({ - className = "", - size = 10, -}: AnimatedProcessingIconProps) => { - const width = Math.round((size * 6) / 10); - const height = size; - - return ( - - + + + + + + + ); }; diff --git a/frontend/src/components/ui/status-badge.tsx b/frontend/src/components/ui/status-badge.tsx index d3b1a323..07dce58b 100644 --- a/frontend/src/components/ui/status-badge.tsx +++ b/frontend/src/components/ui/status-badge.tsx @@ -50,7 +50,7 @@ export const StatusBadge = ({ status, className }: StatusBadgeProps) => { }`} > {status === "processing" && ( - + )} {config.label} From 7d6feef9992b6bfeaf78db085a961dd5c4e72c4e Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 18:09:02 -0300 Subject: [PATCH 08/55] fixed status badge size --- frontend/src/components/ui/status-badge.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/ui/status-badge.tsx b/frontend/src/components/ui/status-badge.tsx index 07dce58b..e57ad3b5 100644 --- a/frontend/src/components/ui/status-badge.tsx +++ b/frontend/src/components/ui/status-badge.tsx @@ -50,7 +50,7 @@ export const StatusBadge = ({ status, className }: StatusBadgeProps) => { }`} > {status === "processing" && ( - + )} {config.label} From 179a7403ccaec179e2d0cad3e88c29a3deca08f1 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 18:09:18 -0300 Subject: [PATCH 09/55] fixed router, langflow files and processors to not use temp names --- src/api/langflow_files.py | 15 ++++++++------- src/api/router.py | 18 ++++++++++-------- src/models/processors.py | 8 ++------ 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/api/langflow_files.py b/src/api/langflow_files.py index 1c83724d..ae36bf04 100644 --- a/src/api/langflow_files.py +++ b/src/api/langflow_files.py @@ -189,19 +189,20 @@ async def upload_and_ingest_user_file( # Create temporary file for task processing import tempfile import os - + # Read file content content = await upload_file.read() - - # Create temporary file + + # Create temporary file with the actual filename (not a temp prefix) + # Store in temp directory but use the real filename + temp_dir = tempfile.gettempdir() safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_") - temp_fd, temp_path = tempfile.mkstemp( - suffix=f"_{safe_filename}" - ) + temp_path = os.path.join(temp_dir, safe_filename) + try: # Write content to temp file - with os.fdopen(temp_fd, 'wb') as temp_file: + with open(temp_path, 'wb') as temp_file: temp_file.write(content) logger.debug("Created temporary file for task processing", temp_path=temp_path) diff --git a/src/api/router.py b/src/api/router.py index 56789d41..857914c0 100644 --- a/src/api/router.py +++ b/src/api/router.py @@ -114,20 +114,22 @@ async def langflow_upload_ingest_task( temp_file_paths = [] try: + # Create temp directory reference once + temp_dir = tempfile.gettempdir() + for upload_file in upload_files: # Read file content content = await upload_file.read() - - # Create temporary file + + # Create temporary file with the actual filename (not a temp prefix) + # Store in temp directory but use the real filename safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_") - temp_fd, temp_path = tempfile.mkstemp( - suffix=f"_{safe_filename}" - ) - + temp_path = os.path.join(temp_dir, safe_filename) + # Write content to temp file - with os.fdopen(temp_fd, 'wb') as temp_file: + with open(temp_path, 'wb') as temp_file: temp_file.write(content) - + temp_file_paths.append(temp_path) logger.debug( diff --git a/src/models/processors.py b/src/models/processors.py index ecec9c49..0ce8e33d 100644 --- a/src/models/processors.py +++ b/src/models/processors.py @@ -574,12 +574,8 @@ class LangflowFileProcessor(TaskProcessor): content = f.read() # Create file tuple for upload - temp_filename = os.path.basename(item) - # Extract original filename from temp file suffix (remove tmp prefix) - if "_" in temp_filename: - filename = temp_filename.split("_", 1)[1] # Get everything after first _ - else: - filename = temp_filename + # The temp file now has the actual filename, no need to extract it + filename = os.path.basename(item) content_type, _ = mimetypes.guess_type(filename) if not content_type: content_type = 'application/octet-stream' From 3a1bde1bb5a6260ab0ade4a5e2ffb713f0ac6d46 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Thu, 2 Oct 2025 17:17:09 -0500 Subject: [PATCH 10/55] refactor page layout styles to use grid --- frontend/components/knowledge-dropdown.tsx | 110 +++++------------- .../components/knowledge-filter-panel.tsx | 4 +- frontend/src/app/globals.css | 25 ++++ frontend/src/app/knowledge/page.tsx | 49 +++----- frontend/src/app/settings/page.tsx | 50 ++++---- frontend/src/components/layout-wrapper.tsx | 69 ++++++----- .../src/components/task-notification-menu.tsx | 4 +- 7 files changed, 145 insertions(+), 166 deletions(-) diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index ee49fc3a..c70fb234 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -6,7 +6,6 @@ import { FolderOpen, Loader2, PlugZap, - Plus, Upload, } from "lucide-react"; import { useRouter } from "next/navigation"; @@ -25,15 +24,7 @@ import { Label } from "@/components/ui/label"; import { useTask } from "@/contexts/task-context"; import { cn } from "@/lib/utils"; -interface KnowledgeDropdownProps { - active?: boolean; - variant?: "navigation" | "button"; -} - -export function KnowledgeDropdown({ - active, - variant = "navigation", -}: KnowledgeDropdownProps) { +export function KnowledgeDropdown() { const { addTask } = useTask(); const router = useRouter(); const [isOpen, setIsOpen] = useState(false); @@ -437,77 +428,35 @@ export function KnowledgeDropdown({ return ( <>
- + /> + )} + + {isOpen && !isLoading && (
@@ -519,12 +468,13 @@ export function KnowledgeDropdown({ disabled={"disabled" in item ? item.disabled : false} title={"tooltip" in item ? item.tooltip : undefined} className={cn( - "w-full px-3 py-2 text-left text-sm hover:bg-accent hover:text-accent-foreground", + "w-full flex flex-nowrap items-center px-3 py-2 text-left text-sm hover:bg-accent hover:text-accent-foreground", "disabled" in item && item.disabled && "opacity-50 cursor-not-allowed hover:bg-transparent hover:text-current" )} > + {item.icon && } {item.label} ))} @@ -561,7 +511,7 @@ export function KnowledgeDropdown({ type="text" placeholder="/path/to/documents" value={folderPath} - onChange={e => setFolderPath(e.target.value)} + onChange={(e) => setFolderPath(e.target.value)} />
@@ -603,7 +553,7 @@ export function KnowledgeDropdown({ type="text" placeholder="s3://bucket/path" value={bucketUrl} - onChange={e => setBucketUrl(e.target.value)} + onChange={(e) => setBucketUrl(e.target.value)} />
diff --git a/frontend/components/knowledge-filter-panel.tsx b/frontend/components/knowledge-filter-panel.tsx index 6bf9285b..a45304e9 100644 --- a/frontend/components/knowledge-filter-panel.tsx +++ b/frontend/components/knowledge-filter-panel.tsx @@ -231,8 +231,8 @@ export function KnowledgeFilterPanel() { }; return ( -
- +
+
diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css index 150d1c56..22601457 100644 --- a/frontend/src/app/globals.css +++ b/frontend/src/app/globals.css @@ -53,6 +53,8 @@ --flow-icon: #2f67d0; --radius: 0.5rem; + + --app-header-height: 53px; } .dark { @@ -108,6 +110,29 @@ } @layer components { + .app-grid-cols-arrangement { + --sidebar-width: 0px; + --notifications-width: 0px; + --filters-width: 0px; + + @media (width >= 48rem) { + --sidebar-width: 288px; + } + + &.notifications-open { + --notifications-width: 320px; + } + &.filters-open { + --filters-width: 320px; + } + display: grid; + height: calc(100% - var(--app-header-height)); + grid-template-columns: var(--sidebar-width) 1fr var(--notifications-width) var( + --filters-width + ); + transition: grid-template-columns 0.3s ease-in-out; + } + .header-arrangement { @apply flex w-full h-[53px] items-center justify-between border-b border-border; } diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 1b8b60ef..34595eb2 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -1,8 +1,8 @@ "use client"; -import type { ColDef } from "ag-grid-community"; +import { themeQuartz, type ColDef } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; -import { Building2, Cloud, HardDrive, Search, Trash2, X } from "lucide-react"; +import { Building2, Cloud, HardDrive, Search, X } from "lucide-react"; import { useRouter } from "next/navigation"; import { type ChangeEvent, useCallback, useRef, useState } from "react"; import { SiGoogledrive } from "react-icons/si"; @@ -46,8 +46,8 @@ function getSourceIcon(connectorType?: string) { function SearchPage() { const router = useRouter(); - const { isMenuOpen, files: taskFiles } = useTask(); - const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = + const { files: taskFiles } = useTask(); + const { selectedFilter, setSelectedFilter, parsedFilterData } = useKnowledgeFilter(); const [selectedRows, setSelectedRows] = useState([]); const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); @@ -144,7 +144,6 @@ function SearchPage() { { field: "avgScore", headerName: "Avg score", - initialFlex: 0.5, cellRenderer: ({ value }: CustomCellRendererProps) => { return ( @@ -229,32 +228,19 @@ function SearchPage() { }; return ( -
-
+ <> +
-

Project Knowledge

- +

Knowledge

{/* Search Input Area */} -
-
-
+
+ +
{selectedFilter?.name && (
setShowBulkDeleteDialog(true)} > - Delete + Delete )} +
+ +
params.data.filename} domLayout="normal" + theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} onSelectionChanged={onSelectionChanged} noRowsOverlayComponent={() => (
@@ -347,15 +337,12 @@ function SearchPage() { selectedRows.length } document${ selectedRows.length > 1 ? "s" : "" - }? This will remove all chunks and data associated with these documents. This action cannot be undone. - -Documents to be deleted: -${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`} + }? This will remove all chunks and data associated with these documents. This action cannot be undone.`} confirmText="Delete All" onConfirm={handleBulkDelete} isLoading={deleteDocumentMutation.isPending} /> -
+ ); } diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index 7530bcbb..587d3946 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -114,7 +114,8 @@ function KnowledgeSourcesPage() { const [chunkOverlap, setChunkOverlap] = useState(50); const [tableStructure, setTableStructure] = useState(false); const [ocr, setOcr] = useState(false); - const [pictureDescriptions, setPictureDescriptions] = useState(false); + const [pictureDescriptions, setPictureDescriptions] = + useState(false); // Fetch settings using React Query const { data: settings = {} } = useGetSettingsQuery({ @@ -164,7 +165,7 @@ function KnowledgeSourcesPage() { onSuccess: () => { console.log("Setting updated successfully"); }, - onError: error => { + onError: (error) => { console.error("Failed to update setting:", error.message); }, }); @@ -303,8 +304,8 @@ function KnowledgeSourcesPage() { // Initialize connectors list with metadata from backend const initialConnectors = connectorTypes - .filter(type => connectorsResult.connectors[type].available) // Only show available connectors - .map(type => ({ + .filter((type) => connectorsResult.connectors[type].available) // Only show available connectors + .map((type) => ({ id: type, name: connectorsResult.connectors[type].name, description: connectorsResult.connectors[type].description, @@ -327,8 +328,8 @@ function KnowledgeSourcesPage() { ); const isConnected = activeConnection !== undefined; - setConnectors(prev => - prev.map(c => + setConnectors((prev) => + prev.map((c) => c.type === connectorType ? { ...c, @@ -347,7 +348,7 @@ function KnowledgeSourcesPage() { const handleConnect = async (connector: Connector) => { setIsConnecting(connector.id); - setSyncResults(prev => ({ ...prev, [connector.id]: null })); + setSyncResults((prev) => ({ ...prev, [connector.id]: null })); try { // Use the shared auth callback URL, same as connectors page @@ -508,9 +509,9 @@ function KnowledgeSourcesPage() { // Watch for task completions and refresh stats useEffect(() => { // Find newly completed tasks by comparing with previous state - const newlyCompletedTasks = tasks.filter(task => { + const newlyCompletedTasks = tasks.filter((task) => { const wasCompleted = - prevTasks.find(prev => prev.task_id === task.task_id)?.status === + prevTasks.find((prev) => prev.task_id === task.task_id)?.status === "completed"; return task.status === "completed" && !wasCompleted; }); @@ -564,7 +565,7 @@ function KnowledgeSourcesPage() { fetch(`/api/reset-flow/retrieval`, { method: "POST", }) - .then(response => { + .then((response) => { if (response.ok) { return response.json(); } @@ -577,7 +578,7 @@ function KnowledgeSourcesPage() { handleModelChange(DEFAULT_AGENT_SETTINGS.llm_model); closeDialog(); // Close after successful completion }) - .catch(error => { + .catch((error) => { console.error("Error restoring retrieval flow:", error); closeDialog(); // Close even on error (could show error toast instead) }); @@ -587,7 +588,7 @@ function KnowledgeSourcesPage() { fetch(`/api/reset-flow/ingest`, { method: "POST", }) - .then(response => { + .then((response) => { if (response.ok) { return response.json(); } @@ -602,14 +603,14 @@ function KnowledgeSourcesPage() { setPictureDescriptions(false); closeDialog(); // Close after successful completion }) - .catch(error => { + .catch((error) => { console.error("Error restoring ingest flow:", error); closeDialog(); // Close even on error (could show error toast instead) }); }; return ( -
+
{/* Connectors Section */}
@@ -700,7 +701,7 @@ function KnowledgeSourcesPage() { {/* Connectors Grid */}
- {connectors.map(connector => ( + {connectors.map((connector) => (
@@ -834,7 +835,7 @@ function KnowledgeSourcesPage() { } confirmText="Proceed" confirmIcon={} - onConfirm={closeDialog => + onConfirm={(closeDialog) => handleEditInLangflow("chat", closeDialog) } variant="warning" @@ -854,7 +855,8 @@ function KnowledgeSourcesPage() { : null} - id="search-query" - type="text" - defaultValue={parsedFilterData?.query} - value={queryInputText} - onChange={(e) => setQueryInputText(e.target.value)} - placeholder="Search chunks..." - /> +
+
+ {selectedFilter?.name && ( +
+ {selectedFilter?.name} + setSelectedFilter(null)} + /> +
+ )} + + setQueryInputText(e.target.value)} + /> +
Date: Fri, 3 Oct 2025 10:11:39 -0300 Subject: [PATCH 13/55] fixed issues with already ingested files, fixed state issues --- frontend/src/app/knowledge/page.tsx | 11 +++++------ frontend/src/contexts/task-context.tsx | 9 +++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 41485289..c6d254c4 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -75,14 +75,13 @@ function SearchPage() { }; }); - const backendFiles = data as File[]; + const backendFiles = (data as File[]).filter((file) => !taskFilesAsFiles.some((taskFile) => taskFile.filename === file.filename && taskFile.status === "processing")); const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { return ( taskFile.status !== "active" && !backendFiles.some( - (backendFile) => backendFile.filename === taskFile.filename, - ) + (backendFile) => backendFile.filename === taskFile.filename,) ); }); @@ -91,7 +90,7 @@ function SearchPage() { const gridRef = useRef(null); - const [columnDefs] = useState[]>([ + const columnDefs = [ { field: "filename", headerName: "Source", @@ -189,8 +188,8 @@ function SearchPage() { resizable: false, sortable: false, initialFlex: 0, - }, - ]); + } + ]; const defaultColDef: ColDef = { resizable: false, diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index b3275422..8928ae8f 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -212,11 +212,12 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { onClick: () => console.log("View task", currentTask.task_id), }, }); + setTimeout(() => { refetchSearch(); - // Remove files for this completed task from the files list - // setFiles((prevFiles) => - // prevFiles.filter((file) => file.task_id !== currentTask.task_id), - // ); + setFiles((prevFiles) => + prevFiles.filter((file) => file.task_id !== currentTask.task_id && file.status !== "failed"), + ); + }, 500); } else if ( previousTask && previousTask.status !== "failed" && previousTask.status !== "error" && From 7b860b3f6f55c7c3a2d09b392f37a4c39a0d51ed Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 09:08:33 -0600 Subject: [PATCH 14/55] fix frontend ID --- frontend/src/app/api/queries/useGetSearchQuery.ts | 5 +++-- frontend/src/app/knowledge/chunks/page.tsx | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/api/queries/useGetSearchQuery.ts b/frontend/src/app/api/queries/useGetSearchQuery.ts index 5383178d..d0c1a3a9 100644 --- a/frontend/src/app/api/queries/useGetSearchQuery.ts +++ b/frontend/src/app/api/queries/useGetSearchQuery.ts @@ -29,6 +29,7 @@ export interface ChunkResult { owner_email?: string; file_size?: number; connector_type?: string; + index?: number; } export interface File { @@ -55,7 +56,7 @@ export interface File { export const useGetSearchQuery = ( query: string, queryData?: ParsedQueryData | null, - options?: Omit, + options?: Omit ) => { const queryClient = useQueryClient(); @@ -184,7 +185,7 @@ export const useGetSearchQuery = ( queryFn: getFiles, ...options, }, - queryClient, + queryClient ); return queryResult; diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index bc00f886..9b02506b 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -86,7 +86,9 @@ function ChunksPageContent() { return; } - setChunks(fileData?.chunks || []); + setChunks( + fileData?.chunks?.map((chunk, i) => ({ ...chunk, index: i + 1 })) || [] + ); }, [data, filename]); // Set selected state for all checkboxes when selectAll changes @@ -246,7 +248,7 @@ function ChunksPageContent() { />
- Chunk {chunk.page} + Chunk {chunk.index} {chunk.text.length} chars From ac333b99d7317b8b3400e280d836454ffae64676 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 09:12:35 -0600 Subject: [PATCH 15/55] added value --- frontend/src/app/knowledge/chunks/page.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index 9b02506b..bbb7f019 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -188,6 +188,7 @@ function ChunksPageContent() { type="text" placeholder="Search your documents..." onChange={(e) => setQueryInputText(e.target.value)} + value={queryInputText} />
From 11a6f0e8d415ce989929c49e566da75d474b11bf Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 10:47:31 -0600 Subject: [PATCH 16/55] updated header and cloud picker header --- frontend/src/app/upload/[provider]/page.tsx | 659 +++++++++--------- .../components/cloud-picker/picker-header.tsx | 8 +- 2 files changed, 332 insertions(+), 335 deletions(-) diff --git a/frontend/src/app/upload/[provider]/page.tsx b/frontend/src/app/upload/[provider]/page.tsx index 7c72ec3d..8e0a306c 100644 --- a/frontend/src/app/upload/[provider]/page.tsx +++ b/frontend/src/app/upload/[provider]/page.tsx @@ -12,367 +12,370 @@ import { useTask } from "@/contexts/task-context"; // CloudFile interface is now imported from the unified cloud picker interface CloudConnector { - id: string; - name: string; - description: string; - status: "not_connected" | "connecting" | "connected" | "error"; - type: string; - connectionId?: string; - clientId: string; - hasAccessToken: boolean; - accessTokenError?: string; + id: string; + name: string; + description: string; + status: "not_connected" | "connecting" | "connected" | "error"; + type: string; + connectionId?: string; + clientId: string; + hasAccessToken: boolean; + accessTokenError?: string; } export default function UploadProviderPage() { - const params = useParams(); - const router = useRouter(); - const provider = params.provider as string; - const { addTask, tasks } = useTask(); + const params = useParams(); + const router = useRouter(); + const provider = params.provider as string; + const { addTask, tasks } = useTask(); - const [connector, setConnector] = useState(null); - const [isLoading, setIsLoading] = useState(true); - const [error, setError] = useState(null); - const [accessToken, setAccessToken] = useState(null); - const [selectedFiles, setSelectedFiles] = useState([]); - const [isIngesting, setIsIngesting] = useState(false); - const [currentSyncTaskId, setCurrentSyncTaskId] = useState( - null, - ); - const [ingestSettings, setIngestSettings] = useState({ - chunkSize: 1000, - chunkOverlap: 200, - ocr: false, - pictureDescriptions: false, - embeddingModel: "text-embedding-3-small", - }); + const [connector, setConnector] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const [accessToken, setAccessToken] = useState(null); + const [selectedFiles, setSelectedFiles] = useState([]); + const [isIngesting, setIsIngesting] = useState(false); + const [currentSyncTaskId, setCurrentSyncTaskId] = useState( + null + ); + const [ingestSettings, setIngestSettings] = useState({ + chunkSize: 1000, + chunkOverlap: 200, + ocr: false, + pictureDescriptions: false, + embeddingModel: "text-embedding-3-small", + }); - useEffect(() => { - const fetchConnectorInfo = async () => { - setIsLoading(true); - setError(null); + useEffect(() => { + const fetchConnectorInfo = async () => { + setIsLoading(true); + setError(null); - try { - // Fetch available connectors to validate the provider - const connectorsResponse = await fetch("/api/connectors"); - if (!connectorsResponse.ok) { - throw new Error("Failed to load connectors"); - } + try { + // Fetch available connectors to validate the provider + const connectorsResponse = await fetch("/api/connectors"); + if (!connectorsResponse.ok) { + throw new Error("Failed to load connectors"); + } - const connectorsResult = await connectorsResponse.json(); - const providerInfo = connectorsResult.connectors[provider]; + const connectorsResult = await connectorsResponse.json(); + const providerInfo = connectorsResult.connectors[provider]; - if (!providerInfo || !providerInfo.available) { - setError( - `Cloud provider "${provider}" is not available or configured.`, - ); - return; - } + if (!providerInfo || !providerInfo.available) { + setError( + `Cloud provider "${provider}" is not available or configured.` + ); + return; + } - // Check connector status - const statusResponse = await fetch( - `/api/connectors/${provider}/status`, - ); - if (!statusResponse.ok) { - throw new Error(`Failed to check ${provider} status`); - } + // Check connector status + const statusResponse = await fetch( + `/api/connectors/${provider}/status` + ); + if (!statusResponse.ok) { + throw new Error(`Failed to check ${provider} status`); + } - const statusData = await statusResponse.json(); - const connections = statusData.connections || []; - const activeConnection = connections.find( - (conn: { is_active: boolean; connection_id: string }) => - conn.is_active, - ); - const isConnected = activeConnection !== undefined; + const statusData = await statusResponse.json(); + const connections = statusData.connections || []; + const activeConnection = connections.find( + (conn: { is_active: boolean; connection_id: string }) => + conn.is_active + ); + const isConnected = activeConnection !== undefined; - let hasAccessToken = false; - let accessTokenError: string | undefined; + let hasAccessToken = false; + let accessTokenError: string | undefined; - // Try to get access token for connected connectors - if (isConnected && activeConnection) { - try { - const tokenResponse = await fetch( - `/api/connectors/${provider}/token?connection_id=${activeConnection.connection_id}`, - ); - if (tokenResponse.ok) { - const tokenData = await tokenResponse.json(); - if (tokenData.access_token) { - hasAccessToken = true; - setAccessToken(tokenData.access_token); - } - } else { - const errorData = await tokenResponse - .json() - .catch(() => ({ error: "Token unavailable" })); - accessTokenError = errorData.error || "Access token unavailable"; - } - } catch { - accessTokenError = "Failed to fetch access token"; - } - } + // Try to get access token for connected connectors + if (isConnected && activeConnection) { + try { + const tokenResponse = await fetch( + `/api/connectors/${provider}/token?connection_id=${activeConnection.connection_id}` + ); + if (tokenResponse.ok) { + const tokenData = await tokenResponse.json(); + if (tokenData.access_token) { + hasAccessToken = true; + setAccessToken(tokenData.access_token); + } + } else { + const errorData = await tokenResponse + .json() + .catch(() => ({ error: "Token unavailable" })); + accessTokenError = errorData.error || "Access token unavailable"; + } + } catch { + accessTokenError = "Failed to fetch access token"; + } + } - setConnector({ - id: provider, - name: providerInfo.name, - description: providerInfo.description, - status: isConnected ? "connected" : "not_connected", - type: provider, - connectionId: activeConnection?.connection_id, - clientId: activeConnection?.client_id, - hasAccessToken, - accessTokenError, - }); - } catch (error) { - console.error("Failed to load connector info:", error); - setError( - error instanceof Error - ? error.message - : "Failed to load connector information", - ); - } finally { - setIsLoading(false); - } - }; + setConnector({ + id: provider, + name: providerInfo.name, + description: providerInfo.description, + status: isConnected ? "connected" : "not_connected", + type: provider, + connectionId: activeConnection?.connection_id, + clientId: activeConnection?.client_id, + hasAccessToken, + accessTokenError, + }); + } catch (error) { + console.error("Failed to load connector info:", error); + setError( + error instanceof Error + ? error.message + : "Failed to load connector information" + ); + } finally { + setIsLoading(false); + } + }; - if (provider) { - fetchConnectorInfo(); - } - }, [provider]); + if (provider) { + fetchConnectorInfo(); + } + }, [provider]); - // Watch for sync task completion and redirect - useEffect(() => { - if (!currentSyncTaskId) return; + // Watch for sync task completion and redirect + useEffect(() => { + if (!currentSyncTaskId) return; - const currentTask = tasks.find( - (task) => task.task_id === currentSyncTaskId, - ); + const currentTask = tasks.find( + (task) => task.task_id === currentSyncTaskId + ); - if (currentTask && currentTask.status === "completed") { - // Task completed successfully, show toast and redirect - setIsIngesting(false); - setTimeout(() => { - router.push("/knowledge"); - }, 2000); // 2 second delay to let user see toast - } else if (currentTask && currentTask.status === "failed") { - // Task failed, clear the tracking but don't redirect - setIsIngesting(false); - setCurrentSyncTaskId(null); - } - }, [tasks, currentSyncTaskId, router]); + if (currentTask && currentTask.status === "completed") { + // Task completed successfully, show toast and redirect + setIsIngesting(false); + setTimeout(() => { + router.push("/knowledge"); + }, 2000); // 2 second delay to let user see toast + } else if (currentTask && currentTask.status === "failed") { + // Task failed, clear the tracking but don't redirect + setIsIngesting(false); + setCurrentSyncTaskId(null); + } + }, [tasks, currentSyncTaskId, router]); - const handleFileSelected = (files: CloudFile[]) => { - setSelectedFiles(files); - console.log(`Selected ${files.length} files from ${provider}:`, files); - // You can add additional handling here like triggering sync, etc. - }; + const handleFileSelected = (files: CloudFile[]) => { + setSelectedFiles(files); + console.log(`Selected ${files.length} files from ${provider}:`, files); + // You can add additional handling here like triggering sync, etc. + }; - const handleSync = async (connector: CloudConnector) => { - if (!connector.connectionId || selectedFiles.length === 0) return; + const handleSync = async (connector: CloudConnector) => { + if (!connector.connectionId || selectedFiles.length === 0) return; - setIsIngesting(true); + setIsIngesting(true); - try { - const syncBody: { - connection_id: string; - max_files?: number; - selected_files?: string[]; - settings?: IngestSettings; - } = { - connection_id: connector.connectionId, - selected_files: selectedFiles.map((file) => file.id), - settings: ingestSettings, - }; + try { + const syncBody: { + connection_id: string; + max_files?: number; + selected_files?: string[]; + settings?: IngestSettings; + } = { + connection_id: connector.connectionId, + selected_files: selectedFiles.map((file) => file.id), + settings: ingestSettings, + }; - const response = await fetch(`/api/connectors/${connector.type}/sync`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(syncBody), - }); + const response = await fetch(`/api/connectors/${connector.type}/sync`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(syncBody), + }); - const result = await response.json(); + const result = await response.json(); - if (response.status === 201) { - const taskIds = result.task_ids; - if (taskIds && taskIds.length > 0) { - const taskId = taskIds[0]; // Use the first task ID - addTask(taskId); - setCurrentSyncTaskId(taskId); - } - } else { - console.error("Sync failed:", result.error); - } - } catch (error) { - console.error("Sync error:", error); - setIsIngesting(false); - } - }; + if (response.status === 201) { + const taskIds = result.task_ids; + if (taskIds && taskIds.length > 0) { + const taskId = taskIds[0]; // Use the first task ID + addTask(taskId); + setCurrentSyncTaskId(taskId); + } + } else { + console.error("Sync failed:", result.error); + } + } catch (error) { + console.error("Sync error:", error); + setIsIngesting(false); + } + }; - const getProviderDisplayName = () => { - const nameMap: { [key: string]: string } = { - google_drive: "Google Drive", - onedrive: "OneDrive", - sharepoint: "SharePoint", - }; - return nameMap[provider] || provider; - }; + const getProviderDisplayName = () => { + const nameMap: { [key: string]: string } = { + google_drive: "Google Drive", + onedrive: "OneDrive", + sharepoint: "SharePoint", + }; + return nameMap[provider] || provider; + }; - if (isLoading) { - return ( -
-
-
-
-

Loading {getProviderDisplayName()} connector...

-
-
-
- ); - } + if (isLoading) { + return ( +
+
+
+
+

Loading {getProviderDisplayName()} connector...

+
+
+
+ ); + } - if (error || !connector) { - return ( -
-
- -
+ if (error || !connector) { + return ( +
+
+ +
-
-
- -

- Provider Not Available -

-

{error}

- -
-
-
- ); - } +
+
+ +

+ Provider Not Available +

+

{error}

+ +
+
+
+ ); + } - if (connector.status !== "connected") { - return ( -
-
- -
+ if (connector.status !== "connected") { + return ( +
+
+ +
-
-
- -

- {connector.name} Not Connected -

-

- You need to connect your {connector.name} account before you can - select files. -

- -
-
-
- ); - } +
+
+ +

+ {connector.name} Not Connected +

+

+ You need to connect your {connector.name} account before you can + select files. +

+ +
+
+
+ ); + } - if (!connector.hasAccessToken) { - return ( -
-
- -
+ if (!connector.hasAccessToken) { + return ( +
+
+ +
-
-
- -

- Access Token Required -

-

- {connector.accessTokenError || - `Unable to get access token for ${connector.name}. Try reconnecting your account.`} -

- -
-
-
- ); - } +
+
+ +

+ Access Token Required +

+

+ {connector.accessTokenError || + `Unable to get access token for ${connector.name}. Try reconnecting your account.`} +

+ +
+
+
+ ); + } - return ( -
-
- -

- Add from {getProviderDisplayName()} -

-
+ return ( +
+
+ +

+ Add from {getProviderDisplayName()} +

+
-
- -
+
+ +
-
-
- - -
-
-
- ); +
+
+ + +
+
+
+ ); } diff --git a/frontend/src/components/cloud-picker/picker-header.tsx b/frontend/src/components/cloud-picker/picker-header.tsx index 05dcaebd..e0d9cfa4 100644 --- a/frontend/src/components/cloud-picker/picker-header.tsx +++ b/frontend/src/components/cloud-picker/picker-header.tsx @@ -51,19 +51,13 @@ export const PickerHeader = ({ Select files from {getProviderName(provider)} to ingest.

-
- csv, json, pdf,{" "} - +16 more{" "} - 150 MB max -
); From ae02fc00e8b90c95e3dbcdb0dd0a4d205aa26f6d Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 10:48:42 -0600 Subject: [PATCH 17/55] updated numbers to match env.example --- frontend/src/app/settings/page.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index 71c0c8e4..07aeb5f8 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -644,19 +644,19 @@ function KnowledgeSourcesPage() {
- 18 + 27 # Google OAuth
- 19 + 28 # Create credentials here:
- 20 + 29 # https://console.cloud.google.com/apis/credentials @@ -664,11 +664,11 @@ function KnowledgeSourcesPage() {
- 21 + 30 GOOGLE_OAUTH_CLIENT_ID=
- 22 + 31 GOOGLE_OAUTH_CLIENT_SECRET=
From a9bb26001b92663bb26de4e5e98a625dd3609de3 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Fri, 3 Oct 2025 11:54:30 -0500 Subject: [PATCH 18/55] fix layout merge conflicts --- frontend/src/app/globals.css | 2 +- frontend/src/app/layout.tsx | 2 +- frontend/src/components/layout-wrapper.tsx | 22 +++++++++++++--------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css index 22601457..e3f4c154 100644 --- a/frontend/src/app/globals.css +++ b/frontend/src/app/globals.css @@ -54,7 +54,7 @@ --radius: 0.5rem; - --app-header-height: 53px; + --app-header-height: 40px; } .dark { diff --git a/frontend/src/app/layout.tsx b/frontend/src/app/layout.tsx index baf9f2d5..10ed826b 100644 --- a/frontend/src/app/layout.tsx +++ b/frontend/src/app/layout.tsx @@ -38,7 +38,7 @@ export default function RootLayout({ return ( + (task) => task.status === "pending" || task.status === "running" || task.status === "processing" @@ -84,10 +83,9 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { // For all other pages, render with Langflow-styled navigation and task menu return ( -
- {/* Header */} -
-
+
+
+
{/* Logo/Title */}
@@ -126,7 +124,6 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
-
-
{children}
+
+ {children} +
{/* Task Notifications Panel */} From b5b777d038601688b84bc27f47d6aa8ded1fe941 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Fri, 3 Oct 2025 12:37:43 -0500 Subject: [PATCH 19/55] design sweep --- frontend/components/knowledge-dropdown.tsx | 10 ++++++---- frontend/src/components/layout-wrapper.tsx | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index c70fb234..b6a9fea5 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -459,8 +459,8 @@ export function KnowledgeDropdown() { {isOpen && !isLoading && ( -
-
+
+
{menuItems.map((item, index) => ( ))} diff --git a/frontend/src/components/layout-wrapper.tsx b/frontend/src/components/layout-wrapper.tsx index 33d0749a..c15f697d 100644 --- a/frontend/src/components/layout-wrapper.tsx +++ b/frontend/src/components/layout-wrapper.tsx @@ -145,7 +145,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
{children} From 088ed2697b5eadf5802cf8b39e2e8c71c93c03e1 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 12:00:13 -0600 Subject: [PATCH 20/55] updates from sync --- frontend/src/app/knowledge/chunks/page.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index bbb7f019..85a4e3f2 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -161,7 +161,7 @@ function ChunksPageContent() {
-
+
{selectedFilter?.name && (
setQueryInputText(e.target.value)} value={queryInputText} />
-
+ {/*
Select all -
+
*/}
@@ -269,6 +269,10 @@ function ChunksPageContent() {
+ + {chunk.score.toFixed(2)} score + + {/* TODO: Update to use active toggle */} {/* Date: Fri, 3 Oct 2025 12:12:47 -0600 Subject: [PATCH 21/55] remove checkboxes --- frontend/src/app/knowledge/chunks/page.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index 85a4e3f2..cde2c3e8 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -240,14 +240,14 @@ function ChunksPageContent() { >
-
+ {/*
handleChunkCardCheckboxChange(index) } /> -
+
*/} Chunk {chunk.index} @@ -282,7 +282,7 @@ function ChunksPageContent() { Active */}
-
+
{chunk.text}
From 589839fedc1199e6060e0b57e9bd9ee08d8081b1 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 15:34:54 -0300 Subject: [PATCH 22/55] implement duplicate handling on backend and frontend --- .../components/duplicate-handling-dialog.tsx | 71 ++++++ frontend/components/knowledge-dropdown.tsx | 233 ++++++++++++------ frontend/src/contexts/task-context.tsx | 3 +- src/api/documents.py | 92 +++++-- src/api/router.py | 13 +- src/main.py | 11 + src/models/processors.py | 139 +++++++++-- src/models/tasks.py | 3 +- src/services/task_service.py | 23 +- 9 files changed, 470 insertions(+), 118 deletions(-) create mode 100644 frontend/components/duplicate-handling-dialog.tsx diff --git a/frontend/components/duplicate-handling-dialog.tsx b/frontend/components/duplicate-handling-dialog.tsx new file mode 100644 index 00000000..2f92ea50 --- /dev/null +++ b/frontend/components/duplicate-handling-dialog.tsx @@ -0,0 +1,71 @@ +"use client"; + +import { RotateCcw } from "lucide-react"; +import type React from "react"; +import { Button } from "./ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "./ui/dialog"; + +interface DuplicateHandlingDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + filename: string; + onOverwrite: () => void | Promise; + isLoading?: boolean; +} + +export const DuplicateHandlingDialog: React.FC< + DuplicateHandlingDialogProps +> = ({ open, onOpenChange, filename, onOverwrite, isLoading = false }) => { + const handleOverwrite = async () => { + try { + await onOverwrite(); + onOpenChange(false); + } catch (error) { + // Error handling is done by the parent component + } + }; + + return ( + + + + Overwrite document + + Overwriting will replace the existing document with another version. + This can't be undone. + + + + + + + + + + ); +}; diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index 9b71ee81..0b106360 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -13,6 +13,7 @@ import { useRouter } from "next/navigation"; import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery"; +import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -41,6 +42,7 @@ export function KnowledgeDropdown({ const [isOpen, setIsOpen] = useState(false); const [showFolderDialog, setShowFolderDialog] = useState(false); const [showS3Dialog, setShowS3Dialog] = useState(false); + const [showDuplicateDialog, setShowDuplicateDialog] = useState(false); const [awsEnabled, setAwsEnabled] = useState(false); const [folderPath, setFolderPath] = useState("/app/documents/"); const [bucketUrl, setBucketUrl] = useState("s3://"); @@ -48,6 +50,8 @@ export function KnowledgeDropdown({ const [s3Loading, setS3Loading] = useState(false); const [fileUploading, setFileUploading] = useState(false); const [isNavigatingToCloud, setIsNavigatingToCloud] = useState(false); + const [pendingFile, setPendingFile] = useState(null); + const [duplicateFilename, setDuplicateFilename] = useState(""); const [cloudConnectors, setCloudConnectors] = useState<{ [key: string]: { name: string; @@ -168,101 +172,52 @@ export function KnowledgeDropdown({ const handleFileChange = async (e: React.ChangeEvent) => { const files = e.target.files; if (files && files.length > 0) { - // Close dropdown and disable button immediately after file selection - setIsOpen(false); - setFileUploading(true); + const file = files[0]; - // Trigger the same file upload event as the chat page - window.dispatchEvent( - new CustomEvent("fileUploadStart", { - detail: { filename: files[0].name }, - }), - ); + // Close dropdown immediately after file selection + setIsOpen(false); try { - const formData = new FormData(); - formData.append("file", files[0]); + // Check if filename already exists (using ORIGINAL filename) + console.log("[Duplicate Check] Checking file:", file.name); + const checkResponse = await fetch( + `/api/documents/check-filename?filename=${encodeURIComponent(file.name)}`, + ); - // Use router upload and ingest endpoint (automatically routes based on configuration) - const uploadIngestRes = await fetch("/api/router/upload_ingest", { - method: "POST", - body: formData, - }); + console.log("[Duplicate Check] Response status:", checkResponse.status); - const uploadIngestJson = await uploadIngestRes.json(); - - if (!uploadIngestRes.ok) { + if (!checkResponse.ok) { + const errorText = await checkResponse.text(); + console.error("[Duplicate Check] Error response:", errorText); throw new Error( - uploadIngestJson?.error || "Upload and ingest failed", + `Failed to check duplicates: ${checkResponse.statusText}`, ); } - // Extract results from the response - handle both unified and simple formats - const fileId = uploadIngestJson?.upload?.id || uploadIngestJson?.id || uploadIngestJson?.task_id; - const filePath = - uploadIngestJson?.upload?.path || - uploadIngestJson?.path || - "uploaded"; - const runJson = uploadIngestJson?.ingestion; - const deleteResult = uploadIngestJson?.deletion; - console.log("c", uploadIngestJson ) - if (!fileId) { - throw new Error("Upload successful but no file id returned"); - } - // Check if ingestion actually succeeded - if ( - runJson && - runJson.status !== "COMPLETED" && - runJson.status !== "SUCCESS" - ) { - const errorMsg = runJson.error || "Ingestion pipeline failed"; - throw new Error( - `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.`, - ); - } - // Log deletion status if provided - if (deleteResult) { - if (deleteResult.status === "deleted") { - console.log( - "File successfully cleaned up from Langflow:", - deleteResult.file_id, - ); - } else if (deleteResult.status === "delete_failed") { - console.warn( - "Failed to cleanup file from Langflow:", - deleteResult.error, - ); + const checkData = await checkResponse.json(); + console.log("[Duplicate Check] Result:", checkData); + + if (checkData.exists) { + // Show duplicate handling dialog + console.log("[Duplicate Check] Duplicate detected, showing dialog"); + setPendingFile(file); + setDuplicateFilename(file.name); + setShowDuplicateDialog(true); + // Reset file input + if (fileInputRef.current) { + fileInputRef.current.value = ""; } + return; } - // Notify UI - window.dispatchEvent( - new CustomEvent("fileUploaded", { - detail: { - file: files[0], - result: { - file_id: fileId, - file_path: filePath, - run: runJson, - deletion: deleteResult, - unified: true, - }, - }, - }), - ); - refetchTasks(); + // No duplicate, proceed with upload + console.log("[Duplicate Check] No duplicate, proceeding with upload"); + await uploadFile(file, false); } catch (error) { - window.dispatchEvent( - new CustomEvent("fileUploadError", { - detail: { - filename: files[0].name, - error: error instanceof Error ? error.message : "Upload failed", - }, - }), - ); - } finally { - window.dispatchEvent(new CustomEvent("fileUploadComplete")); - setFileUploading(false); + console.error("[Duplicate Check] Exception:", error); + toast.error("Failed to check for duplicates", { + description: error instanceof Error ? error.message : "Unknown error", + }); } } @@ -272,6 +227,111 @@ export function KnowledgeDropdown({ } }; + const uploadFile = async (file: File, replace: boolean) => { + setFileUploading(true); + + // Trigger the same file upload event as the chat page + window.dispatchEvent( + new CustomEvent("fileUploadStart", { + detail: { filename: file.name }, + }), + ); + + try { + const formData = new FormData(); + formData.append("file", file); + formData.append("replace_duplicates", replace.toString()); + + // Use router upload and ingest endpoint (automatically routes based on configuration) + const uploadIngestRes = await fetch("/api/router/upload_ingest", { + method: "POST", + body: formData, + }); + + const uploadIngestJson = await uploadIngestRes.json(); + + if (!uploadIngestRes.ok) { + throw new Error(uploadIngestJson?.error || "Upload and ingest failed"); + } + + // Extract results from the response - handle both unified and simple formats + const fileId = + uploadIngestJson?.upload?.id || + uploadIngestJson?.id || + uploadIngestJson?.task_id; + const filePath = + uploadIngestJson?.upload?.path || uploadIngestJson?.path || "uploaded"; + const runJson = uploadIngestJson?.ingestion; + const deleteResult = uploadIngestJson?.deletion; + console.log("c", uploadIngestJson); + if (!fileId) { + throw new Error("Upload successful but no file id returned"); + } + // Check if ingestion actually succeeded + if ( + runJson && + runJson.status !== "COMPLETED" && + runJson.status !== "SUCCESS" + ) { + const errorMsg = runJson.error || "Ingestion pipeline failed"; + throw new Error( + `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.`, + ); + } + // Log deletion status if provided + if (deleteResult) { + if (deleteResult.status === "deleted") { + console.log( + "File successfully cleaned up from Langflow:", + deleteResult.file_id, + ); + } else if (deleteResult.status === "delete_failed") { + console.warn( + "Failed to cleanup file from Langflow:", + deleteResult.error, + ); + } + } + // Notify UI + window.dispatchEvent( + new CustomEvent("fileUploaded", { + detail: { + file: file, + result: { + file_id: fileId, + file_path: filePath, + run: runJson, + deletion: deleteResult, + unified: true, + }, + }, + }), + ); + + refetchTasks(); + } catch (error) { + window.dispatchEvent( + new CustomEvent("fileUploadError", { + detail: { + filename: file.name, + error: error instanceof Error ? error.message : "Upload failed", + }, + }), + ); + } finally { + window.dispatchEvent(new CustomEvent("fileUploadComplete")); + setFileUploading(false); + } + }; + + const handleOverwriteFile = async () => { + if (pendingFile) { + await uploadFile(pendingFile, true); + setPendingFile(null); + setDuplicateFilename(""); + } + }; + const handleFolderUpload = async () => { if (!folderPath.trim()) return; @@ -611,6 +671,15 @@ export function KnowledgeDropdown({
+ + {/* Duplicate Handling Dialog */} + ); } diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index 8928ae8f..26e8ca00 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -135,7 +135,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { taskFileEntries.forEach(([filePath, fileInfo]) => { if (typeof fileInfo === "object" && fileInfo) { - const fileName = filePath.split("/").pop() || filePath; + // Use the filename from backend if available, otherwise extract from path + const fileName = (fileInfo as any).filename || filePath.split("/").pop() || filePath; const fileStatus = fileInfo.status as string; // Map backend file status to our TaskFile status diff --git a/src/api/documents.py b/src/api/documents.py index 82afb349..a367c9fe 100644 --- a/src/api/documents.py +++ b/src/api/documents.py @@ -6,14 +6,13 @@ from config.settings import INDEX_NAME logger = get_logger(__name__) -async def delete_documents_by_filename(request: Request, document_service, session_manager): - """Delete all documents with a specific filename""" - data = await request.json() - filename = data.get("filename") - +async def check_filename_exists(request: Request, document_service, session_manager): + """Check if a document with a specific filename already exists""" + filename = request.query_params.get("filename") + if not filename: - return JSONResponse({"error": "filename is required"}, status_code=400) - + return JSONResponse({"error": "filename parameter is required"}, status_code=400) + user = request.state.user jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token) @@ -22,34 +21,99 @@ async def delete_documents_by_filename(request: Request, document_service, sessi opensearch_client = session_manager.get_user_opensearch_client( user.user_id, jwt_token ) - + + # Search for any document with this exact filename + # Try both .keyword (exact match) and regular field (analyzed match) + search_body = { + "query": { + "bool": { + "should": [ + {"term": {"filename.keyword": filename}}, + {"term": {"filename": filename}} + ], + "minimum_should_match": 1 + } + }, + "size": 1, + "_source": ["filename"] + } + + logger.debug(f"Checking filename existence: {filename}") + + response = await opensearch_client.search( + index=INDEX_NAME, + body=search_body + ) + + # Check if any hits were found + hits = response.get("hits", {}).get("hits", []) + exists = len(hits) > 0 + + logger.debug(f"Filename check result - exists: {exists}, hits: {len(hits)}") + + return JSONResponse({ + "exists": exists, + "filename": filename + }, status_code=200) + + except Exception as e: + logger.error("Error checking filename existence", filename=filename, error=str(e)) + error_str = str(e) + if "AuthenticationException" in error_str: + return JSONResponse({"error": "Access denied: insufficient permissions"}, status_code=403) + else: + return JSONResponse({"error": str(e)}, status_code=500) + + +async def delete_documents_by_filename(request: Request, document_service, session_manager): + """Delete all documents with a specific filename""" + data = await request.json() + filename = data.get("filename") + + if not filename: + return JSONResponse({"error": "filename is required"}, status_code=400) + + user = request.state.user + jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token) + + try: + # Get user's OpenSearch client + opensearch_client = session_manager.get_user_opensearch_client( + user.user_id, jwt_token + ) + # Delete by query to remove all chunks of this document + # Use both .keyword and regular field to ensure we catch all variations delete_query = { "query": { "bool": { - "must": [ + "should": [ + {"term": {"filename.keyword": filename}}, {"term": {"filename": filename}} - ] + ], + "minimum_should_match": 1 } } } - + + logger.debug(f"Deleting documents with filename: {filename}") + result = await opensearch_client.delete_by_query( index=INDEX_NAME, body=delete_query, conflicts="proceed" ) - + deleted_count = result.get("deleted", 0) logger.info(f"Deleted {deleted_count} chunks for filename {filename}", user_id=user.user_id) - + return JSONResponse({ "success": True, "deleted_chunks": deleted_count, "filename": filename, "message": f"All documents with filename '{filename}' deleted successfully" }, status_code=200) - + except Exception as e: logger.error("Error deleting documents by filename", filename=filename, error=str(e)) error_str = str(e) diff --git a/src/api/router.py b/src/api/router.py index 23ce5bdf..327757be 100644 --- a/src/api/router.py +++ b/src/api/router.py @@ -77,6 +77,7 @@ async def langflow_upload_ingest_task( settings_json = form.get("settings") tweaks_json = form.get("tweaks") delete_after_ingest = form.get("delete_after_ingest", "true").lower() == "true" + replace_duplicates = form.get("replace_duplicates", "false").lower() == "true" # Parse JSON fields if provided settings = None @@ -112,7 +113,8 @@ async def langflow_upload_ingest_task( import tempfile import os temp_file_paths = [] - + original_filenames = [] + try: # Create temp directory reference once temp_dir = tempfile.gettempdir() @@ -121,8 +123,11 @@ async def langflow_upload_ingest_task( # Read file content content = await upload_file.read() - # Create temporary file with the actual filename (not a temp prefix) - # Store in temp directory but use the real filename + # Store ORIGINAL filename (not transformed) + original_filenames.append(upload_file.filename) + + # Create temporary file with TRANSFORMED filename for filesystem safety + # Transform: spaces and / to underscore safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_") temp_path = os.path.join(temp_dir, safe_filename) @@ -153,6 +158,7 @@ async def langflow_upload_ingest_task( task_id = await task_service.create_langflow_upload_task( user_id=user_id, file_paths=temp_file_paths, + original_filenames=original_filenames, langflow_file_service=langflow_file_service, session_manager=session_manager, jwt_token=jwt_token, @@ -162,6 +168,7 @@ async def langflow_upload_ingest_task( tweaks=tweaks, settings=settings, delete_after_ingest=delete_after_ingest, + replace_duplicates=replace_duplicates, ) logger.debug("Langflow upload task created successfully", task_id=task_id) diff --git a/src/main.py b/src/main.py index 230ded79..bf6da342 100644 --- a/src/main.py +++ b/src/main.py @@ -953,6 +953,17 @@ async def create_app(): methods=["POST", "GET"], ), # Document endpoints + Route( + "/documents/check-filename", + require_auth(services["session_manager"])( + partial( + documents.check_filename_exists, + document_service=services["document_service"], + session_manager=services["session_manager"], + ) + ), + methods=["GET"], + ), Route( "/documents/delete-by-filename", require_auth(services["session_manager"])( diff --git a/src/models/processors.py b/src/models/processors.py index a1d72777..6d7b74b4 100644 --- a/src/models/processors.py +++ b/src/models/processors.py @@ -55,6 +55,108 @@ class TaskProcessor: await asyncio.sleep(retry_delay) retry_delay *= 2 # Exponential backoff + async def check_filename_exists( + self, + filename: str, + opensearch_client, + ) -> bool: + """ + Check if a document with the given filename already exists in OpenSearch. + Returns True if any chunks with this filename exist. + """ + from config.settings import INDEX_NAME + import asyncio + + max_retries = 3 + retry_delay = 1.0 + + for attempt in range(max_retries): + try: + # Search for any document with this exact filename + search_body = { + "query": { + "term": { + "filename.keyword": filename + } + }, + "size": 1, + "_source": False + } + + response = await opensearch_client.search( + index=INDEX_NAME, + body=search_body + ) + + # Check if any hits were found + hits = response.get("hits", {}).get("hits", []) + return len(hits) > 0 + + except (asyncio.TimeoutError, Exception) as e: + if attempt == max_retries - 1: + logger.error( + "OpenSearch filename check failed after retries", + filename=filename, + error=str(e), + attempt=attempt + 1 + ) + # On final failure, assume document doesn't exist (safer to reprocess than skip) + logger.warning( + "Assuming filename doesn't exist due to connection issues", + filename=filename + ) + return False + else: + logger.warning( + "OpenSearch filename check failed, retrying", + filename=filename, + error=str(e), + attempt=attempt + 1, + retry_in=retry_delay + ) + await asyncio.sleep(retry_delay) + retry_delay *= 2 # Exponential backoff + + async def delete_document_by_filename( + self, + filename: str, + opensearch_client, + ) -> None: + """ + Delete all chunks of a document with the given filename from OpenSearch. + """ + from config.settings import INDEX_NAME + + try: + # Delete all documents with this filename + delete_body = { + "query": { + "term": { + "filename.keyword": filename + } + } + } + + response = await opensearch_client.delete_by_query( + index=INDEX_NAME, + body=delete_body + ) + + deleted_count = response.get("deleted", 0) + logger.info( + "Deleted existing document chunks", + filename=filename, + deleted_count=deleted_count + ) + + except Exception as e: + logger.error( + "Failed to delete existing document", + filename=filename, + error=str(e) + ) + raise + async def process_document_standard( self, file_path: str, @@ -527,6 +629,7 @@ class LangflowFileProcessor(TaskProcessor): tweaks: dict = None, settings: dict = None, delete_after_ingest: bool = True, + replace_duplicates: bool = False, ): super().__init__() self.langflow_file_service = langflow_file_service @@ -539,6 +642,7 @@ class LangflowFileProcessor(TaskProcessor): self.tweaks = tweaks or {} self.settings = settings self.delete_after_ingest = delete_after_ingest + self.replace_duplicates = replace_duplicates async def process_item( self, upload_task: UploadTask, item: str, file_task: FileTask @@ -554,33 +658,40 @@ class LangflowFileProcessor(TaskProcessor): file_task.updated_at = time.time() try: - # Compute hash and check if already exists - from utils.hash_utils import hash_id - file_hash = hash_id(item) + # Use the ORIGINAL filename stored in file_task (not the transformed temp path) + # This ensures we check/store the original filename with spaces, etc. + original_filename = file_task.filename or os.path.basename(item) - # Check if document already exists + # Check if document with same filename already exists opensearch_client = self.session_manager.get_user_opensearch_client( self.owner_user_id, self.jwt_token ) - if await self.check_document_exists(file_hash, opensearch_client): - file_task.status = TaskStatus.COMPLETED - file_task.result = {"status": "unchanged", "id": file_hash} + + filename_exists = await self.check_filename_exists(original_filename, opensearch_client) + + if filename_exists and not self.replace_duplicates: + # Duplicate exists and user hasn't confirmed replacement + file_task.status = TaskStatus.FAILED + file_task.error = f"File with name '{original_filename}' already exists" file_task.updated_at = time.time() - upload_task.successful_files += 1 + upload_task.failed_files += 1 return + elif filename_exists and self.replace_duplicates: + # Delete existing document before uploading new one + logger.info(f"Replacing existing document: {original_filename}") + await self.delete_document_by_filename(original_filename, opensearch_client) # Read file content for processing with open(item, 'rb') as f: content = f.read() - # Create file tuple for upload - # The temp file now has the actual filename, no need to extract it - filename = os.path.basename(item) - content_type, _ = mimetypes.guess_type(filename) + # Create file tuple for upload using ORIGINAL filename + # This ensures the document is indexed with the original name + content_type, _ = mimetypes.guess_type(original_filename) if not content_type: content_type = 'application/octet-stream' - - file_tuple = (filename, content, content_type) + + file_tuple = (original_filename, content, content_type) # Get JWT token using same logic as DocumentFileProcessor # This will handle anonymous JWT creation if needed diff --git a/src/models/tasks.py b/src/models/tasks.py index 236927ab..253cabb5 100644 --- a/src/models/tasks.py +++ b/src/models/tasks.py @@ -20,7 +20,8 @@ class FileTask: retry_count: int = 0 created_at: float = field(default_factory=time.time) updated_at: float = field(default_factory=time.time) - + filename: Optional[str] = None # Original filename for display + @property def duration_seconds(self) -> float: """Duration in seconds from creation to last update""" diff --git a/src/services/task_service.py b/src/services/task_service.py index be5312a0..eb5825c0 100644 --- a/src/services/task_service.py +++ b/src/services/task_service.py @@ -59,6 +59,7 @@ class TaskService: file_paths: list, langflow_file_service, session_manager, + original_filenames: list = None, jwt_token: str = None, owner_name: str = None, owner_email: str = None, @@ -66,6 +67,7 @@ class TaskService: tweaks: dict = None, settings: dict = None, delete_after_ingest: bool = True, + replace_duplicates: bool = False, ) -> str: """Create a new upload task for Langflow file processing with upload and ingest""" # Use LangflowFileProcessor with user context @@ -82,18 +84,31 @@ class TaskService: tweaks=tweaks, settings=settings, delete_after_ingest=delete_after_ingest, + replace_duplicates=replace_duplicates, ) - return await self.create_custom_task(user_id, file_paths, processor) + return await self.create_custom_task(user_id, file_paths, processor, original_filenames) - async def create_custom_task(self, user_id: str, items: list, processor) -> str: + async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: list = None) -> str: """Create a new task with custom processor for any type of items""" + import os # Store anonymous tasks under a stable key so they can be retrieved later store_user_id = user_id or AnonymousUser().user_id task_id = str(uuid.uuid4()) + + # Create file tasks with original filenames if provided + file_tasks = {} + for i, item in enumerate(items): + if original_filenames and i < len(original_filenames): + filename = original_filenames[i] + else: + filename = os.path.basename(str(item)) + + file_tasks[str(item)] = FileTask(file_path=str(item), filename=filename) + upload_task = UploadTask( task_id=task_id, total_files=len(items), - file_tasks={str(item): FileTask(file_path=str(item)) for item in items}, + file_tasks=file_tasks, ) # Attach the custom processor to the task @@ -268,6 +283,7 @@ class TaskService: "created_at": file_task.created_at, "updated_at": file_task.updated_at, "duration_seconds": file_task.duration_seconds, + "filename": file_task.filename, } # Count running and pending files @@ -322,6 +338,7 @@ class TaskService: "created_at": file_task.created_at, "updated_at": file_task.updated_at, "duration_seconds": file_task.duration_seconds, + "filename": file_task.filename, } if file_task.status.value == "running": From fb29f72598aad139e4c0623d861094a18e7b8fba Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 15:39:51 -0300 Subject: [PATCH 23/55] update task toast --- frontend/src/contexts/task-context.tsx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index 26e8ca00..d25130f7 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -205,9 +205,19 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { previousTask && previousTask.status !== "completed" && currentTask.status === "completed" ) { - // Task just completed - show success toast - toast.success("Task completed successfully", { - description: `Task ${currentTask.task_id} has finished processing.`, + // Task just completed - show success toast with file counts + const successfulFiles = currentTask.successful_files || 0; + const failedFiles = currentTask.failed_files || 0; + + let description = ""; + if (failedFiles > 0) { + description = `${successfulFiles} file${successfulFiles !== 1 ? 's' : ''} uploaded successfully, ${failedFiles} file${failedFiles !== 1 ? 's' : ''} failed`; + } else { + description = `${successfulFiles} file${successfulFiles !== 1 ? 's' : ''} uploaded successfully`; + } + + toast.success("Task completed", { + description, action: { label: "View", onClick: () => console.log("View task", currentTask.task_id), From 9336aa287e87032bd7385e3824e548df5de26f0e Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:12:09 -0300 Subject: [PATCH 24/55] fixed lint --- frontend/src/app/knowledge/page.tsx | 110 ++++++++++++++++++---------- 1 file changed, 71 insertions(+), 39 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index c6d254c4..9e933903 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -1,10 +1,15 @@ "use client"; -import type { ColDef } from "ag-grid-community"; +import type { ColDef, GetRowIdParams } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; import { Building2, Cloud, HardDrive, Search, Trash2, X } from "lucide-react"; import { useRouter } from "next/navigation"; -import { type ChangeEvent, useCallback, useRef, useState } from "react"; +import { + type ChangeEvent, + useCallback, + useRef, + useState, +} from "react"; import { SiGoogledrive } from "react-icons/si"; import { TbBrandOnedrive } from "react-icons/tb"; import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; @@ -54,15 +59,10 @@ function SearchPage() { const deleteDocumentMutation = useDeleteDocument(); - const { data = [], isFetching } = useGetSearchQuery( + const { data: searchData = [], isFetching } = useGetSearchQuery( parsedFilterData?.query || "*", parsedFilterData, ); - - const handleTableSearch = (e: ChangeEvent) => { - gridRef.current?.api.setGridOption("quickFilterText", e.target.value); - }; - // Convert TaskFiles to File format and merge with backend results const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { return { @@ -75,60 +75,91 @@ function SearchPage() { }; }); - const backendFiles = (data as File[]).filter((file) => !taskFilesAsFiles.some((taskFile) => taskFile.filename === file.filename && taskFile.status === "processing")); + // Create a map of task files by filename for quick lookup + const taskFileMap = new Map( + taskFilesAsFiles.map((file) => [file.filename, file]), + ); + + // Override backend files with task file status if they exist + const backendFiles = (searchData as File[]) + .map((file) => { + const taskFile = taskFileMap.get(file.filename); + if (taskFile) { + // Override backend file with task file data (includes status) + return { ...file, ...taskFile }; + } + return file; + }) + .filter((file) => { + // Only filter out files that are currently processing AND in taskFiles + const taskFile = taskFileMap.get(file.filename); + return !taskFile || taskFile.status !== "processing"; + }); const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { return ( taskFile.status !== "active" && !backendFiles.some( - (backendFile) => backendFile.filename === taskFile.filename,) + (backendFile) => backendFile.filename === taskFile.filename, + ) ); }); // Combine task files first, then backend files const fileResults = [...backendFiles, ...filteredTaskFiles]; + const handleTableSearch = (e: ChangeEvent) => { + gridRef.current?.api.setGridOption("quickFilterText", e.target.value); + }; + const gridRef = useRef(null); const columnDefs = [ { field: "filename", headerName: "Source", - checkboxSelection: (data) => (data?.data?.status || "active") === "active", + checkboxSelection: (params: CustomCellRendererProps) => + (params?.data?.status || "active") === "active", headerCheckboxSelection: true, initialFlex: 2, minWidth: 220, cellRenderer: ({ data, value }: CustomCellRendererProps) => { + // Read status directly from data on each render + const status = data?.status || "active"; + const isActive = status === "active"; + console.log(data?.filename, status, "a"); return ( -
{((data?.status || "active") !== "active") && -
- } -
+
+
+ +
); }, }, { field: "size", headerName: "Size", - valueFormatter: (params) => + valueFormatter: (params: CustomCellRendererProps) => params.value ? `${Math.round(params.value / 1024)} KB` : "-", }, { @@ -138,13 +169,13 @@ function SearchPage() { { field: "owner", headerName: "Owner", - valueFormatter: (params) => + valueFormatter: (params: CustomCellRendererProps) => params.data?.owner_name || params.data?.owner_email || "—", }, { field: "chunkCount", headerName: "Chunks", - valueFormatter: (params) => params.data?.chunkCount?.toString() || "-", + valueFormatter: (params: CustomCellRendererProps) => params.data?.chunkCount?.toString() || "-", }, { field: "avgScore", @@ -162,6 +193,7 @@ function SearchPage() { field: "status", headerName: "Status", cellRenderer: ({ data }: CustomCellRendererProps) => { + console.log(data?.filename, data?.status, "b"); // Default to 'active' status if no status is provided const status = data?.status || "active"; return ; @@ -188,7 +220,7 @@ function SearchPage() { resizable: false, sortable: false, initialFlex: 0, - } + }, ]; const defaultColDef: ColDef = { @@ -323,7 +355,7 @@ function SearchPage() {
[]} defaultColDef={defaultColDef} loading={isFetching} ref={gridRef} @@ -331,7 +363,7 @@ function SearchPage() { rowSelection="multiple" rowMultiSelectWithClick={false} suppressRowClickSelection={true} - getRowId={(params) => params.data.filename} + getRowId={(params: GetRowIdParams) => params.data?.filename} domLayout="normal" onSelectionChanged={onSelectionChanged} noRowsOverlayComponent={() => ( From ca9bee8222bd615501c90e8e7f01913243c3d08d Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:12:18 -0300 Subject: [PATCH 25/55] removed file from query when overwriting --- frontend/components/knowledge-dropdown.tsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index 0b106360..7fe84259 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -1,5 +1,6 @@ "use client"; +import { useQueryClient } from "@tanstack/react-query"; import { ChevronDown, Cloud, @@ -26,6 +27,7 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { useTask } from "@/contexts/task-context"; import { cn } from "@/lib/utils"; +import type { File as SearchFile } from "@/src/app/api/queries/useGetSearchQuery"; interface KnowledgeDropdownProps { active?: boolean; @@ -38,6 +40,7 @@ export function KnowledgeDropdown({ }: KnowledgeDropdownProps) { const { addTask } = useTask(); const { refetch: refetchTasks } = useGetTasksQuery(); + const queryClient = useQueryClient(); const router = useRouter(); const [isOpen, setIsOpen] = useState(false); const [showFolderDialog, setShowFolderDialog] = useState(false); @@ -326,6 +329,15 @@ export function KnowledgeDropdown({ const handleOverwriteFile = async () => { if (pendingFile) { + // Remove the old file from all search query caches before overwriting + queryClient.setQueriesData({ queryKey: ["search"] }, (oldData: []) => { + if (!oldData) return oldData; + // Filter out the file that's being overwritten + return oldData.filter( + (file: SearchFile) => file.filename !== pendingFile.name, + ); + }); + await uploadFile(pendingFile, true); setPendingFile(null); setDuplicateFilename(""); @@ -676,7 +688,6 @@ export function KnowledgeDropdown({ From 4de9a0d085848abbab76db05b9f06c9d3d81198f Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:12:24 -0300 Subject: [PATCH 26/55] removed unused prop --- frontend/components/duplicate-handling-dialog.tsx | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/frontend/components/duplicate-handling-dialog.tsx b/frontend/components/duplicate-handling-dialog.tsx index 2f92ea50..d5cb2edf 100644 --- a/frontend/components/duplicate-handling-dialog.tsx +++ b/frontend/components/duplicate-handling-dialog.tsx @@ -15,21 +15,16 @@ import { interface DuplicateHandlingDialogProps { open: boolean; onOpenChange: (open: boolean) => void; - filename: string; onOverwrite: () => void | Promise; isLoading?: boolean; } export const DuplicateHandlingDialog: React.FC< DuplicateHandlingDialogProps -> = ({ open, onOpenChange, filename, onOverwrite, isLoading = false }) => { +> = ({ open, onOpenChange, onOverwrite, isLoading = false }) => { const handleOverwrite = async () => { - try { - await onOverwrite(); - onOpenChange(false); - } catch (error) { - // Error handling is done by the parent component - } + await onOverwrite(); + onOpenChange(false); }; return ( From 1fdb251a47a7168f9431e00c8f714a9e088db09a Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:15:32 -0300 Subject: [PATCH 27/55] UI minor tweaks --- frontend/components/ui/input.tsx | 2 +- frontend/src/app/knowledge/chunks/page.tsx | 626 +++++++++++---------- frontend/src/app/knowledge/page.tsx | 3 +- 3 files changed, 316 insertions(+), 315 deletions(-) diff --git a/frontend/components/ui/input.tsx b/frontend/components/ui/input.tsx index ffcda454..86e638a1 100644 --- a/frontend/components/ui/input.tsx +++ b/frontend/components/ui/input.tsx @@ -44,7 +44,7 @@ const Input = React.forwardRef( placeholder={placeholder} className={cn( "primary-input", - icon && "pl-9", + icon && "!pl-9", type === "password" && "!pr-8", icon ? inputClassName : className )} diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index cb96eddc..7d49a56e 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -1,289 +1,291 @@ "use client"; import { ArrowLeft, Check, Copy, Loader2, Search } from "lucide-react"; -import { Suspense, useCallback, useEffect, useMemo, useState } from "react"; import { useRouter, useSearchParams } from "next/navigation"; +import { Suspense, useCallback, useEffect, useMemo, useState } from "react"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; import { useTask } from "@/contexts/task-context"; import { - type ChunkResult, - type File, - useGetSearchQuery, + type ChunkResult, + type File, + useGetSearchQuery, } from "../../api/queries/useGetSearchQuery"; -import { Label } from "@/components/ui/label"; -import { Checkbox } from "@/components/ui/checkbox"; -import { Input } from "@/components/ui/input"; const getFileTypeLabel = (mimetype: string) => { - if (mimetype === "application/pdf") return "PDF"; - if (mimetype === "text/plain") return "Text"; - if (mimetype === "application/msword") return "Word Document"; - return "Unknown"; + if (mimetype === "application/pdf") return "PDF"; + if (mimetype === "text/plain") return "Text"; + if (mimetype === "application/msword") return "Word Document"; + return "Unknown"; }; function ChunksPageContent() { - const router = useRouter(); - const searchParams = useSearchParams(); - const { isMenuOpen } = useTask(); - const { parsedFilterData, isPanelOpen } = useKnowledgeFilter(); + const router = useRouter(); + const searchParams = useSearchParams(); + const { isMenuOpen } = useTask(); + const { parsedFilterData, isPanelOpen } = useKnowledgeFilter(); - const filename = searchParams.get("filename"); - const [chunks, setChunks] = useState([]); - const [chunksFilteredByQuery, setChunksFilteredByQuery] = useState< - ChunkResult[] - >([]); - const [selectedChunks, setSelectedChunks] = useState>(new Set()); - const [activeCopiedChunkIndex, setActiveCopiedChunkIndex] = useState< - number | null - >(null); + const filename = searchParams.get("filename"); + const [chunks, setChunks] = useState([]); + const [chunksFilteredByQuery, setChunksFilteredByQuery] = useState< + ChunkResult[] + >([]); + const [selectedChunks, setSelectedChunks] = useState>(new Set()); + const [activeCopiedChunkIndex, setActiveCopiedChunkIndex] = useState< + number | null + >(null); - // Calculate average chunk length - const averageChunkLength = useMemo( - () => - chunks.reduce((acc, chunk) => acc + chunk.text.length, 0) / - chunks.length || 0, - [chunks] - ); + // Calculate average chunk length + const averageChunkLength = useMemo( + () => + chunks.reduce((acc, chunk) => acc + chunk.text.length, 0) / + chunks.length || 0, + [chunks], + ); - const [selectAll, setSelectAll] = useState(false); - const [queryInputText, setQueryInputText] = useState( - parsedFilterData?.query ?? "" - ); + const [selectAll, setSelectAll] = useState(false); + const [queryInputText, setQueryInputText] = useState( + parsedFilterData?.query ?? "", + ); - // Use the same search query as the knowledge page, but we'll filter for the specific file - const { data = [], isFetching } = useGetSearchQuery("*", parsedFilterData); + // Use the same search query as the knowledge page, but we'll filter for the specific file + const { data = [], isFetching } = useGetSearchQuery("*", parsedFilterData); - useEffect(() => { - if (queryInputText === "") { - setChunksFilteredByQuery(chunks); - } else { - setChunksFilteredByQuery( - chunks.filter((chunk) => - chunk.text.toLowerCase().includes(queryInputText.toLowerCase()) - ) - ); - } - }, [queryInputText, chunks]); + useEffect(() => { + if (queryInputText === "") { + setChunksFilteredByQuery(chunks); + } else { + setChunksFilteredByQuery( + chunks.filter((chunk) => + chunk.text.toLowerCase().includes(queryInputText.toLowerCase()), + ), + ); + } + }, [queryInputText, chunks]); - const handleCopy = useCallback((text: string, index: number) => { - // Trim whitespace and remove new lines/tabs for cleaner copy - navigator.clipboard.writeText(text.trim().replace(/[\n\r\t]/gm, "")); - setActiveCopiedChunkIndex(index); - setTimeout(() => setActiveCopiedChunkIndex(null), 10 * 1000); // 10 seconds - }, []); + const handleCopy = useCallback((text: string, index: number) => { + // Trim whitespace and remove new lines/tabs for cleaner copy + navigator.clipboard.writeText(text.trim().replace(/[\n\r\t]/gm, "")); + setActiveCopiedChunkIndex(index); + setTimeout(() => setActiveCopiedChunkIndex(null), 10 * 1000); // 10 seconds + }, []); - const fileData = (data as File[]).find( - (file: File) => file.filename === filename - ); + const fileData = (data as File[]).find( + (file: File) => file.filename === filename, + ); - // Extract chunks for the specific file - useEffect(() => { - if (!filename || !(data as File[]).length) { - setChunks([]); - return; - } + // Extract chunks for the specific file + useEffect(() => { + if (!filename || !(data as File[]).length) { + setChunks([]); + return; + } - setChunks(fileData?.chunks || []); - }, [data, filename]); + setChunks(fileData?.chunks || []); + }, [data, filename]); - // Set selected state for all checkboxes when selectAll changes - useEffect(() => { - if (selectAll) { - setSelectedChunks(new Set(chunks.map((_, index) => index))); - } else { - setSelectedChunks(new Set()); - } - }, [selectAll, setSelectedChunks, chunks]); + // Set selected state for all checkboxes when selectAll changes + useEffect(() => { + if (selectAll) { + setSelectedChunks(new Set(chunks.map((_, index) => index))); + } else { + setSelectedChunks(new Set()); + } + }, [selectAll, setSelectedChunks, chunks]); - const handleBack = useCallback(() => { - router.push("/knowledge"); - }, [router]); + const handleBack = useCallback(() => { + router.push("/knowledge"); + }, [router]); - const handleChunkCardCheckboxChange = useCallback( - (index: number) => { - setSelectedChunks((prevSelected) => { - const newSelected = new Set(prevSelected); - if (newSelected.has(index)) { - newSelected.delete(index); - } else { - newSelected.add(index); - } - return newSelected; - }); - }, - [setSelectedChunks] - ); + const handleChunkCardCheckboxChange = useCallback( + (index: number) => { + setSelectedChunks((prevSelected) => { + const newSelected = new Set(prevSelected); + if (newSelected.has(index)) { + newSelected.delete(index); + } else { + newSelected.add(index); + } + return newSelected; + }); + }, + [setSelectedChunks], + ); - if (!filename) { - return ( -
-
- -

No file specified

-

- Please select a file from the knowledge page -

-
-
- ); - } + if (!filename) { + return ( +
+
+ +

No file specified

+

+ Please select a file from the knowledge page +

+
+
+ ); + } - return ( -
-
- {/* Header */} -
-
- -

- {/* Removes file extension from filename */} - {filename.replace(/\.[^/.]+$/, "")} -

-
-
-
- : null} - id="search-query" - type="text" - defaultValue={parsedFilterData?.query} - value={queryInputText} - onChange={(e) => setQueryInputText(e.target.value)} - placeholder="Search chunks..." - /> -
-
- - setSelectAll(!!handleSelectAll) - } - /> - -
-
-
+ return ( +
+
+ {/* Header */} +
+
+ +

+ {/* Removes file extension from filename */} + {filename.replace(/\.[^/.]+$/, "")} +

+
+
+
+ : null + } + id="search-query" + type="text" + defaultValue={parsedFilterData?.query} + value={queryInputText} + onChange={(e) => setQueryInputText(e.target.value)} + placeholder="Search chunks..." + /> +
+
+ + setSelectAll(!!handleSelectAll) + } + /> + +
+
+
- {/* Content Area - matches knowledge page structure */} -
- {isFetching ? ( -
-
- -

- Loading chunks... -

-
-
- ) : chunks.length === 0 ? ( -
-
- -

No chunks found

-

- This file may not have been indexed yet -

-
-
- ) : ( -
- {chunksFilteredByQuery.map((chunk, index) => ( -
-
-
-
- - handleChunkCardCheckboxChange(index) - } - /> -
- - Chunk {chunk.page} - - - {chunk.text.length} chars - -
- -
-
+ {/* Content Area - matches knowledge page structure */} +
+ {isFetching ? ( +
+
+ +

+ Loading chunks... +

+
+
+ ) : chunks.length === 0 ? ( +
+
+ +

No chunks found

+

+ This file may not have been indexed yet +

+
+
+ ) : ( +
+ {chunksFilteredByQuery.map((chunk, index) => ( +
+
+
+
+ + handleChunkCardCheckboxChange(index) + } + /> +
+ + Chunk {chunk.page} + + + {chunk.text.length} chars + +
+ +
+
- {/* TODO: Update to use active toggle */} - {/* + {/* TODO: Update to use active toggle */} + {/* Active */} -
-
- {chunk.text} -
-
- ))} -
- )} -
-
- {/* Right panel - Summary (TODO), Technical details, */} -
-
-

Technical details

-
-
-
Total chunks
-
- {chunks.length} -
-
-
-
Avg length
-
- {averageChunkLength.toFixed(0)} chars -
-
- {/* TODO: Uncomment after data is available */} - {/*
+
+
+ {chunk.text} +
+
+ ))} +
+ )} +
+
+ {/* Right panel - Summary (TODO), Technical details, */} +
+
+

Technical details

+
+
+
Total chunks
+
+ {chunks.length} +
+
+
+
Avg length
+
+ {averageChunkLength.toFixed(0)} chars +
+
+ {/* TODO: Uncomment after data is available */} + {/*
Process time
@@ -293,76 +295,76 @@ function ChunksPageContent() {
*/} -
-
-
-

Original document

-
-
-
Name
-
- {fileData?.filename} -
-
-
-
Type
-
- {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} -
-
-
-
Size
-
- {fileData?.size - ? `${Math.round(fileData.size / 1024)} KB` - : "Unknown"} -
-
-
-
Uploaded
-
- N/A -
-
- {/* TODO: Uncomment after data is available */} - {/*
+
+
+
+

Original document

+
+
+
Name
+
+ {fileData?.filename} +
+
+
+
Type
+
+ {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} +
+
+
+
Size
+
+ {fileData?.size + ? `${Math.round(fileData.size / 1024)} KB` + : "Unknown"} +
+
+
+
Uploaded
+
+ N/A +
+
+ {/* TODO: Uncomment after data is available */} + {/*
Source
*/} -
-
Updated
-
- N/A -
-
-
-
-
-
- ); +
+
Updated
+
+ N/A +
+
+ +
+
+
+ ); } function ChunksPage() { - return ( - -
- -

Loading...

-
-
- } - > - - - ); + return ( + +
+ +

Loading...

+
+
+ } + > + + + ); } export default function ProtectedChunksPage() { - return ( - - - - ); + return ( + + + + ); } diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 9e933903..5395d85d 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -310,14 +310,13 @@ function SearchPage() { )}
From 99cc806a559f526e9e4a837c41096eef19fd3116 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 13:53:39 -0600 Subject: [PATCH 28/55] fix empty state --- frontend/src/app/knowledge/chunks/page.tsx | 103 +++++++++++---------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index cde2c3e8..31b741a4 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -224,10 +224,9 @@ function ChunksPageContent() { ) : chunks.length === 0 ? (
- -

No chunks found

-

- This file may not have been indexed yet +

No knowledge

+

+ Clear the knowledge filter or return to the knowledge page

@@ -292,24 +291,29 @@ function ChunksPageContent() {
{/* Right panel - Summary (TODO), Technical details, */} -
-
-

Technical details

-
-
-
Total chunks
-
- {chunks.length} -
-
-
-
Avg length
-
- {averageChunkLength.toFixed(0)} chars -
-
- {/* TODO: Uncomment after data is available */} - {/*
+ {chunks.length > 0 && ( +
+
+

+ Technical details +

+
+
+
+ Total chunks +
+
+ {chunks.length} +
+
+
+
Avg length
+
+ {averageChunkLength.toFixed(0)} chars +
+
+ {/* TODO: Uncomment after data is available */} + {/*
Process time
@@ -319,51 +323,54 @@ function ChunksPageContent() {
*/} -
-
-
-

Original document

-
- {/*
+
+
+
+

+ Original document +

+
+ {/*
Name
{fileData?.filename}
*/} -
-
Type
-
- {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} -
-
-
-
Size
-
- {fileData?.size - ? `${Math.round(fileData.size / 1024)} KB` - : "Unknown"} -
-
- {/*
+
+
Type
+
+ {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} +
+
+
+
Size
+
+ {fileData?.size + ? `${Math.round(fileData.size / 1024)} KB` + : "Unknown"} +
+
+ {/*
Uploaded
N/A
*/} - {/* TODO: Uncomment after data is available */} - {/*
+ {/* TODO: Uncomment after data is available */} + {/*
Source
*/} - {/*
+ {/*
Updated
N/A
*/} -
+
+
-
+ )}
); } From 946d3edc89d09af8fabd5e6b1981466d76fae7ee Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:54:40 -0300 Subject: [PATCH 29/55] refresh tasks on entering page, make failed files persist --- frontend/src/app/knowledge/page.tsx | 7 ++++++- frontend/src/contexts/task-context.tsx | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 5395d85d..73cc2c7d 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -7,6 +7,7 @@ import { useRouter } from "next/navigation"; import { type ChangeEvent, useCallback, + useEffect, useRef, useState, } from "react"; @@ -51,7 +52,7 @@ function getSourceIcon(connectorType?: string) { function SearchPage() { const router = useRouter(); - const { isMenuOpen, files: taskFiles } = useTask(); + const { isMenuOpen, files: taskFiles, refreshTasks } = useTask(); const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = useKnowledgeFilter(); const [selectedRows, setSelectedRows] = useState([]); @@ -59,6 +60,10 @@ function SearchPage() { const deleteDocumentMutation = useDeleteDocument(); + useEffect(() => { + refreshTasks(); + }, [refreshTasks]); + const { data: searchData = [], isFetching } = useGetSearchQuery( parsedFilterData?.query || "*", parsedFilterData, diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index d25130f7..69a1214f 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -226,7 +226,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { setTimeout(() => { refetchSearch(); setFiles((prevFiles) => - prevFiles.filter((file) => file.task_id !== currentTask.task_id && file.status !== "failed"), + prevFiles.filter((file) => file.task_id !== currentTask.task_id || file.status === "failed"), ); }, 500); } else if ( @@ -258,6 +258,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { ); const refreshTasks = useCallback(async () => { + setFiles([]); await refetchTasks(); }, [refetchTasks]); From 509b6c613228fa5ec592533d6a5e69b8bfa76aa2 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 17:06:07 -0300 Subject: [PATCH 30/55] make view button open menu --- frontend/src/contexts/task-context.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index 69a1214f..a8693526 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -220,7 +220,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { description, action: { label: "View", - onClick: () => console.log("View task", currentTask.task_id), + onClick: () => setIsMenuOpen(true), }, }); setTimeout(() => { @@ -252,7 +252,9 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { (_taskId: string) => { // React Query will automatically handle polling when tasks are active // Just trigger a refetch to get the latest data - refetchTasks(); + setTimeout(() => { + refetchTasks(); + }, 500); }, [refetchTasks], ); From 7201a914be856775a0bafc7ef445a8882766d3ae Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 17:12:36 -0300 Subject: [PATCH 31/55] open tasks menu when clicking view --- frontend/src/components/task-notification-menu.tsx | 11 +++++++++-- frontend/src/contexts/task-context.tsx | 10 +++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/task-notification-menu.tsx b/frontend/src/components/task-notification-menu.tsx index e17f9579..fed7e6f1 100644 --- a/frontend/src/components/task-notification-menu.tsx +++ b/frontend/src/components/task-notification-menu.tsx @@ -1,6 +1,6 @@ "use client" -import { useState } from 'react' +import { useEffect, useState } from 'react' import { Bell, CheckCircle, XCircle, Clock, Loader2, ChevronDown, ChevronUp, X } from 'lucide-react' import { Button } from '@/components/ui/button' import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card' @@ -8,9 +8,16 @@ import { Badge } from '@/components/ui/badge' import { useTask, Task } from '@/contexts/task-context' export function TaskNotificationMenu() { - const { tasks, isFetching, isMenuOpen, cancelTask } = useTask() + const { tasks, isFetching, isMenuOpen, isRecentTasksExpanded, cancelTask } = useTask() const [isExpanded, setIsExpanded] = useState(false) + // Sync local state with context state + useEffect(() => { + if (isRecentTasksExpanded) { + setIsExpanded(true) + } + }, [isRecentTasksExpanded]) + // Don't render if menu is closed if (!isMenuOpen) return null diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index a8693526..f5d0c0c4 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -43,6 +43,8 @@ interface TaskContextType { isFetching: boolean; isMenuOpen: boolean; toggleMenu: () => void; + isRecentTasksExpanded: boolean; + setRecentTasksExpanded: (expanded: boolean) => void; // React Query states isLoading: boolean; error: Error | null; @@ -53,6 +55,7 @@ const TaskContext = createContext(undefined); export function TaskProvider({ children }: { children: React.ReactNode }) { const [files, setFiles] = useState([]); const [isMenuOpen, setIsMenuOpen] = useState(false); + const [isRecentTasksExpanded, setIsRecentTasksExpanded] = useState(false); const previousTasksRef = useRef([]); const { isAuthenticated, isNoAuthMode } = useAuth(); @@ -220,7 +223,10 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { description, action: { label: "View", - onClick: () => setIsMenuOpen(true), + onClick: () => { + setIsMenuOpen(true); + setIsRecentTasksExpanded(true); + }, }, }); setTimeout(() => { @@ -302,6 +308,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { isFetching, isMenuOpen, toggleMenu, + isRecentTasksExpanded, + setRecentTasksExpanded: setIsRecentTasksExpanded, isLoading, error, }; From 5ea70cc7797b652eb829617d466d18e706a8adae Mon Sep 17 00:00:00 2001 From: Edwin Jose Date: Fri, 3 Oct 2025 16:17:05 -0400 Subject: [PATCH 32/55] Refactor file upload task to use filename mapping (#195) Changed the handling of original filenames in Langflow upload tasks to use a mapping from file paths to original filenames instead of a list. Updated both the API router and TaskService to support this change, improving reliability when associating uploaded files with their original names. --- src/api/router.py | 83 ++++++++++++++++++++---------------- src/services/task_service.py | 25 ++++++----- 2 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/api/router.py b/src/api/router.py index 327757be..15a9b116 100644 --- a/src/api/router.py +++ b/src/api/router.py @@ -13,27 +13,27 @@ logger = get_logger(__name__) async def upload_ingest_router( - request: Request, - document_service=None, - langflow_file_service=None, + request: Request, + document_service=None, + langflow_file_service=None, session_manager=None, - task_service=None + task_service=None, ): """ Router endpoint that automatically routes upload requests based on configuration. - + - If DISABLE_INGEST_WITH_LANGFLOW is True: uses traditional OpenRAG upload (/upload) - If DISABLE_INGEST_WITH_LANGFLOW is False (default): uses Langflow upload-ingest via task service - + This provides a single endpoint that users can call regardless of backend configuration. All langflow uploads are processed as background tasks for better scalability. """ try: logger.debug( - "Router upload_ingest endpoint called", - disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW + "Router upload_ingest endpoint called", + disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW, ) - + # Route based on configuration if DISABLE_INGEST_WITH_LANGFLOW: # Route to traditional OpenRAG upload @@ -42,8 +42,10 @@ async def upload_ingest_router( else: # Route to Langflow upload and ingest using task service logger.debug("Routing to Langflow upload-ingest pipeline via task service") - return await langflow_upload_ingest_task(request, langflow_file_service, session_manager, task_service) - + return await langflow_upload_ingest_task( + request, langflow_file_service, session_manager, task_service + ) + except Exception as e: logger.error("Error in upload_ingest_router", error=str(e)) error_msg = str(e) @@ -57,17 +59,14 @@ async def upload_ingest_router( async def langflow_upload_ingest_task( - request: Request, - langflow_file_service, - session_manager, - task_service + request: Request, langflow_file_service, session_manager, task_service ): """Task-based langflow upload and ingest for single/multiple files""" try: logger.debug("Task-based langflow upload_ingest endpoint called") form = await request.form() upload_files = form.getlist("file") - + if not upload_files or len(upload_files) == 0: logger.error("No files provided in task-based upload request") return JSONResponse({"error": "Missing files"}, status_code=400) @@ -82,10 +81,11 @@ async def langflow_upload_ingest_task( # Parse JSON fields if provided settings = None tweaks = None - + if settings_json: try: import json + settings = json.loads(settings_json) except json.JSONDecodeError as e: logger.error("Invalid settings JSON", error=str(e)) @@ -94,6 +94,7 @@ async def langflow_upload_ingest_task( if tweaks_json: try: import json + tweaks = json.loads(tweaks_json) except json.JSONDecodeError as e: logger.error("Invalid tweaks JSON", error=str(e)) @@ -107,11 +108,14 @@ async def langflow_upload_ingest_task( jwt_token = getattr(request.state, "jwt_token", None) if not user_id: - return JSONResponse({"error": "User authentication required"}, status_code=401) + return JSONResponse( + {"error": "User authentication required"}, status_code=401 + ) # Create temporary files for task processing import tempfile import os + temp_file_paths = [] original_filenames = [] @@ -132,7 +136,7 @@ async def langflow_upload_ingest_task( temp_path = os.path.join(temp_dir, safe_filename) # Write content to temp file - with open(temp_path, 'wb') as temp_file: + with open(temp_path, "wb") as temp_file: temp_file.write(content) temp_file_paths.append(temp_path) @@ -143,22 +147,22 @@ async def langflow_upload_ingest_task( user_id=user_id, has_settings=bool(settings), has_tweaks=bool(tweaks), - delete_after_ingest=delete_after_ingest + delete_after_ingest=delete_after_ingest, ) # Create langflow upload task - print(f"tweaks: {tweaks}") - print(f"settings: {settings}") - print(f"jwt_token: {jwt_token}") - print(f"user_name: {user_name}") - print(f"user_email: {user_email}") - print(f"session_id: {session_id}") - print(f"delete_after_ingest: {delete_after_ingest}") - print(f"temp_file_paths: {temp_file_paths}") + logger.debug( + f"Preparing to create langflow upload task: tweaks={tweaks}, settings={settings}, jwt_token={jwt_token}, user_name={user_name}, user_email={user_email}, session_id={session_id}, delete_after_ingest={delete_after_ingest}, temp_file_paths={temp_file_paths}", + ) + # Create a map between temp_file_paths and original_filenames + file_path_to_original_filename = dict(zip(temp_file_paths, original_filenames)) + logger.debug( + f"File path to original filename map: {file_path_to_original_filename}", + ) task_id = await task_service.create_langflow_upload_task( user_id=user_id, file_paths=temp_file_paths, - original_filenames=original_filenames, + original_filenames=file_path_to_original_filename, langflow_file_service=langflow_file_service, session_manager=session_manager, jwt_token=jwt_token, @@ -172,20 +176,24 @@ async def langflow_upload_ingest_task( ) logger.debug("Langflow upload task created successfully", task_id=task_id) - - return JSONResponse({ - "task_id": task_id, - "message": f"Langflow upload task created for {len(upload_files)} file(s)", - "file_count": len(upload_files) - }, status_code=202) # 202 Accepted for async processing - + + return JSONResponse( + { + "task_id": task_id, + "message": f"Langflow upload task created for {len(upload_files)} file(s)", + "file_count": len(upload_files), + }, + status_code=202, + ) # 202 Accepted for async processing + except Exception: # Clean up temp files on error from utils.file_utils import safe_unlink + for temp_path in temp_file_paths: safe_unlink(temp_path) raise - + except Exception as e: logger.error( "Task-based langflow upload_ingest endpoint failed", @@ -193,5 +201,6 @@ async def langflow_upload_ingest_task( error=str(e), ) import traceback + logger.error("Full traceback", traceback=traceback.format_exc()) return JSONResponse({"error": str(e)}, status_code=500) diff --git a/src/services/task_service.py b/src/services/task_service.py index eb5825c0..735ad483 100644 --- a/src/services/task_service.py +++ b/src/services/task_service.py @@ -1,6 +1,5 @@ import asyncio import random -from typing import Dict, Optional import time import uuid @@ -59,7 +58,7 @@ class TaskService: file_paths: list, langflow_file_service, session_manager, - original_filenames: list = None, + original_filenames: dict | None = None, jwt_token: str = None, owner_name: str = None, owner_email: str = None, @@ -88,7 +87,7 @@ class TaskService: ) return await self.create_custom_task(user_id, file_paths, processor, original_filenames) - async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: list = None) -> str: + async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: dict | None = None) -> str: """Create a new task with custom processor for any type of items""" import os # Store anonymous tasks under a stable key so they can be retrieved later @@ -96,14 +95,18 @@ class TaskService: task_id = str(uuid.uuid4()) # Create file tasks with original filenames if provided - file_tasks = {} - for i, item in enumerate(items): - if original_filenames and i < len(original_filenames): - filename = original_filenames[i] - else: - filename = os.path.basename(str(item)) - - file_tasks[str(item)] = FileTask(file_path=str(item), filename=filename) + normalized_originals = ( + {str(k): v for k, v in original_filenames.items()} if original_filenames else {} + ) + file_tasks = { + str(item): FileTask( + file_path=str(item), + filename=normalized_originals.get( + str(item), os.path.basename(str(item)) + ), + ) + for item in items + } upload_task = UploadTask( task_id=task_id, From 85de6c8026f89e7421827c4b61762fbb6d0f70b0 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Fri, 3 Oct 2025 15:56:08 -0500 Subject: [PATCH 33/55] sweeeeeeeep --- frontend/components/knowledge-dropdown.tsx | 18 +- .../src/app/api/queries/useGetSearchQuery.ts | 2 +- frontend/src/app/chat/page.tsx | 176 +++++++++--------- frontend/src/app/knowledge/page.tsx | 59 ++++-- .../app/settings/icons/google-drive-icon.tsx | 3 +- .../src/app/settings/icons/one-drive-icon.tsx | 3 +- .../app/settings/icons/share-point-icon.tsx | 3 +- .../src/components/AgGrid/agGridStyles.css | 3 + .../ui/animated-processing-icon.tsx | 99 ++++++---- frontend/src/components/ui/status-badge.tsx | 4 +- .../src/contexts/knowledge-filter-context.tsx | 5 + 11 files changed, 215 insertions(+), 160 deletions(-) diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index b6a9fea5..6a1f7df0 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -3,10 +3,10 @@ import { ChevronDown, Cloud, + File, FolderOpen, Loader2, PlugZap, - Upload, } from "lucide-react"; import { useRouter } from "next/navigation"; import { useEffect, useRef, useState } from "react"; @@ -23,6 +23,9 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { useTask } from "@/contexts/task-context"; import { cn } from "@/lib/utils"; +import GoogleDriveIcon from "@/app/settings/icons/google-drive-icon"; +import SharePointIcon from "@/app/settings/icons/share-point-icon"; +import OneDriveIcon from "@/app/settings/icons/one-drive-icon"; export function KnowledgeDropdown() { const { addTask } = useTask(); @@ -48,6 +51,15 @@ export function KnowledgeDropdown() { const fileInputRef = useRef(null); const dropdownRef = useRef(null); + const connectorIconMap: Record< + string, + React.ComponentType<{ className?: string }> + > = { + google_drive: GoogleDriveIcon, + sharepoint: SharePointIcon, + onedrive: OneDriveIcon, + }; + // Check AWS availability and cloud connectors on mount useEffect(() => { const checkAvailability = async () => { @@ -368,7 +380,7 @@ export function KnowledgeDropdown() { .filter(([, info]) => info.available) .map(([type, info]) => ({ label: info.name, - icon: PlugZap, + icon: connectorIconMap[type] || PlugZap, onClick: async () => { setIsOpen(false); if (info.connected && info.hasToken) { @@ -395,7 +407,7 @@ export function KnowledgeDropdown() { const menuItems = [ { label: "Add File", - icon: Upload, + icon: File, onClick: handleFileUpload, }, { diff --git a/frontend/src/app/api/queries/useGetSearchQuery.ts b/frontend/src/app/api/queries/useGetSearchQuery.ts index 5383178d..15aec51c 100644 --- a/frontend/src/app/api/queries/useGetSearchQuery.ts +++ b/frontend/src/app/api/queries/useGetSearchQuery.ts @@ -179,7 +179,7 @@ export const useGetSearchQuery = ( const queryResult = useQuery( { - queryKey: ["search", queryData], + queryKey: ["search", queryData, query], placeholderData: (prev) => prev, queryFn: getFiles, ...options, diff --git a/frontend/src/app/chat/page.tsx b/frontend/src/app/chat/page.tsx index 01ee43c7..c543fc35 100644 --- a/frontend/src/app/chat/page.tsx +++ b/frontend/src/app/chat/page.tsx @@ -1,7 +1,6 @@ "use client"; import { - AtSign, Bot, Check, ChevronDown, @@ -11,7 +10,6 @@ import { Loader2, Plus, Settings, - Upload, User, X, Zap, @@ -150,8 +148,8 @@ function ChatPage() { const streamAbortRef = useRef(null); const streamIdRef = useRef(0); const lastLoadedConversationRef = useRef(null); - const { addTask, isMenuOpen } = useTask(); - const { selectedFilter, parsedFilterData, isPanelOpen, setSelectedFilter } = + const { addTask } = useTask(); + const { selectedFilter, parsedFilterData, setSelectedFilter } = useKnowledgeFilter(); const scrollToBottom = () => { @@ -258,7 +256,7 @@ function ChatPage() { "Upload failed with status:", response.status, "Response:", - errorText, + errorText ); throw new Error("Failed to process document"); } @@ -468,7 +466,7 @@ function ChatPage() { console.log( "Loading conversation with", conversationData.messages.length, - "messages", + "messages" ); // Convert backend message format to frontend Message interface const convertedMessages: Message[] = conversationData.messages.map( @@ -596,7 +594,7 @@ function ChatPage() { ) === "string" ? toolCall.function?.arguments || toolCall.arguments : JSON.stringify( - toolCall.function?.arguments || toolCall.arguments, + toolCall.function?.arguments || toolCall.arguments ), result: toolCall.result, status: "completed", @@ -615,7 +613,7 @@ function ChatPage() { } return message; - }, + } ); setMessages(convertedMessages); @@ -704,7 +702,7 @@ function ChatPage() { console.log( "Chat page received file upload error event:", filename, - error, + error ); // Replace the last message with error message @@ -718,43 +716,43 @@ function ChatPage() { window.addEventListener( "fileUploadStart", - handleFileUploadStart as EventListener, + handleFileUploadStart as EventListener ); window.addEventListener( "fileUploaded", - handleFileUploaded as EventListener, + handleFileUploaded as EventListener ); window.addEventListener( "fileUploadComplete", - handleFileUploadComplete as EventListener, + handleFileUploadComplete as EventListener ); window.addEventListener( "fileUploadError", - handleFileUploadError as EventListener, + handleFileUploadError as EventListener ); return () => { window.removeEventListener( "fileUploadStart", - handleFileUploadStart as EventListener, + handleFileUploadStart as EventListener ); window.removeEventListener( "fileUploaded", - handleFileUploaded as EventListener, + handleFileUploaded as EventListener ); window.removeEventListener( "fileUploadComplete", - handleFileUploadComplete as EventListener, + handleFileUploadComplete as EventListener ); window.removeEventListener( "fileUploadError", - handleFileUploadError as EventListener, + handleFileUploadError as EventListener ); }; }, [endpoint, setPreviousResponseIds]); const { data: nudges = [], cancel: cancelNudges } = useGetNudgesQuery( - previousResponseIds[endpoint], + previousResponseIds[endpoint] ); const handleSSEStream = async (userMessage: Message) => { @@ -859,7 +857,7 @@ function ChatPage() { console.log( "Received chunk:", chunk.type || chunk.object, - chunk, + chunk ); // Extract response ID if present @@ -875,14 +873,14 @@ function ChatPage() { if (chunk.delta.function_call) { console.log( "Function call in delta:", - chunk.delta.function_call, + chunk.delta.function_call ); // Check if this is a new function call if (chunk.delta.function_call.name) { console.log( "New function call:", - chunk.delta.function_call.name, + chunk.delta.function_call.name ); const functionCall: FunctionCall = { name: chunk.delta.function_call.name, @@ -898,7 +896,7 @@ function ChatPage() { else if (chunk.delta.function_call.arguments) { console.log( "Function call arguments delta:", - chunk.delta.function_call.arguments, + chunk.delta.function_call.arguments ); const lastFunctionCall = currentFunctionCalls[currentFunctionCalls.length - 1]; @@ -910,14 +908,14 @@ function ChatPage() { chunk.delta.function_call.arguments; console.log( "Accumulated arguments:", - lastFunctionCall.argumentsString, + lastFunctionCall.argumentsString ); // Try to parse arguments if they look complete if (lastFunctionCall.argumentsString.includes("}")) { try { const parsed = JSON.parse( - lastFunctionCall.argumentsString, + lastFunctionCall.argumentsString ); lastFunctionCall.arguments = parsed; lastFunctionCall.status = "completed"; @@ -925,7 +923,7 @@ function ChatPage() { } catch (e) { console.log( "Arguments not yet complete or invalid JSON:", - e, + e ); } } @@ -958,7 +956,7 @@ function ChatPage() { else if (toolCall.function.arguments) { console.log( "Tool call arguments delta:", - toolCall.function.arguments, + toolCall.function.arguments ); const lastFunctionCall = currentFunctionCalls[ @@ -972,7 +970,7 @@ function ChatPage() { toolCall.function.arguments; console.log( "Accumulated tool arguments:", - lastFunctionCall.argumentsString, + lastFunctionCall.argumentsString ); // Try to parse arguments if they look complete @@ -981,7 +979,7 @@ function ChatPage() { ) { try { const parsed = JSON.parse( - lastFunctionCall.argumentsString, + lastFunctionCall.argumentsString ); lastFunctionCall.arguments = parsed; lastFunctionCall.status = "completed"; @@ -989,7 +987,7 @@ function ChatPage() { } catch (e) { console.log( "Tool arguments not yet complete or invalid JSON:", - e, + e ); } } @@ -1021,7 +1019,7 @@ function ChatPage() { console.log( "Error parsing function call on finish:", fc, - e, + e ); } } @@ -1037,12 +1035,12 @@ function ChatPage() { console.log( "🟢 CREATING function call (added):", chunk.item.id, - chunk.item.tool_name || chunk.item.name, + chunk.item.tool_name || chunk.item.name ); // Try to find an existing pending call to update (created by earlier deltas) let existing = currentFunctionCalls.find( - (fc) => fc.id === chunk.item.id, + (fc) => fc.id === chunk.item.id ); if (!existing) { existing = [...currentFunctionCalls] @@ -1051,7 +1049,7 @@ function ChatPage() { (fc) => fc.status === "pending" && !fc.id && - fc.name === (chunk.item.tool_name || chunk.item.name), + fc.name === (chunk.item.tool_name || chunk.item.name) ); } @@ -1064,7 +1062,7 @@ function ChatPage() { chunk.item.inputs || existing.arguments; console.log( "🟢 UPDATED existing pending function call with id:", - existing.id, + existing.id ); } else { const functionCall: FunctionCall = { @@ -1082,7 +1080,7 @@ function ChatPage() { currentFunctionCalls.map((fc) => ({ id: fc.id, name: fc.name, - })), + })) ); } } @@ -1093,7 +1091,7 @@ function ChatPage() { ) { console.log( "Function args delta (Realtime API):", - chunk.delta, + chunk.delta ); const lastFunctionCall = currentFunctionCalls[currentFunctionCalls.length - 1]; @@ -1104,7 +1102,7 @@ function ChatPage() { lastFunctionCall.argumentsString += chunk.delta || ""; console.log( "Accumulated arguments (Realtime API):", - lastFunctionCall.argumentsString, + lastFunctionCall.argumentsString ); } } @@ -1115,26 +1113,26 @@ function ChatPage() { ) { console.log( "Function args done (Realtime API):", - chunk.arguments, + chunk.arguments ); const lastFunctionCall = currentFunctionCalls[currentFunctionCalls.length - 1]; if (lastFunctionCall) { try { lastFunctionCall.arguments = JSON.parse( - chunk.arguments || "{}", + chunk.arguments || "{}" ); lastFunctionCall.status = "completed"; console.log( "Parsed function arguments (Realtime API):", - lastFunctionCall.arguments, + lastFunctionCall.arguments ); } catch (e) { lastFunctionCall.arguments = { raw: chunk.arguments }; lastFunctionCall.status = "error"; console.log( "Error parsing function arguments (Realtime API):", - e, + e ); } } @@ -1148,14 +1146,14 @@ function ChatPage() { console.log( "🔵 UPDATING function call (done):", chunk.item.id, - chunk.item.tool_name || chunk.item.name, + chunk.item.tool_name || chunk.item.name ); console.log( "🔵 Looking for existing function calls:", currentFunctionCalls.map((fc) => ({ id: fc.id, name: fc.name, - })), + })) ); // Find existing function call by ID or name @@ -1163,14 +1161,14 @@ function ChatPage() { (fc) => fc.id === chunk.item.id || fc.name === chunk.item.tool_name || - fc.name === chunk.item.name, + fc.name === chunk.item.name ); if (functionCall) { console.log( "🔵 FOUND existing function call, updating:", functionCall.id, - functionCall.name, + functionCall.name ); // Update existing function call with completion data functionCall.status = @@ -1193,7 +1191,7 @@ function ChatPage() { "🔴 WARNING: Could not find existing function call to update:", chunk.item.id, chunk.item.tool_name, - chunk.item.name, + chunk.item.name ); } } @@ -1214,7 +1212,7 @@ function ChatPage() { fc.name === chunk.item.name || fc.name === chunk.item.type || fc.name.includes(chunk.item.type.replace("_call", "")) || - chunk.item.type.includes(fc.name), + chunk.item.type.includes(fc.name) ); if (functionCall) { @@ -1258,12 +1256,12 @@ function ChatPage() { "🟡 CREATING tool call (added):", chunk.item.id, chunk.item.tool_name || chunk.item.name, - chunk.item.type, + chunk.item.type ); // Dedupe by id or pending with same name let existing = currentFunctionCalls.find( - (fc) => fc.id === chunk.item.id, + (fc) => fc.id === chunk.item.id ); if (!existing) { existing = [...currentFunctionCalls] @@ -1275,7 +1273,7 @@ function ChatPage() { fc.name === (chunk.item.tool_name || chunk.item.name || - chunk.item.type), + chunk.item.type) ); } @@ -1291,7 +1289,7 @@ function ChatPage() { chunk.item.inputs || existing.arguments; console.log( "🟡 UPDATED existing pending tool call with id:", - existing.id, + existing.id ); } else { const functionCall = { @@ -1312,7 +1310,7 @@ function ChatPage() { id: fc.id, name: fc.name, type: fc.type, - })), + })) ); } } @@ -1590,7 +1588,7 @@ function ChatPage() { const handleForkConversation = ( messageIndex: number, - event?: React.MouseEvent, + event?: React.MouseEvent ) => { // Prevent any default behavior and stop event propagation if (event) { @@ -1655,7 +1653,7 @@ function ChatPage() { const renderFunctionCalls = ( functionCalls: FunctionCall[], - messageIndex?: number, + messageIndex?: number ) => { if (!functionCalls || functionCalls.length === 0) return null; @@ -1906,7 +1904,7 @@ function ChatPage() { if (isFilterDropdownOpen) { const filteredFilters = availableFilters.filter((filter) => - filter.name.toLowerCase().includes(filterSearchTerm.toLowerCase()), + filter.name.toLowerCase().includes(filterSearchTerm.toLowerCase()) ); if (e.key === "Escape") { @@ -1924,7 +1922,7 @@ function ChatPage() { if (e.key === "ArrowDown") { e.preventDefault(); setSelectedFilterIndex((prev) => - prev < filteredFilters.length - 1 ? prev + 1 : 0, + prev < filteredFilters.length - 1 ? prev + 1 : 0 ); return; } @@ -1932,7 +1930,7 @@ function ChatPage() { if (e.key === "ArrowUp") { e.preventDefault(); setSelectedFilterIndex((prev) => - prev > 0 ? prev - 1 : filteredFilters.length - 1, + prev > 0 ? prev - 1 : filteredFilters.length - 1 ); return; } @@ -2029,7 +2027,7 @@ function ChatPage() { // Get button position for popover anchoring const button = document.querySelector( - "[data-filter-button]", + "[data-filter-button]" ) as HTMLElement; if (button) { const rect = button.getBoundingClientRect(); @@ -2045,20 +2043,10 @@ function ChatPage() { }; return ( -
+
{/* Debug header - only show in debug mode */} {isDebugMode && ( -
+
{/* Async Mode Toggle */} @@ -2164,7 +2152,7 @@ function ChatPage() {
{renderFunctionCalls( message.functionCalls || [], - index, + index )}
@@ -2193,7 +2181,7 @@ function ChatPage() {
{renderFunctionCalls( streamingMessage.functionCalls, - messages.length, + messages.length )}
)} -
- setTextareaHeight(height)} - maxRows={7} - minRows={2} - placeholder="Type to ask a question..." - disabled={loading} - className={`w-full bg-transparent px-4 ${ - selectedFilter ? "pt-2" : "pt-4" - } focus-visible:outline-none resize-none`} - rows={2} - /> +
+ setTextareaHeight(height)} + maxRows={7} + minRows={2} + placeholder="Type to ask a question..." + disabled={loading} + className={`w-full bg-transparent px-4 ${ + selectedFilter ? "pt-2" : "pt-4" + } focus-visible:outline-none resize-none`} + rows={2} + /> {/* Safe area at bottom for buttons */} -
-
filter.name .toLowerCase() - .includes(filterSearchTerm.toLowerCase()), + .includes(filterSearchTerm.toLowerCase()) ) .map((filter, index) => (
); diff --git a/frontend/src/contexts/knowledge-filter-context.tsx b/frontend/src/contexts/knowledge-filter-context.tsx index 043f6fae..1e23fdf0 100644 --- a/frontend/src/contexts/knowledge-filter-context.tsx +++ b/frontend/src/contexts/knowledge-filter-context.tsx @@ -44,6 +44,8 @@ interface KnowledgeFilterContextType { createMode: boolean; startCreateMode: () => void; endCreateMode: () => void; + queryOverride: string; + setQueryOverride: (query: string) => void; } const KnowledgeFilterContext = createContext< @@ -73,6 +75,7 @@ export function KnowledgeFilterProvider({ useState(null); const [isPanelOpen, setIsPanelOpen] = useState(false); const [createMode, setCreateMode] = useState(false); + const [queryOverride, setQueryOverride] = useState(''); const setSelectedFilter = (filter: KnowledgeFilter | null) => { setSelectedFilterState(filter); @@ -148,6 +151,8 @@ export function KnowledgeFilterProvider({ createMode, startCreateMode, endCreateMode, + queryOverride, + setQueryOverride, }; return ( From dd5db8acc1e7d5cec2e6ed2dddac58348734a660 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Fri, 3 Oct 2025 17:18:38 -0500 Subject: [PATCH 34/55] cleaning up search override functionality --- .../components/knowledge-filter-panel.tsx | 2 ++ frontend/src/app/knowledge/page.tsx | 34 +++++++++++++++++-- frontend/src/components/layout-wrapper.tsx | 2 +- .../src/contexts/knowledge-filter-context.tsx | 8 ++++- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/frontend/components/knowledge-filter-panel.tsx b/frontend/components/knowledge-filter-panel.tsx index a45304e9..e8c26e37 100644 --- a/frontend/components/knowledge-filter-panel.tsx +++ b/frontend/components/knowledge-filter-panel.tsx @@ -50,6 +50,7 @@ export const filterAccentClasses: Record = { export function KnowledgeFilterPanel() { const { + queryOverride, selectedFilter, parsedFilterData, setSelectedFilter, @@ -320,6 +321,7 @@ export function KnowledgeFilterPanel() { className="font-mono placeholder:font-mono" onChange={(e) => setQuery(e.target.value)} rows={2} + disabled={!!queryOverride} />
diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 9e135ddd..1cbc7563 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -2,12 +2,13 @@ import { themeQuartz, type ColDef } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; -import { Cloud, FileIcon, Search, X } from "lucide-react"; +import { ArrowRight, Cloud, FileIcon, Search, X } from "lucide-react"; import { useRouter } from "next/navigation"; import { type ChangeEvent, FormEvent, useCallback, + useEffect, useRef, useState, } from "react"; @@ -28,6 +29,7 @@ import { filterAccentClasses } from "@/components/knowledge-filter-panel"; import GoogleDriveIcon from "../settings/icons/google-drive-icon"; import OneDriveIcon from "../settings/icons/one-drive-icon"; import SharePointIcon from "../settings/icons/share-point-icon"; +import { cn } from "@/lib/utils"; // Function to get the appropriate icon for a connector type function getSourceIcon(connectorType?: string) { @@ -244,6 +246,11 @@ function SearchPage() { } }; + // Reset the query text when the selected filter changes + useEffect(() => { + setSearchQueryInput(queryOverride); + }, [queryOverride]); + return ( <>
@@ -257,7 +264,7 @@ function SearchPage() { className="flex flex-1 gap-3 max-w-full" onSubmit={handleSearch} > -
+
{selectedFilter?.name && (
+ {queryOverride && ( + + )} +
{/* +
+ csv, json, pdf,{" "} + +16 more{" "} + 150 MB max +
); From c2f50fe139563453b8edd5cf311095d0ab71a9e3 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Mon, 6 Oct 2025 08:53:39 -0600 Subject: [PATCH 40/55] revert --- frontend/src/components/cloud-picker/picker-header.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/cloud-picker/picker-header.tsx b/frontend/src/components/cloud-picker/picker-header.tsx index 54407aa7..05dcaebd 100644 --- a/frontend/src/components/cloud-picker/picker-header.tsx +++ b/frontend/src/components/cloud-picker/picker-header.tsx @@ -57,7 +57,7 @@ export const PickerHeader = ({ className="bg-foreground text-background hover:bg-foreground/90 font-semibold" > - Add Files + {isPickerOpen ? "Opening Picker..." : "Add Files"}
csv, json, pdf,{" "} From 47b84a3d8a2e74d7db88ae578a328a6e60d10214 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Mon, 6 Oct 2025 08:55:12 -0600 Subject: [PATCH 41/55] revert --- frontend/src/app/upload/[provider]/page.tsx | 659 ++++++++++---------- 1 file changed, 328 insertions(+), 331 deletions(-) diff --git a/frontend/src/app/upload/[provider]/page.tsx b/frontend/src/app/upload/[provider]/page.tsx index 8e0a306c..7c72ec3d 100644 --- a/frontend/src/app/upload/[provider]/page.tsx +++ b/frontend/src/app/upload/[provider]/page.tsx @@ -12,370 +12,367 @@ import { useTask } from "@/contexts/task-context"; // CloudFile interface is now imported from the unified cloud picker interface CloudConnector { - id: string; - name: string; - description: string; - status: "not_connected" | "connecting" | "connected" | "error"; - type: string; - connectionId?: string; - clientId: string; - hasAccessToken: boolean; - accessTokenError?: string; + id: string; + name: string; + description: string; + status: "not_connected" | "connecting" | "connected" | "error"; + type: string; + connectionId?: string; + clientId: string; + hasAccessToken: boolean; + accessTokenError?: string; } export default function UploadProviderPage() { - const params = useParams(); - const router = useRouter(); - const provider = params.provider as string; - const { addTask, tasks } = useTask(); + const params = useParams(); + const router = useRouter(); + const provider = params.provider as string; + const { addTask, tasks } = useTask(); - const [connector, setConnector] = useState(null); - const [isLoading, setIsLoading] = useState(true); - const [error, setError] = useState(null); - const [accessToken, setAccessToken] = useState(null); - const [selectedFiles, setSelectedFiles] = useState([]); - const [isIngesting, setIsIngesting] = useState(false); - const [currentSyncTaskId, setCurrentSyncTaskId] = useState( - null - ); - const [ingestSettings, setIngestSettings] = useState({ - chunkSize: 1000, - chunkOverlap: 200, - ocr: false, - pictureDescriptions: false, - embeddingModel: "text-embedding-3-small", - }); + const [connector, setConnector] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const [accessToken, setAccessToken] = useState(null); + const [selectedFiles, setSelectedFiles] = useState([]); + const [isIngesting, setIsIngesting] = useState(false); + const [currentSyncTaskId, setCurrentSyncTaskId] = useState( + null, + ); + const [ingestSettings, setIngestSettings] = useState({ + chunkSize: 1000, + chunkOverlap: 200, + ocr: false, + pictureDescriptions: false, + embeddingModel: "text-embedding-3-small", + }); - useEffect(() => { - const fetchConnectorInfo = async () => { - setIsLoading(true); - setError(null); + useEffect(() => { + const fetchConnectorInfo = async () => { + setIsLoading(true); + setError(null); - try { - // Fetch available connectors to validate the provider - const connectorsResponse = await fetch("/api/connectors"); - if (!connectorsResponse.ok) { - throw new Error("Failed to load connectors"); - } + try { + // Fetch available connectors to validate the provider + const connectorsResponse = await fetch("/api/connectors"); + if (!connectorsResponse.ok) { + throw new Error("Failed to load connectors"); + } - const connectorsResult = await connectorsResponse.json(); - const providerInfo = connectorsResult.connectors[provider]; + const connectorsResult = await connectorsResponse.json(); + const providerInfo = connectorsResult.connectors[provider]; - if (!providerInfo || !providerInfo.available) { - setError( - `Cloud provider "${provider}" is not available or configured.` - ); - return; - } + if (!providerInfo || !providerInfo.available) { + setError( + `Cloud provider "${provider}" is not available or configured.`, + ); + return; + } - // Check connector status - const statusResponse = await fetch( - `/api/connectors/${provider}/status` - ); - if (!statusResponse.ok) { - throw new Error(`Failed to check ${provider} status`); - } + // Check connector status + const statusResponse = await fetch( + `/api/connectors/${provider}/status`, + ); + if (!statusResponse.ok) { + throw new Error(`Failed to check ${provider} status`); + } - const statusData = await statusResponse.json(); - const connections = statusData.connections || []; - const activeConnection = connections.find( - (conn: { is_active: boolean; connection_id: string }) => - conn.is_active - ); - const isConnected = activeConnection !== undefined; + const statusData = await statusResponse.json(); + const connections = statusData.connections || []; + const activeConnection = connections.find( + (conn: { is_active: boolean; connection_id: string }) => + conn.is_active, + ); + const isConnected = activeConnection !== undefined; - let hasAccessToken = false; - let accessTokenError: string | undefined; + let hasAccessToken = false; + let accessTokenError: string | undefined; - // Try to get access token for connected connectors - if (isConnected && activeConnection) { - try { - const tokenResponse = await fetch( - `/api/connectors/${provider}/token?connection_id=${activeConnection.connection_id}` - ); - if (tokenResponse.ok) { - const tokenData = await tokenResponse.json(); - if (tokenData.access_token) { - hasAccessToken = true; - setAccessToken(tokenData.access_token); - } - } else { - const errorData = await tokenResponse - .json() - .catch(() => ({ error: "Token unavailable" })); - accessTokenError = errorData.error || "Access token unavailable"; - } - } catch { - accessTokenError = "Failed to fetch access token"; - } - } + // Try to get access token for connected connectors + if (isConnected && activeConnection) { + try { + const tokenResponse = await fetch( + `/api/connectors/${provider}/token?connection_id=${activeConnection.connection_id}`, + ); + if (tokenResponse.ok) { + const tokenData = await tokenResponse.json(); + if (tokenData.access_token) { + hasAccessToken = true; + setAccessToken(tokenData.access_token); + } + } else { + const errorData = await tokenResponse + .json() + .catch(() => ({ error: "Token unavailable" })); + accessTokenError = errorData.error || "Access token unavailable"; + } + } catch { + accessTokenError = "Failed to fetch access token"; + } + } - setConnector({ - id: provider, - name: providerInfo.name, - description: providerInfo.description, - status: isConnected ? "connected" : "not_connected", - type: provider, - connectionId: activeConnection?.connection_id, - clientId: activeConnection?.client_id, - hasAccessToken, - accessTokenError, - }); - } catch (error) { - console.error("Failed to load connector info:", error); - setError( - error instanceof Error - ? error.message - : "Failed to load connector information" - ); - } finally { - setIsLoading(false); - } - }; + setConnector({ + id: provider, + name: providerInfo.name, + description: providerInfo.description, + status: isConnected ? "connected" : "not_connected", + type: provider, + connectionId: activeConnection?.connection_id, + clientId: activeConnection?.client_id, + hasAccessToken, + accessTokenError, + }); + } catch (error) { + console.error("Failed to load connector info:", error); + setError( + error instanceof Error + ? error.message + : "Failed to load connector information", + ); + } finally { + setIsLoading(false); + } + }; - if (provider) { - fetchConnectorInfo(); - } - }, [provider]); + if (provider) { + fetchConnectorInfo(); + } + }, [provider]); - // Watch for sync task completion and redirect - useEffect(() => { - if (!currentSyncTaskId) return; + // Watch for sync task completion and redirect + useEffect(() => { + if (!currentSyncTaskId) return; - const currentTask = tasks.find( - (task) => task.task_id === currentSyncTaskId - ); + const currentTask = tasks.find( + (task) => task.task_id === currentSyncTaskId, + ); - if (currentTask && currentTask.status === "completed") { - // Task completed successfully, show toast and redirect - setIsIngesting(false); - setTimeout(() => { - router.push("/knowledge"); - }, 2000); // 2 second delay to let user see toast - } else if (currentTask && currentTask.status === "failed") { - // Task failed, clear the tracking but don't redirect - setIsIngesting(false); - setCurrentSyncTaskId(null); - } - }, [tasks, currentSyncTaskId, router]); + if (currentTask && currentTask.status === "completed") { + // Task completed successfully, show toast and redirect + setIsIngesting(false); + setTimeout(() => { + router.push("/knowledge"); + }, 2000); // 2 second delay to let user see toast + } else if (currentTask && currentTask.status === "failed") { + // Task failed, clear the tracking but don't redirect + setIsIngesting(false); + setCurrentSyncTaskId(null); + } + }, [tasks, currentSyncTaskId, router]); - const handleFileSelected = (files: CloudFile[]) => { - setSelectedFiles(files); - console.log(`Selected ${files.length} files from ${provider}:`, files); - // You can add additional handling here like triggering sync, etc. - }; + const handleFileSelected = (files: CloudFile[]) => { + setSelectedFiles(files); + console.log(`Selected ${files.length} files from ${provider}:`, files); + // You can add additional handling here like triggering sync, etc. + }; - const handleSync = async (connector: CloudConnector) => { - if (!connector.connectionId || selectedFiles.length === 0) return; + const handleSync = async (connector: CloudConnector) => { + if (!connector.connectionId || selectedFiles.length === 0) return; - setIsIngesting(true); + setIsIngesting(true); - try { - const syncBody: { - connection_id: string; - max_files?: number; - selected_files?: string[]; - settings?: IngestSettings; - } = { - connection_id: connector.connectionId, - selected_files: selectedFiles.map((file) => file.id), - settings: ingestSettings, - }; + try { + const syncBody: { + connection_id: string; + max_files?: number; + selected_files?: string[]; + settings?: IngestSettings; + } = { + connection_id: connector.connectionId, + selected_files: selectedFiles.map((file) => file.id), + settings: ingestSettings, + }; - const response = await fetch(`/api/connectors/${connector.type}/sync`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(syncBody), - }); + const response = await fetch(`/api/connectors/${connector.type}/sync`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(syncBody), + }); - const result = await response.json(); + const result = await response.json(); - if (response.status === 201) { - const taskIds = result.task_ids; - if (taskIds && taskIds.length > 0) { - const taskId = taskIds[0]; // Use the first task ID - addTask(taskId); - setCurrentSyncTaskId(taskId); - } - } else { - console.error("Sync failed:", result.error); - } - } catch (error) { - console.error("Sync error:", error); - setIsIngesting(false); - } - }; + if (response.status === 201) { + const taskIds = result.task_ids; + if (taskIds && taskIds.length > 0) { + const taskId = taskIds[0]; // Use the first task ID + addTask(taskId); + setCurrentSyncTaskId(taskId); + } + } else { + console.error("Sync failed:", result.error); + } + } catch (error) { + console.error("Sync error:", error); + setIsIngesting(false); + } + }; - const getProviderDisplayName = () => { - const nameMap: { [key: string]: string } = { - google_drive: "Google Drive", - onedrive: "OneDrive", - sharepoint: "SharePoint", - }; - return nameMap[provider] || provider; - }; + const getProviderDisplayName = () => { + const nameMap: { [key: string]: string } = { + google_drive: "Google Drive", + onedrive: "OneDrive", + sharepoint: "SharePoint", + }; + return nameMap[provider] || provider; + }; - if (isLoading) { - return ( -
-
-
-
-

Loading {getProviderDisplayName()} connector...

-
-
-
- ); - } + if (isLoading) { + return ( +
+
+
+
+

Loading {getProviderDisplayName()} connector...

+
+
+
+ ); + } - if (error || !connector) { - return ( -
-
- -
+ if (error || !connector) { + return ( +
+
+ +
-
-
- -

- Provider Not Available -

-

{error}

- -
-
-
- ); - } +
+
+ +

+ Provider Not Available +

+

{error}

+ +
+
+
+ ); + } - if (connector.status !== "connected") { - return ( -
-
- -
+ if (connector.status !== "connected") { + return ( +
+
+ +
-
-
- -

- {connector.name} Not Connected -

-

- You need to connect your {connector.name} account before you can - select files. -

- -
-
-
- ); - } +
+
+ +

+ {connector.name} Not Connected +

+

+ You need to connect your {connector.name} account before you can + select files. +

+ +
+
+
+ ); + } - if (!connector.hasAccessToken) { - return ( -
-
- -
+ if (!connector.hasAccessToken) { + return ( +
+
+ +
-
-
- -

- Access Token Required -

-

- {connector.accessTokenError || - `Unable to get access token for ${connector.name}. Try reconnecting your account.`} -

- -
-
-
- ); - } +
+
+ +

+ Access Token Required +

+

+ {connector.accessTokenError || + `Unable to get access token for ${connector.name}. Try reconnecting your account.`} +

+ +
+
+
+ ); + } - return ( -
-
- -

- Add from {getProviderDisplayName()} -

-
+ return ( +
+
+ +

+ Add from {getProviderDisplayName()} +

+
-
- -
+
+ +
-
-
- - -
-
-
- ); +
+
+ + +
+
+
+ ); } From 641a24aa0664cf3a1f33f98da2ecabd85127f2ed Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Mon, 6 Oct 2025 11:57:51 -0300 Subject: [PATCH 42/55] removed unused file --- .../app/api/queries/useGetTaskStatusQuery.ts | 80 ------------------- 1 file changed, 80 deletions(-) delete mode 100644 frontend/src/app/api/queries/useGetTaskStatusQuery.ts diff --git a/frontend/src/app/api/queries/useGetTaskStatusQuery.ts b/frontend/src/app/api/queries/useGetTaskStatusQuery.ts deleted file mode 100644 index 17cd2d16..00000000 --- a/frontend/src/app/api/queries/useGetTaskStatusQuery.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { - type UseQueryOptions, - useQuery, - useQueryClient, -} from "@tanstack/react-query"; - -export interface TaskStatus { - task_id: string; - status: - | "pending" - | "running" - | "processing" - | "completed" - | "failed" - | "error"; - total_files?: number; - processed_files?: number; - successful_files?: number; - failed_files?: number; - running_files?: number; - pending_files?: number; - created_at: string; - updated_at: string; - duration_seconds?: number; - result?: Record; - error?: string; - files?: Record>; -} - -export const useGetTaskStatusQuery = ( - taskId: string, - options?: Omit, "queryKey" | "queryFn"> -) => { - const queryClient = useQueryClient(); - - async function getTaskStatus(): Promise { - if (!taskId) { - return null; - } - - const response = await fetch(`/api/tasks/${taskId}`); - - if (!response.ok) { - if (response.status === 404) { - return null; // Task not found - } - throw new Error("Failed to fetch task status"); - } - - return response.json(); - } - - const queryResult = useQuery( - { - queryKey: ["task-status", taskId], - queryFn: getTaskStatus, - refetchInterval: (data) => { - // Only poll if the task is still active - if (!data) { - return false; // Stop polling if no data - } - - const isActive = - data.status === "pending" || - data.status === "running" || - data.status === "processing"; - - return isActive ? 3000 : false; // Poll every 3 seconds if active - }, - refetchIntervalInBackground: true, - staleTime: 0, // Always consider data stale to ensure fresh updates - gcTime: 5 * 60 * 1000, // Keep in cache for 5 minutes - enabled: !!taskId, // Only run if taskId is provided - ...options, - }, - queryClient, - ); - - return queryResult; -}; From 2547af298f262a7a7b1e503585abfc1fa4200925 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Mon, 6 Oct 2025 10:40:03 -0500 Subject: [PATCH 43/55] make knowledge search a component used for knowledge and chunk page --- .../components/knowledge-filter-panel.tsx | 2 +- .../components/knowledge-search-input.tsx | 100 +++++++++++++ frontend/src/app/knowledge/chunks/page.tsx | 132 ++++++++---------- frontend/src/app/knowledge/page.tsx | 121 ++-------------- 4 files changed, 171 insertions(+), 184 deletions(-) create mode 100644 frontend/components/knowledge-search-input.tsx diff --git a/frontend/components/knowledge-filter-panel.tsx b/frontend/components/knowledge-filter-panel.tsx index e8c26e37..d0dd8196 100644 --- a/frontend/components/knowledge-filter-panel.tsx +++ b/frontend/components/knowledge-filter-panel.tsx @@ -321,7 +321,7 @@ export function KnowledgeFilterPanel() { className="font-mono placeholder:font-mono" onChange={(e) => setQuery(e.target.value)} rows={2} - disabled={!!queryOverride} + disabled={!!queryOverride && !createMode} />
diff --git a/frontend/components/knowledge-search-input.tsx b/frontend/components/knowledge-search-input.tsx new file mode 100644 index 00000000..fd840628 --- /dev/null +++ b/frontend/components/knowledge-search-input.tsx @@ -0,0 +1,100 @@ +import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; +import { + ChangeEvent, + FormEvent, + useCallback, + useEffect, + useState, +} from "react"; +import { filterAccentClasses } from "./knowledge-filter-panel"; +import { ArrowRight, Search, X } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; + +export const KnowledgeSearchInput = () => { + const { + selectedFilter, + setSelectedFilter, + parsedFilterData, + queryOverride, + setQueryOverride, + } = useKnowledgeFilter(); + + const [searchQueryInput, setSearchQueryInput] = useState(queryOverride || ""); + + const handleSearch = useCallback( + (e?: FormEvent) => { + if (e) e.preventDefault(); + setQueryOverride(searchQueryInput.trim()); + }, + [searchQueryInput, setQueryOverride] + ); + + // Reset the query text when the selected filter changes + useEffect(() => { + setSearchQueryInput(queryOverride); + }, [queryOverride]); + + return ( + +
+ {selectedFilter?.name && ( +
+ {selectedFilter?.name} + setSelectedFilter(null)} + /> +
+ )} + + ) => + setSearchQueryInput(e.target.value) + } + /> + {queryOverride && ( + + )} + +
+ + ); +}; diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index cb96eddc..c6a3d5e8 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -6,15 +6,13 @@ import { useRouter, useSearchParams } from "next/navigation"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; -import { useTask } from "@/contexts/task-context"; import { type ChunkResult, type File, useGetSearchQuery, } from "../../api/queries/useGetSearchQuery"; -import { Label } from "@/components/ui/label"; import { Checkbox } from "@/components/ui/checkbox"; -import { Input } from "@/components/ui/input"; +import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; const getFileTypeLabel = (mimetype: string) => { if (mimetype === "application/pdf") return "PDF"; @@ -26,8 +24,7 @@ const getFileTypeLabel = (mimetype: string) => { function ChunksPageContent() { const router = useRouter(); const searchParams = useSearchParams(); - const { isMenuOpen } = useTask(); - const { parsedFilterData, isPanelOpen } = useKnowledgeFilter(); + const { parsedFilterData, queryOverride } = useKnowledgeFilter(); const filename = searchParams.get("filename"); const [chunks, setChunks] = useState([]); @@ -47,25 +44,25 @@ function ChunksPageContent() { [chunks] ); - const [selectAll, setSelectAll] = useState(false); - const [queryInputText, setQueryInputText] = useState( - parsedFilterData?.query ?? "" - ); + // const [selectAll, setSelectAll] = useState(false); // Use the same search query as the knowledge page, but we'll filter for the specific file - const { data = [], isFetching } = useGetSearchQuery("*", parsedFilterData); + const { data = [], isFetching } = useGetSearchQuery( + queryOverride, + parsedFilterData + ); - useEffect(() => { - if (queryInputText === "") { - setChunksFilteredByQuery(chunks); - } else { - setChunksFilteredByQuery( - chunks.filter((chunk) => - chunk.text.toLowerCase().includes(queryInputText.toLowerCase()) - ) - ); - } - }, [queryInputText, chunks]); + // useEffect(() => { + // if (queryInputText === "") { + // setChunksFilteredByQuery(chunks); + // } else { + // setChunksFilteredByQuery( + // chunks.filter((chunk) => + // chunk.text.toLowerCase().includes(queryInputText.toLowerCase()) + // ) + // ); + // } + // }, [queryInputText, chunks]); const handleCopy = useCallback((text: string, index: number) => { // Trim whitespace and remove new lines/tabs for cleaner copy @@ -89,13 +86,13 @@ function ChunksPageContent() { }, [data, filename]); // Set selected state for all checkboxes when selectAll changes - useEffect(() => { - if (selectAll) { - setSelectedChunks(new Set(chunks.map((_, index) => index))); - } else { - setSelectedChunks(new Set()); - } - }, [selectAll, setSelectedChunks, chunks]); + // useEffect(() => { + // if (selectAll) { + // setSelectedChunks(new Set(chunks.map((_, index) => index))); + // } else { + // setSelectedChunks(new Set()); + // } + // }, [selectAll, setSelectedChunks, chunks]); const handleBack = useCallback(() => { router.push("/knowledge"); @@ -131,25 +128,17 @@ function ChunksPageContent() { } return ( -
-
+
+
{/* Header */} -
-
-

@@ -157,39 +146,12 @@ function ChunksPageContent() { {filename.replace(/\.[^/.]+$/, "")}

-
-
- : null} - id="search-query" - type="text" - defaultValue={parsedFilterData?.query} - value={queryInputText} - onChange={(e) => setQueryInputText(e.target.value)} - placeholder="Search chunks..." - /> -
-
- - setSelectAll(!!handleSelectAll) - } - /> - -
-
+ {/* Search input */} +
{/* Content Area - matches knowledge page structure */} -
+
{isFetching ? (
@@ -211,7 +173,23 @@ function ChunksPageContent() {
) : (
- {chunksFilteredByQuery.map((chunk, index) => ( + {/* TODO - add chunk selection when sync and delete are ready */} + {/*
+ + setSelectAll(!!handleSelectAll) + } + /> + +
*/} + {chunks.map((chunk, index) => (
([]); const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); @@ -74,14 +66,6 @@ function SearchPage() { parsedFilterData ); - const handleSearch = useCallback( - (e?: FormEvent) => { - if (e) e.preventDefault(); - setQueryOverride(searchQueryInput.trim()); - }, - [searchQueryInput, setQueryOverride] - ); - // Convert TaskFiles to File format and merge with backend results const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { return { @@ -246,11 +230,6 @@ function SearchPage() { } }; - // Reset the query text when the selected filter changes - useEffect(() => { - setSearchQueryInput(queryOverride); - }, [queryOverride]); - return ( <>
@@ -260,78 +239,9 @@ function SearchPage() { {/* Search Input Area */}
-
-
- {selectedFilter?.name && ( -
- {selectedFilter?.name} - setSelectedFilter(null)} - /> -
- )} - - ) => - setSearchQueryInput(e.target.value) - } - /> - {queryOverride && ( - - )} - -
- {/* */} - {/* //TODO: Implement sync button */} - {/* */} - {selectedRows.length > 0 && ( - - )} -
+ {selectedRows.length > 0 && ( + + )}
From 1643e10d5f215443c9039c1e07d8024613f96546 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Mon, 6 Oct 2025 09:53:14 -0600 Subject: [PATCH 44/55] Cleanup commented code --- frontend/src/app/knowledge/chunks/page.tsx | 32 +++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index 9ae3619c..6288bcfc 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -13,8 +13,8 @@ import { type File, useGetSearchQuery, } from "../../api/queries/useGetSearchQuery"; -import { Label } from "@/components/ui/label"; -import { Checkbox } from "@/components/ui/checkbox"; +// import { Label } from "@/components/ui/label"; +// import { Checkbox } from "@/components/ui/checkbox"; import { filterAccentClasses } from "@/components/knowledge-filter-panel"; const getFileTypeLabel = (mimetype: string) => { @@ -106,20 +106,20 @@ function ChunksPageContent() { router.push("/knowledge"); }, [router]); - const handleChunkCardCheckboxChange = useCallback( - (index: number) => { - setSelectedChunks((prevSelected) => { - const newSelected = new Set(prevSelected); - if (newSelected.has(index)) { - newSelected.delete(index); - } else { - newSelected.add(index); - } - return newSelected; - }); - }, - [setSelectedChunks] - ); + // const handleChunkCardCheckboxChange = useCallback( + // (index: number) => { + // setSelectedChunks((prevSelected) => { + // const newSelected = new Set(prevSelected); + // if (newSelected.has(index)) { + // newSelected.delete(index); + // } else { + // newSelected.add(index); + // } + // return newSelected; + // }); + // }, + // [setSelectedChunks] + // ); if (!filename) { return ( From ca8c16bd8b04f36f5f281842ace828bed0c343eb Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 6 Oct 2025 12:11:56 -0400 Subject: [PATCH 45/55] set backend container name --- docker-compose-cpu.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml index 570bc3b8..0c09254a 100644 --- a/docker-compose-cpu.yml +++ b/docker-compose-cpu.yml @@ -43,7 +43,7 @@ services: # build: # context: . # dockerfile: Dockerfile.backend - # container_name: openrag-backend + container_name: openrag-backend depends_on: - langflow environment: diff --git a/docker-compose.yml b/docker-compose.yml index b97f7cca..be9bcbc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: # build: # context: . # dockerfile: Dockerfile.backend - # container_name: openrag-backend + container_name: openrag-backend depends_on: - langflow environment: From ee0d58a627548dfa02113b2719bb175235111189 Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 6 Oct 2025 12:13:09 -0400 Subject: [PATCH 46/55] clear default nudges --- frontend/src/app/api/queries/useGetNudgesQuery.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/frontend/src/app/api/queries/useGetNudgesQuery.ts b/frontend/src/app/api/queries/useGetNudgesQuery.ts index a9fe37a4..2e313e0c 100644 --- a/frontend/src/app/api/queries/useGetNudgesQuery.ts +++ b/frontend/src/app/api/queries/useGetNudgesQuery.ts @@ -7,9 +7,6 @@ import { type Nudge = string; const DEFAULT_NUDGES = [ - "Show me this quarter's top 10 deals", - "Summarize recent client interactions", - "Search OpenSearch for mentions of our competitors", ]; export const useGetNudgesQuery = ( From 14c3a8f3d1d0e14e4bdead50ae9d9f041f5afeb4 Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 6 Oct 2025 12:14:03 -0400 Subject: [PATCH 47/55] tui: start/stop all services --- src/tui/screens/welcome.py | 176 ++++++++++++++++++------------------- 1 file changed, 86 insertions(+), 90 deletions(-) diff --git a/src/tui/screens/welcome.py b/src/tui/screens/welcome.py index c93f5561..9c121022 100644 --- a/src/tui/screens/welcome.py +++ b/src/tui/screens/welcome.py @@ -118,9 +118,16 @@ class WelcomeScreen(Screen): welcome_text.append(ascii_art, style="bold white") welcome_text.append("Terminal User Interface for OpenRAG\n\n", style="dim") - if self.services_running: + # Check if all services are running + all_services_running = self.services_running and self.docling_running + + if all_services_running: welcome_text.append( - "✓ Services are currently running\n\n", style="bold green" + "✓ All services are running\n\n", style="bold green" + ) + elif self.services_running or self.docling_running: + welcome_text.append( + "⚠ Some services are running\n\n", style="bold yellow" ) elif self.has_oauth_config: welcome_text.append( @@ -140,16 +147,19 @@ class WelcomeScreen(Screen): buttons = [] - if self.services_running: - # Services running - show app link first, then stop services + # Check if all services (native + container) are running + all_services_running = self.services_running and self.docling_running + + if all_services_running: + # All services running - show app link first, then stop all buttons.append( Button("Launch OpenRAG", variant="success", id="open-app-btn") ) buttons.append( - Button("Stop Container Services", variant="error", id="stop-services-btn") + Button("Stop All Services", variant="error", id="stop-all-services-btn") ) else: - # Services not running - show setup options and start services + # Some or no services running - show setup options and start all if has_oauth: # If OAuth is configured, only show advanced setup buttons.append( @@ -165,25 +175,7 @@ class WelcomeScreen(Screen): ) buttons.append( - Button("Start Container Services", variant="primary", id="start-services-btn") - ) - - # Native services controls - if self.docling_running: - buttons.append( - Button( - "Stop Native Services", - variant="warning", - id="stop-native-services-btn", - ) - ) - else: - buttons.append( - Button( - "Start Native Services", - variant="primary", - id="start-native-services-btn", - ) + Button("Start All Services", variant="primary", id="start-all-services-btn") ) # Always show status option @@ -213,7 +205,7 @@ class WelcomeScreen(Screen): ) # Set default button focus - if self.services_running: + if self.services_running and self.docling_running: self.default_button_id = "open-app-btn" elif self.has_oauth_config: self.default_button_id = "advanced-setup-btn" @@ -234,7 +226,7 @@ class WelcomeScreen(Screen): def _focus_appropriate_button(self) -> None: """Focus the appropriate button based on current state.""" try: - if self.services_running: + if self.services_running and self.docling_running: self.query_one("#open-app-btn").focus() elif self.has_oauth_config: self.query_one("#advanced-setup-btn").focus() @@ -253,20 +245,16 @@ class WelcomeScreen(Screen): self.action_monitor() elif event.button.id == "diagnostics-btn": self.action_diagnostics() - elif event.button.id == "start-services-btn": - self.action_start_stop_services() - elif event.button.id == "stop-services-btn": - self.action_start_stop_services() - elif event.button.id == "start-native-services-btn": - self.action_start_native_services() - elif event.button.id == "stop-native-services-btn": - self.action_stop_native_services() + elif event.button.id == "start-all-services-btn": + self.action_start_all_services() + elif event.button.id == "stop-all-services-btn": + self.action_stop_all_services() elif event.button.id == "open-app-btn": self.action_open_app() def action_default_action(self) -> None: """Handle Enter key - go to default action based on state.""" - if self.services_running: + if self.services_running and self.docling_running: self.action_open_app() elif self.has_oauth_config: self.action_full_setup() @@ -297,28 +285,13 @@ class WelcomeScreen(Screen): self.app.push_screen(DiagnosticsScreen()) - def action_start_stop_services(self) -> None: - """Start or stop all services (containers + docling).""" - if self.services_running: - # Stop services - show modal with progress - if self.container_manager.is_available(): - command_generator = self.container_manager.stop_services() - modal = CommandOutputModal( - "Stopping Services", - command_generator, - on_complete=self._on_services_operation_complete, - ) - self.app.push_screen(modal) - else: - # Start services - show modal with progress - if self.container_manager.is_available(): - command_generator = self.container_manager.start_services() - modal = CommandOutputModal( - "Starting Services", - command_generator, - on_complete=self._on_services_operation_complete, - ) - self.app.push_screen(modal) + def action_start_all_services(self) -> None: + """Start all services (native first, then containers).""" + self.run_worker(self._start_all_services()) + + def action_stop_all_services(self) -> None: + """Stop all services (containers first, then native).""" + self.run_worker(self._stop_all_services()) async def _on_services_operation_complete(self) -> None: """Handle completion of services start/stop operation.""" @@ -334,7 +307,7 @@ class WelcomeScreen(Screen): def _update_default_button(self) -> None: """Update the default button target based on state.""" - if self.services_running: + if self.services_running and self.docling_running: self.default_button_id = "open-app-btn" elif self.has_oauth_config: self.default_button_id = "advanced-setup-btn" @@ -362,51 +335,74 @@ class WelcomeScreen(Screen): self.call_after_refresh(self._focus_appropriate_button) - def action_start_native_services(self) -> None: - """Start native services (docling).""" - if self.docling_running: - self.notify("Native services are already running.", severity="warning") - return - - self.run_worker(self._start_native_services()) - - async def _start_native_services(self) -> None: - """Worker task to start native services.""" - try: + async def _start_all_services(self) -> None: + """Start all services: native first, then containers.""" + # Step 1: Start native services (docling-serve) + if not self.docling_manager.is_running(): + self.notify("Starting native services...", severity="information") success, message = await self.docling_manager.start() if success: - self.docling_running = True self.notify(message, severity="information") else: self.notify(f"Failed to start native services: {message}", severity="error") - except Exception as exc: - self.notify(f"Error starting native services: {exc}", severity="error") - finally: - self.docling_running = self.docling_manager.is_running() + # Continue anyway - user might want containers even if native fails + else: + self.notify("Native services already running", severity="information") + + # Update state + self.docling_running = self.docling_manager.is_running() + + # Step 2: Start container services + if self.container_manager.is_available(): + command_generator = self.container_manager.start_services() + modal = CommandOutputModal( + "Starting Container Services", + command_generator, + on_complete=self._on_services_operation_complete, + ) + self.app.push_screen(modal) + else: + self.notify("No container runtime available", severity="warning") await self._refresh_welcome_content() - def action_stop_native_services(self) -> None: - """Stop native services (docling).""" - if not self.docling_running and not self.docling_manager.is_running(): - self.notify("Native services are not running.", severity="warning") - return + async def _stop_all_services(self) -> None: + """Stop all services: containers first, then native.""" + # Step 1: Stop container services + if self.container_manager.is_available() and self.services_running: + command_generator = self.container_manager.stop_services() + modal = CommandOutputModal( + "Stopping Container Services", + command_generator, + on_complete=self._on_stop_containers_complete, + ) + self.app.push_screen(modal) + else: + # No containers to stop, go directly to stopping native services + await self._stop_native_services_after_containers() - self.run_worker(self._stop_native_services()) + async def _on_stop_containers_complete(self) -> None: + """Called after containers are stopped, now stop native services.""" + # Update container state + self._detect_services_sync() - async def _stop_native_services(self) -> None: - """Worker task to stop native services.""" - try: + # Now stop native services + await self._stop_native_services_after_containers() + + async def _stop_native_services_after_containers(self) -> None: + """Stop native services after containers have been stopped.""" + if self.docling_manager.is_running(): + self.notify("Stopping native services...", severity="information") success, message = await self.docling_manager.stop() if success: - self.docling_running = False self.notify(message, severity="information") else: self.notify(f"Failed to stop native services: {message}", severity="error") - except Exception as exc: - self.notify(f"Error stopping native services: {exc}", severity="error") - finally: - self.docling_running = self.docling_manager.is_running() - await self._refresh_welcome_content() + else: + self.notify("Native services already stopped", severity="information") + + # Update state + self.docling_running = self.docling_manager.is_running() + await self._refresh_welcome_content() def action_open_app(self) -> None: """Open the OpenRAG app in the default browser.""" From 9d1aced1e748773310e4da9b02da2d98ab8d8873 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Mon, 6 Oct 2025 12:02:36 -0500 Subject: [PATCH 48/55] refactor the grid layout to inlcude banner and header rows --- frontend/src/app/globals.css | 31 ++++++-- frontend/src/app/knowledge/chunks/page.tsx | 2 - frontend/src/components/layout-wrapper.tsx | 89 ++++++++++------------ frontend/src/contexts/layout-context.tsx | 34 --------- 4 files changed, 64 insertions(+), 92 deletions(-) delete mode 100644 frontend/src/contexts/layout-context.tsx diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css index e3f4c154..630cdf53 100644 --- a/frontend/src/app/globals.css +++ b/frontend/src/app/globals.css @@ -110,31 +110,46 @@ } @layer components { - .app-grid-cols-arrangement { + .app-grid-arrangement { --sidebar-width: 0px; --notifications-width: 0px; --filters-width: 0px; + --app-header-height: 53px; + --top-banner-height: 0px; @media (width >= 48rem) { --sidebar-width: 288px; } - &.notifications-open { --notifications-width: 320px; } &.filters-open { --filters-width: 320px; } + &.banner-visible { + --top-banner-height: 52px; + } display: grid; - height: calc(100% - var(--app-header-height)); - grid-template-columns: var(--sidebar-width) 1fr var(--notifications-width) var( - --filters-width - ); - transition: grid-template-columns 0.3s ease-in-out; + height: 100%; + width: 100%; + grid-template-rows: + var(--top-banner-height) + var(--app-header-height) + 1fr; + grid-template-columns: + var(--sidebar-width) + 1fr + var(--notifications-width) + var(--filters-width); + grid-template-areas: + "banner banner banner banner" + "header header header header" + "nav main notifications filters"; + transition: grid-template-columns 0.25s ease-in-out; } .header-arrangement { - @apply flex w-full h-[53px] items-center justify-between border-b border-border; + @apply flex w-full items-center justify-between border-b border-border; } .header-start-display { diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index e1ea9ed9..3c8b4339 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -6,8 +6,6 @@ import { useRouter, useSearchParams } from "next/navigation"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; -import { useLayout } from "@/contexts/layout-context"; -import { useTask } from "@/contexts/task-context"; import { type ChunkResult, type File, diff --git a/frontend/src/components/layout-wrapper.tsx b/frontend/src/components/layout-wrapper.tsx index b89ab5d5..dbd04fea 100644 --- a/frontend/src/components/layout-wrapper.tsx +++ b/frontend/src/components/layout-wrapper.tsx @@ -16,7 +16,6 @@ import { UserNav } from "@/components/user-nav"; import { useAuth } from "@/contexts/auth-context"; import { useChat } from "@/contexts/chat-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; -import { LayoutProvider } from "@/contexts/layout-context"; // import { GitHubStarButton } from "@/components/github-star-button" // import { DiscordLink } from "@/components/discord-link" import { useTask } from "@/contexts/task-context"; @@ -34,7 +33,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { refreshConversations, startNewConversation, } = useChat(); - const { isLoading: isSettingsLoading, data: settings } = useGetSettingsQuery({ + const { isLoading: isSettingsLoading } = useGetSettingsQuery({ enabled: isAuthenticated || isNoAuthMode, }); const { @@ -75,14 +74,6 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { const isUnhealthy = health?.status === "unhealthy" || isError; const isBannerVisible = !isHealthLoading && isUnhealthy; - // Dynamic height calculations based on banner visibility - const headerHeight = 53; - const bannerHeight = 52; // Approximate banner height - const totalTopOffset = isBannerVisible - ? headerHeight + bannerHeight - : headerHeight; - const mainContentHeight = `calc(100vh - ${totalTopOffset}px)`; - // Show loading state when backend isn't ready if (isLoading || isSettingsLoading) { return ( @@ -102,9 +93,18 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { // For all other pages, render with Langflow-styled navigation and task menu return ( -
- -
+
+
+ +
+
{/* Logo/Title */}
@@ -144,44 +144,37 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
-
- {/* Sidebar Navigation */} - - {/* Main Content */} -
-
- {children} -
-
+ {/* Sidebar Navigation */} + - {/* Task Notifications Panel */} - + {/* Main Content */} +
+
+ {children} +
+
- {/* Knowledge Filter Panel */} - -
+ {/* Task Notifications Panel */} + + + {/* Knowledge Filter Panel */} +
); } diff --git a/frontend/src/contexts/layout-context.tsx b/frontend/src/contexts/layout-context.tsx deleted file mode 100644 index f40ea28c..00000000 --- a/frontend/src/contexts/layout-context.tsx +++ /dev/null @@ -1,34 +0,0 @@ -"use client"; - -import { createContext, useContext } from "react"; - -interface LayoutContextType { - headerHeight: number; - totalTopOffset: number; -} - -const LayoutContext = createContext(undefined); - -export function useLayout() { - const context = useContext(LayoutContext); - if (context === undefined) { - throw new Error("useLayout must be used within a LayoutProvider"); - } - return context; -} - -export function LayoutProvider({ - children, - headerHeight, - totalTopOffset -}: { - children: React.ReactNode; - headerHeight: number; - totalTopOffset: number; -}) { - return ( - - {children} - - ); -} \ No newline at end of file From 983eb82d4fe79011304ea541a243a0a66d0dc363 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Mon, 6 Oct 2025 13:15:21 -0500 Subject: [PATCH 49/55] styles --- frontend/src/app/globals.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css index 630cdf53..5a2ff074 100644 --- a/frontend/src/app/globals.css +++ b/frontend/src/app/globals.css @@ -145,7 +145,8 @@ "banner banner banner banner" "header header header header" "nav main notifications filters"; - transition: grid-template-columns 0.25s ease-in-out; + transition: grid-template-columns 0.25s ease-in-out, + grid-template-rows 0.25s ease-in-out; } .header-arrangement { From 41e2d39dd061d5582466ad1340755da255008ff6 Mon Sep 17 00:00:00 2001 From: Edwin Jose Date: Mon, 6 Oct 2025 14:36:17 -0400 Subject: [PATCH 50/55] feat: Show globe icon for HTML documents in knowledge page (#188) * Show globe icon for HTML documents in knowledge page Updated getSourceIcon to display a globe icon for documents with 'text/html' mimetype, regardless of connector type. This improves visual identification of web-based documents in the grid. * changed page to get connector type as url * removed wrong docs * fix formatting * format --------- Co-authored-by: Lucas Oliveira --- frontend/src/app/knowledge/page.tsx | 651 ++++++++++++++-------------- 1 file changed, 331 insertions(+), 320 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 64eeb49c..2cd6f382 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -2,14 +2,22 @@ import type { ColDef, GetRowIdParams } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; -import { Building2, Cloud, HardDrive, Search, Trash2, X } from "lucide-react"; +import { + Building2, + Cloud, + Globe, + HardDrive, + Search, + Trash2, + X, +} from "lucide-react"; import { useRouter } from "next/navigation"; import { - type ChangeEvent, - useCallback, - useEffect, - useRef, - useState, + type ChangeEvent, + useCallback, + useEffect, + useRef, + useState, } from "react"; import { SiGoogledrive } from "react-icons/si"; import { TbBrandOnedrive } from "react-icons/tb"; @@ -31,250 +39,255 @@ import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; // Function to get the appropriate icon for a connector type function getSourceIcon(connectorType?: string) { - switch (connectorType) { - case "google_drive": - return ( - - ); - case "onedrive": - return ( - - ); - case "sharepoint": - return ; - case "s3": - return ; - default: - return ( - - ); - } + switch (connectorType) { + case "url": + return ; + case "google_drive": + return ( + + ); + case "onedrive": + return ( + + ); + case "sharepoint": + return ; + case "s3": + return ; + default: + return ( + + ); + } } function SearchPage() { - const router = useRouter(); - const { isMenuOpen, files: taskFiles, refreshTasks } = useTask(); + const router = useRouter(); + const { isMenuOpen, files: taskFiles, refreshTasks } = useTask(); const { totalTopOffset } = useLayout(); - const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = - useKnowledgeFilter(); - const [selectedRows, setSelectedRows] = useState([]); - const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); + const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = + useKnowledgeFilter(); + const [selectedRows, setSelectedRows] = useState([]); + const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); - const deleteDocumentMutation = useDeleteDocument(); + const deleteDocumentMutation = useDeleteDocument(); - useEffect(() => { - refreshTasks(); - }, [refreshTasks]); + useEffect(() => { + refreshTasks(); + }, [refreshTasks]); - const { data: searchData = [], isFetching } = useGetSearchQuery( - parsedFilterData?.query || "*", - parsedFilterData, - ); - // Convert TaskFiles to File format and merge with backend results - const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { - return { - filename: taskFile.filename, - mimetype: taskFile.mimetype, - source_url: taskFile.source_url, - size: taskFile.size, - connector_type: taskFile.connector_type, - status: taskFile.status, - }; - }); + const { data: searchData = [], isFetching } = useGetSearchQuery( + parsedFilterData?.query || "*", + parsedFilterData, + ); + // Convert TaskFiles to File format and merge with backend results + const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { + return { + filename: taskFile.filename, + mimetype: taskFile.mimetype, + source_url: taskFile.source_url, + size: taskFile.size, + connector_type: taskFile.connector_type, + status: taskFile.status, + }; + }); - // Create a map of task files by filename for quick lookup - const taskFileMap = new Map( - taskFilesAsFiles.map((file) => [file.filename, file]), - ); + // Create a map of task files by filename for quick lookup + const taskFileMap = new Map( + taskFilesAsFiles.map((file) => [file.filename, file]), + ); - // Override backend files with task file status if they exist - const backendFiles = (searchData as File[]) - .map((file) => { - const taskFile = taskFileMap.get(file.filename); - if (taskFile) { - // Override backend file with task file data (includes status) - return { ...file, ...taskFile }; - } - return file; - }) - .filter((file) => { - // Only filter out files that are currently processing AND in taskFiles - const taskFile = taskFileMap.get(file.filename); - return !taskFile || taskFile.status !== "processing"; - }); + // Override backend files with task file status if they exist + const backendFiles = (searchData as File[]) + .map((file) => { + const taskFile = taskFileMap.get(file.filename); + if (taskFile) { + // Override backend file with task file data (includes status) + return { ...file, ...taskFile }; + } + return file; + }) + .filter((file) => { + // Only filter out files that are currently processing AND in taskFiles + const taskFile = taskFileMap.get(file.filename); + return !taskFile || taskFile.status !== "processing"; + }); - const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { - return ( - taskFile.status !== "active" && - !backendFiles.some( - (backendFile) => backendFile.filename === taskFile.filename, - ) - ); - }); + const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { + return ( + taskFile.status !== "active" && + !backendFiles.some( + (backendFile) => backendFile.filename === taskFile.filename, + ) + ); + }); - // Combine task files first, then backend files - const fileResults = [...backendFiles, ...filteredTaskFiles]; + // Combine task files first, then backend files + const fileResults = [...backendFiles, ...filteredTaskFiles]; - const handleTableSearch = (e: ChangeEvent) => { - gridRef.current?.api.setGridOption("quickFilterText", e.target.value); - }; + const handleTableSearch = (e: ChangeEvent) => { + gridRef.current?.api.setGridOption("quickFilterText", e.target.value); + }; - const gridRef = useRef(null); + const gridRef = useRef(null); - const columnDefs = [ - { - field: "filename", - headerName: "Source", - checkboxSelection: (params: CustomCellRendererProps) => - (params?.data?.status || "active") === "active", - headerCheckboxSelection: true, - initialFlex: 2, - minWidth: 220, - cellRenderer: ({ data, value }: CustomCellRendererProps) => { - // Read status directly from data on each render - const status = data?.status || "active"; - const isActive = status === "active"; - console.log(data?.filename, status, "a"); - return ( -
-
- -
- ); - }, - }, - { - field: "size", - headerName: "Size", - valueFormatter: (params: CustomCellRendererProps) => - params.value ? `${Math.round(params.value / 1024)} KB` : "-", - }, - { - field: "mimetype", - headerName: "Type", - }, - { - field: "owner", - headerName: "Owner", - valueFormatter: (params: CustomCellRendererProps) => - params.data?.owner_name || params.data?.owner_email || "—", - }, - { - field: "chunkCount", - headerName: "Chunks", - valueFormatter: (params: CustomCellRendererProps) => params.data?.chunkCount?.toString() || "-", - }, - { - field: "avgScore", - headerName: "Avg score", - initialFlex: 0.5, - cellRenderer: ({ value }: CustomCellRendererProps) => { - return ( - - {value?.toFixed(2) ?? "-"} - - ); - }, - }, - { - field: "status", - headerName: "Status", - cellRenderer: ({ data }: CustomCellRendererProps) => { - console.log(data?.filename, data?.status, "b"); - // Default to 'active' status if no status is provided - const status = data?.status || "active"; - return ; - }, - }, - { - cellRenderer: ({ data }: CustomCellRendererProps) => { - const status = data?.status || "active"; - if (status !== "active") { - return null; - } - return ; - }, - cellStyle: { - alignItems: "center", - display: "flex", - justifyContent: "center", - padding: 0, - }, - colId: "actions", - filter: false, - minWidth: 0, - width: 40, - resizable: false, - sortable: false, - initialFlex: 0, - }, - ]; + const columnDefs = [ + { + field: "filename", + headerName: "Source", + checkboxSelection: (params: CustomCellRendererProps) => + (params?.data?.status || "active") === "active", + headerCheckboxSelection: true, + initialFlex: 2, + minWidth: 220, + cellRenderer: ({ data, value }: CustomCellRendererProps) => { + // Read status directly from data on each render + const status = data?.status || "active"; + const isActive = status === "active"; + console.log(data?.filename, status, "a"); + return ( +
+
+ +
+ ); + }, + }, + { + field: "size", + headerName: "Size", + valueFormatter: (params: CustomCellRendererProps) => + params.value ? `${Math.round(params.value / 1024)} KB` : "-", + }, + { + field: "mimetype", + headerName: "Type", + }, + { + field: "owner", + headerName: "Owner", + valueFormatter: (params: CustomCellRendererProps) => + params.data?.owner_name || params.data?.owner_email || "—", + }, + { + field: "chunkCount", + headerName: "Chunks", + valueFormatter: (params: CustomCellRendererProps) => + params.data?.chunkCount?.toString() || "-", + }, + { + field: "avgScore", + headerName: "Avg score", + initialFlex: 0.5, + cellRenderer: ({ value }: CustomCellRendererProps) => { + return ( + + {value?.toFixed(2) ?? "-"} + + ); + }, + }, + { + field: "status", + headerName: "Status", + cellRenderer: ({ data }: CustomCellRendererProps) => { + console.log(data?.filename, data?.status, "b"); + // Default to 'active' status if no status is provided + const status = data?.status || "active"; + return ; + }, + }, + { + cellRenderer: ({ data }: CustomCellRendererProps) => { + const status = data?.status || "active"; + if (status !== "active") { + return null; + } + return ; + }, + cellStyle: { + alignItems: "center", + display: "flex", + justifyContent: "center", + padding: 0, + }, + colId: "actions", + filter: false, + minWidth: 0, + width: 40, + resizable: false, + sortable: false, + initialFlex: 0, + }, + ]; - const defaultColDef: ColDef = { - resizable: false, - suppressMovable: true, - initialFlex: 1, - minWidth: 100, - }; + const defaultColDef: ColDef = { + resizable: false, + suppressMovable: true, + initialFlex: 1, + minWidth: 100, + }; - const onSelectionChanged = useCallback(() => { - if (gridRef.current) { - const selectedNodes = gridRef.current.api.getSelectedRows(); - setSelectedRows(selectedNodes); - } - }, []); + const onSelectionChanged = useCallback(() => { + if (gridRef.current) { + const selectedNodes = gridRef.current.api.getSelectedRows(); + setSelectedRows(selectedNodes); + } + }, []); - const handleBulkDelete = async () => { - if (selectedRows.length === 0) return; + const handleBulkDelete = async () => { + if (selectedRows.length === 0) return; - try { - // Delete each file individually since the API expects one filename at a time - const deletePromises = selectedRows.map((row) => - deleteDocumentMutation.mutateAsync({ filename: row.filename }), - ); + try { + // Delete each file individually since the API expects one filename at a time + const deletePromises = selectedRows.map((row) => + deleteDocumentMutation.mutateAsync({ filename: row.filename }), + ); - await Promise.all(deletePromises); + await Promise.all(deletePromises); - toast.success( - `Successfully deleted ${selectedRows.length} document${ - selectedRows.length > 1 ? "s" : "" - }`, - ); - setSelectedRows([]); - setShowBulkDeleteDialog(false); + toast.success( + `Successfully deleted ${selectedRows.length} document${ + selectedRows.length > 1 ? "s" : "" + }`, + ); + setSelectedRows([]); + setShowBulkDeleteDialog(false); - // Clear selection in the grid - if (gridRef.current) { - gridRef.current.api.deselectAll(); - } - } catch (error) { - toast.error( - error instanceof Error - ? error.message - : "Failed to delete some documents", - ); - } - }; + // Clear selection in the grid + if (gridRef.current) { + gridRef.current.api.deselectAll(); + } + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Failed to delete some documents", + ); + } + }; return (
- {/* Search Input Area */} -
-
-
- {selectedFilter?.name && ( -
- {selectedFilter?.name} - setSelectedFilter(null)} - /> -
- )} - - -
- {/* */} - {/* //TODO: Implement sync button */} - {/* */} - {selectedRows.length > 0 && ( - - )} -
-
- []} - defaultColDef={defaultColDef} - loading={isFetching} - ref={gridRef} - rowData={fileResults} - rowSelection="multiple" - rowMultiSelectWithClick={false} - suppressRowClickSelection={true} - getRowId={(params: GetRowIdParams) => params.data?.filename} - domLayout="normal" - onSelectionChanged={onSelectionChanged} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> -
+ {selectedRows.length > 0 && ( + + )} + +
+ []} + defaultColDef={defaultColDef} + loading={isFetching} + ref={gridRef} + rowData={fileResults} + rowSelection="multiple" + rowMultiSelectWithClick={false} + suppressRowClickSelection={true} + getRowId={(params: GetRowIdParams) => params.data?.filename} + domLayout="normal" + onSelectionChanged={onSelectionChanged} + noRowsOverlayComponent={() => ( +
+
+ No knowledge +
+
+ Add files from local or your preferred cloud. +
+
+ )} + /> +
- {/* Bulk Delete Confirmation Dialog */} - 1 ? "s" : "" - }? This will remove all chunks and data associated with these documents. This action cannot be undone. + {/* Bulk Delete Confirmation Dialog */} + 1 ? "s" : "" + }? This will remove all chunks and data associated with these documents. This action cannot be undone. Documents to be deleted: ${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`} - confirmText="Delete All" - onConfirm={handleBulkDelete} - isLoading={deleteDocumentMutation.isPending} - /> -
- ); + confirmText="Delete All" + onConfirm={handleBulkDelete} + isLoading={deleteDocumentMutation.isPending} + /> +
+ ); } export default function ProtectedSearchPage() { - return ( - - - - ); + return ( + + + + ); } From c450ecc50afefcb262b445f08293b423e0daff0a Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 6 Oct 2025 14:46:51 -0400 Subject: [PATCH 51/55] tui: docling pid and host detection --- .gitignore | 2 + src/tui/managers/docling_manager.py | 331 +++++++++++++++++++++++----- src/tui/screens/welcome.py | 42 ++-- 3 files changed, 298 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 9c99e617..484db58d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ wheels/ .DS_Store config/ + +.docling.pid diff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.py index 5b1100cc..6fecfff9 100644 --- a/src/tui/managers/docling_manager.py +++ b/src/tui/managers/docling_manager.py @@ -31,10 +31,14 @@ class DoclingManager: self._process: Optional[subprocess.Popen] = None self._port = 5001 - self._host = "127.0.0.1" + self._host = self._get_host_for_containers() # Get appropriate host IP based on runtime self._running = False self._external_process = False + # PID file to track docling-serve across sessions (in current working directory) + from pathlib import Path + self._pid_file = Path.cwd() / ".docling.pid" + # Log storage - simplified, no queue self._log_buffer: List[str] = [] self._max_log_lines = 1000 @@ -42,22 +46,198 @@ class DoclingManager: self._initialized = True - def cleanup(self): - """Cleanup resources and stop any running processes.""" - if self._process and self._process.poll() is None: - self._add_log_entry("Cleaning up docling-serve process on exit") - try: - self._process.terminate() - self._process.wait(timeout=5) - except subprocess.TimeoutExpired: - self._process.kill() - self._process.wait() - except Exception as e: - self._add_log_entry(f"Error during cleanup: {e}") + # Try to recover existing process from PID file + self._recover_from_pid_file() - self._running = False - self._process = None + def _get_host_for_containers(self) -> str: + """ + Return a host IP that containers can reach (a bridge/CNI gateway). + Prefers Docker/Podman network gateways; falls back to bridge interfaces. + """ + import subprocess, json, shutil, re, logging + logger = logging.getLogger(__name__) + + def run(cmd, timeout=2, text=True): + return subprocess.run(cmd, capture_output=True, text=text, timeout=timeout) + + gateways = [] + compose_gateways = [] # Highest priority - compose project networks + active_gateways = [] # Medium priority - networks with containers + + # ---- Docker: enumerate networks and collect gateways + if shutil.which("docker"): + try: + ls = run(["docker", "network", "ls", "--format", "{{.Name}}"]) + if ls.returncode == 0: + for name in filter(None, ls.stdout.splitlines()): + try: + insp = run(["docker", "network", "inspect", name, "--format", "{{json .}}"]) + if insp.returncode == 0 and insp.stdout.strip(): + nw = json.loads(insp.stdout)[0] if insp.stdout.strip().startswith("[") else json.loads(insp.stdout) + ipam = nw.get("IPAM", {}) + containers = nw.get("Containers", {}) + for cfg in ipam.get("Config", []) or []: + gw = cfg.get("Gateway") + if gw: + # Highest priority: compose networks (ending in _default) + if name.endswith("_default"): + compose_gateways.append(gw) + # Medium priority: networks with active containers + elif len(containers) > 0: + active_gateways.append(gw) + # Low priority: empty networks + else: + gateways.append(gw) + except Exception: + pass + except Exception: + pass + + # ---- Podman: enumerate networks and collect gateways (netavark) + if shutil.which("podman"): + try: + # modern podman supports JSON format + ls = run(["podman", "network", "ls", "--format", "json"]) + if ls.returncode == 0 and ls.stdout.strip(): + for net in json.loads(ls.stdout): + name = net.get("name") or net.get("Name") + if not name: + continue + try: + insp = run(["podman", "network", "inspect", name, "--format", "json"]) + if insp.returncode == 0 and insp.stdout.strip(): + arr = json.loads(insp.stdout) + for item in (arr if isinstance(arr, list) else [arr]): + for sn in item.get("subnets", []) or []: + gw = sn.get("gateway") + if gw: + # Prioritize compose/project networks + if name.endswith("_default") or "_" in name: + compose_gateways.append(gw) + else: + gateways.append(gw) + except Exception: + pass + except Exception: + pass + + # ---- Fallback: parse host interfaces for common bridges + if not gateways: + try: + if shutil.which("ip"): + show = run(["ip", "-o", "-4", "addr", "show"]) + if show.returncode == 0: + for line in show.stdout.splitlines(): + # e.g. "12: br-3f0f... inet 172.18.0.1/16 ..." + m = re.search(r"^\d+:\s+([a-zA-Z0-9_.:-]+)\s+.*\binet\s+(\d+\.\d+\.\d+\.\d+)/", line) + if not m: + continue + ifname, ip = m.group(1), m.group(2) + if ifname == "docker0" or ifname.startswith(("br-", "cni")): + gateways.append(ip) + else: + # As a last resort, try net-tools ifconfig output + if shutil.which("ifconfig"): + show = run(["ifconfig"]) + for block in show.stdout.split("\n\n"): + if any(block.strip().startswith(n) for n in ("docker0", "cni", "br-")): + m = re.search(r"inet (?:addr:)?(\d+\.\d+\.\d+\.\d+)", block) + if m: + gateways.append(m.group(1)) + except Exception: + pass + + # Dedup, prioritizing: 1) compose networks, 2) active networks, 3) all others + seen, uniq = set(), [] + # First: compose project networks (_default suffix) + for ip in compose_gateways: + if ip not in seen: + uniq.append(ip) + seen.add(ip) + # Second: networks with active containers + for ip in active_gateways: + if ip not in seen: + uniq.append(ip) + seen.add(ip) + # Third: all other gateways + for ip in gateways: + if ip not in seen: + uniq.append(ip) + seen.add(ip) + + if uniq: + if len(uniq) > 1: + logger.info("Container-reachable host IP candidates: %s", ", ".join(uniq)) + else: + logger.info("Container-reachable host IP: %s", uniq[0]) + return uniq[0] + + # Nothing found: warn clearly + logger.warning( + "No container bridge IP found. If using rootless Podman (slirp4netns), there is no host bridge; publish ports or use 10.0.2.2 from the container." + ) + # Returning localhost is honest only for same-namespace; keep it explicit: + return "127.0.0.1" + + def cleanup(self): + """Cleanup resources but keep docling-serve running across sessions.""" + # Don't stop the process on exit - let it persist + # Just clean up our references + self._add_log_entry("TUI exiting - docling-serve will continue running") + # Note: We keep the PID file so we can reconnect in future sessions + def _save_pid(self, pid: int) -> None: + """Save the process PID to a file for persistence across sessions.""" + try: + self._pid_file.write_text(str(pid)) + self._add_log_entry(f"Saved PID {pid} to {self._pid_file}") + except Exception as e: + self._add_log_entry(f"Failed to save PID file: {e}") + + def _load_pid(self) -> Optional[int]: + """Load the process PID from file.""" + try: + if self._pid_file.exists(): + pid_str = self._pid_file.read_text().strip() + if pid_str.isdigit(): + return int(pid_str) + except Exception as e: + self._add_log_entry(f"Failed to load PID file: {e}") + return None + + def _clear_pid_file(self) -> None: + """Remove the PID file.""" + try: + if self._pid_file.exists(): + self._pid_file.unlink() + self._add_log_entry("Cleared PID file") + except Exception as e: + self._add_log_entry(f"Failed to clear PID file: {e}") + + def _is_process_running(self, pid: int) -> bool: + """Check if a process with the given PID is running.""" + try: + # Send signal 0 to check if process exists (doesn't actually send a signal) + os.kill(pid, 0) + return True + except OSError: + return False + + def _recover_from_pid_file(self) -> None: + """Try to recover connection to existing docling-serve process from PID file.""" + pid = self._load_pid() + if pid is not None: + if self._is_process_running(pid): + self._add_log_entry(f"Recovered existing docling-serve process (PID: {pid})") + # Mark as external process since we didn't start it in this session + self._external_process = True + self._running = True + # Note: We don't have a Popen object for this process, but that's OK + # We'll detect it's running via port check + else: + self._add_log_entry(f"Stale PID file found (PID: {pid} not running)") + self._clear_pid_file() + def _add_log_entry(self, message: str) -> None: """Add a log entry to the buffer (thread-safe).""" timestamp = time.strftime("%Y-%m-%d %H:%M:%S") @@ -70,43 +250,35 @@ class DoclingManager: self._log_buffer = self._log_buffer[-self._max_log_lines:] def is_running(self) -> bool: - """Check if docling serve is running.""" - # First check our internal state - internal_running = self._running and self._process is not None and self._process.poll() is None - - # If we think it's not running, check if something is listening on the port - # This handles cases where docling-serve was started outside the TUI - if not internal_running: - try: - import socket - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(0.5) - result = s.connect_ex((self._host, self._port)) - s.close() - - # If port is in use, something is running there - if result == 0: - # Only log this once when we first detect external process - if not self._external_process: - self._add_log_entry(f"Detected external docling-serve running on {self._host}:{self._port}") - # Set a flag to indicate this is an external process - self._external_process = True - return True - except Exception as e: - # Only log errors occasionally to avoid spam - if not hasattr(self, '_last_port_error') or self._last_port_error != str(e): - self._add_log_entry(f"Error checking port: {e}") - self._last_port_error = str(e) - else: - # If we started it, it's not external + """Check if docling serve is running (by PID only).""" + # Check if we have a direct process handle + if self._process is not None and self._process.poll() is None: + self._running = True self._external_process = False + return True - return internal_running + # Check if we have a PID from file + pid = self._load_pid() + if pid is not None and self._is_process_running(pid): + self._running = True + self._external_process = True + return True + + # No running process found + self._running = False + self._external_process = False + return False def get_status(self) -> Dict[str, Any]: """Get current status of docling serve.""" if self.is_running(): - pid = self._process.pid if self._process else None + # Try to get PID from process handle first, then from PID file + pid = None + if self._process: + pid = self._process.pid + else: + pid = self._load_pid() + return { "status": "running", "port": self._port, @@ -127,13 +299,28 @@ class DoclingManager: "pid": None } - async def start(self, port: int = 5001, host: str = "127.0.0.1", enable_ui: bool = False) -> Tuple[bool, str]: + async def start(self, port: int = 5001, host: Optional[str] = None, enable_ui: bool = False) -> Tuple[bool, str]: """Start docling serve as external process.""" if self.is_running(): return False, "Docling serve is already running" self._port = port - self._host = host + # Use provided host or the bridge IP we detected in __init__ + if host is not None: + self._host = host + # else: keep self._host as already set in __init__ + + # Check if port is already in use before trying to start + import socket + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(0.5) + result = s.connect_ex((self._host, self._port)) + s.close() + if result == 0: + return False, f"Port {self._port} on {self._host} is already in use by another process. Please stop it first." + except Exception as e: + self._add_log_entry(f"Error checking port availability: {e}") # Clear log buffer when starting self._log_buffer = [] @@ -146,14 +333,14 @@ class DoclingManager: if shutil.which("uv") and (os.path.exists("pyproject.toml") or os.getenv("VIRTUAL_ENV")): cmd = [ "uv", "run", "python", "-m", "docling_serve", "run", - "--host", host, - "--port", str(port), + "--host", self._host, + "--port", str(self._port), ] else: cmd = [ sys.executable, "-m", "docling_serve", "run", - "--host", host, - "--port", str(port), + "--host", self._host, + "--port", str(self._port), ] if enable_ui: @@ -173,6 +360,9 @@ class DoclingManager: self._running = True self._add_log_entry("External process started") + # Save the PID to file for persistence + self._save_pid(self._process.pid) + # Start a thread to capture output self._start_output_capture() @@ -192,11 +382,11 @@ class DoclingManager: import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(0.5) - result = s.connect_ex((host, port)) + result = s.connect_ex((self._host, self._port)) s.close() if result == 0: - self._add_log_entry(f"Docling-serve is now listening on {host}:{port}") + self._add_log_entry(f"Docling-serve is now listening on {self._host}:{self._port}") break except: pass @@ -298,9 +488,12 @@ class DoclingManager: try: self._add_log_entry("Stopping docling-serve process") + pid_to_stop = None + if self._process: - # We started this process, so we can stop it directly - self._add_log_entry(f"Terminating our process (PID: {self._process.pid})") + # We have a direct process handle + pid_to_stop = self._process.pid + self._add_log_entry(f"Terminating our process (PID: {pid_to_stop})") self._process.terminate() # Wait for it to stop @@ -315,16 +508,32 @@ class DoclingManager: self._add_log_entry("Process force killed") elif self._external_process: - # This is an external process, we can't stop it directly - self._add_log_entry("Cannot stop external docling-serve process - it was started outside the TUI") - self._running = False - self._external_process = False - return False, "Cannot stop external docling-serve process. Please stop it manually." + # This is a process we recovered from PID file + pid_to_stop = self._load_pid() + if pid_to_stop and self._is_process_running(pid_to_stop): + self._add_log_entry(f"Stopping process from PID file (PID: {pid_to_stop})") + try: + os.kill(pid_to_stop, 15) # SIGTERM + # Wait a bit for graceful shutdown + await asyncio.sleep(2) + if self._is_process_running(pid_to_stop): + # Still running, force kill + self._add_log_entry(f"Force killing process (PID: {pid_to_stop})") + os.kill(pid_to_stop, 9) # SIGKILL + except Exception as e: + self._add_log_entry(f"Error stopping external process: {e}") + return False, f"Error stopping external process: {str(e)}" + else: + self._add_log_entry("External process not found") + return False, "Process not found" self._running = False self._process = None self._external_process = False + # Clear the PID file since we intentionally stopped the service + self._clear_pid_file() + self._add_log_entry("Docling serve stopped successfully") return True, "Docling serve stopped successfully" diff --git a/src/tui/screens/welcome.py b/src/tui/screens/welcome.py index 9c121022..217b0611 100644 --- a/src/tui/screens/welcome.py +++ b/src/tui/screens/welcome.py @@ -336,8 +336,31 @@ class WelcomeScreen(Screen): self.call_after_refresh(self._focus_appropriate_button) async def _start_all_services(self) -> None: - """Start all services: native first, then containers.""" - # Step 1: Start native services (docling-serve) + """Start all services: containers first, then native services.""" + # Step 1: Start container services first (to create the network) + if self.container_manager.is_available(): + command_generator = self.container_manager.start_services() + modal = CommandOutputModal( + "Starting Container Services", + command_generator, + on_complete=self._on_containers_started_start_native, + ) + self.app.push_screen(modal) + else: + self.notify("No container runtime available", severity="warning") + # Still try to start native services + await self._start_native_services_after_containers() + + async def _on_containers_started_start_native(self) -> None: + """Called after containers start successfully, now start native services.""" + # Update container state + self._detect_services_sync() + + # Now start native services (docling-serve can now detect the compose network) + await self._start_native_services_after_containers() + + async def _start_native_services_after_containers(self) -> None: + """Start native services after containers have been started.""" if not self.docling_manager.is_running(): self.notify("Starting native services...", severity="information") success, message = await self.docling_manager.start() @@ -345,25 +368,12 @@ class WelcomeScreen(Screen): self.notify(message, severity="information") else: self.notify(f"Failed to start native services: {message}", severity="error") - # Continue anyway - user might want containers even if native fails else: self.notify("Native services already running", severity="information") # Update state self.docling_running = self.docling_manager.is_running() - - # Step 2: Start container services - if self.container_manager.is_available(): - command_generator = self.container_manager.start_services() - modal = CommandOutputModal( - "Starting Container Services", - command_generator, - on_complete=self._on_services_operation_complete, - ) - self.app.push_screen(modal) - else: - self.notify("No container runtime available", severity="warning") - await self._refresh_welcome_content() + await self._refresh_welcome_content() async def _stop_all_services(self) -> None: """Stop all services: containers first, then native.""" From 99fd1d7215823e9ea862b2a8e10802035d27c9fa Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Mon, 6 Oct 2025 13:49:45 -0500 Subject: [PATCH 52/55] fix merge conflicts --- frontend/components/knowledge-dropdown.tsx | 60 +- frontend/src/app/knowledge/page.tsx | 656 ++++++++++----------- 2 files changed, 310 insertions(+), 406 deletions(-) diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index 7fe84259..d9f92355 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -7,7 +7,6 @@ import { FolderOpen, Loader2, PlugZap, - Plus, Upload, } from "lucide-react"; import { useRouter } from "next/navigation"; @@ -29,15 +28,7 @@ import { useTask } from "@/contexts/task-context"; import { cn } from "@/lib/utils"; import type { File as SearchFile } from "@/src/app/api/queries/useGetSearchQuery"; -interface KnowledgeDropdownProps { - active?: boolean; - variant?: "navigation" | "button"; -} - -export function KnowledgeDropdown({ - active, - variant = "navigation", -}: KnowledgeDropdownProps) { +export function KnowledgeDropdown() { const { addTask } = useTask(); const { refetch: refetchTasks } = useGetTasksQuery(); const queryClient = useQueryClient(); @@ -498,28 +489,16 @@ export function KnowledgeDropdown({ return ( <>
- + {isOpen && !isLoading && (
diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 64eeb49c..e7dbac08 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -2,22 +2,19 @@ import type { ColDef, GetRowIdParams } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; -import { Building2, Cloud, HardDrive, Search, Trash2, X } from "lucide-react"; +import { Cloud, FileIcon, Search, Trash2, X } from "lucide-react"; import { useRouter } from "next/navigation"; import { - type ChangeEvent, - useCallback, - useEffect, - useRef, - useState, + type ChangeEvent, + useCallback, + useEffect, + useRef, + useState, } from "react"; -import { SiGoogledrive } from "react-icons/si"; -import { TbBrandOnedrive } from "react-icons/tb"; import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; -import { useLayout } from "@/contexts/layout-context"; import { useTask } from "@/contexts/task-context"; import { type File, useGetSearchQuery } from "../api/queries/useGetSearchQuery"; import "@/components/AgGrid/registerAgGridModules"; @@ -28,319 +25,272 @@ import { filterAccentClasses } from "@/components/knowledge-filter-panel"; import { StatusBadge } from "@/components/ui/status-badge"; import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog"; import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; +import GoogleDriveIcon from "../settings/icons/google-drive-icon"; +import OneDriveIcon from "../settings/icons/one-drive-icon"; +import SharePointIcon from "../settings/icons/share-point-icon"; +import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; // Function to get the appropriate icon for a connector type function getSourceIcon(connectorType?: string) { - switch (connectorType) { - case "google_drive": - return ( - - ); - case "onedrive": - return ( - - ); - case "sharepoint": - return ; - case "s3": - return ; - default: - return ( - - ); - } + switch (connectorType) { + case "google_drive": + return ( + + ); + case "onedrive": + return ; + case "sharepoint": + return ( + + ); + case "s3": + return ; + default: + return ( + + ); + } } function SearchPage() { - const router = useRouter(); - const { isMenuOpen, files: taskFiles, refreshTasks } = useTask(); - const { totalTopOffset } = useLayout(); - const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = - useKnowledgeFilter(); - const [selectedRows, setSelectedRows] = useState([]); - const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); + const router = useRouter(); + const { isMenuOpen, files: taskFiles, refreshTasks } = useTask(); + const { + selectedFilter, + setSelectedFilter, + parsedFilterData, + isPanelOpen, + queryOverride, + } = useKnowledgeFilter(); + const [selectedRows, setSelectedRows] = useState([]); + const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); - const deleteDocumentMutation = useDeleteDocument(); + const deleteDocumentMutation = useDeleteDocument(); - useEffect(() => { - refreshTasks(); - }, [refreshTasks]); + useEffect(() => { + refreshTasks(); + }, [refreshTasks]); - const { data: searchData = [], isFetching } = useGetSearchQuery( - parsedFilterData?.query || "*", - parsedFilterData, - ); - // Convert TaskFiles to File format and merge with backend results - const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { - return { - filename: taskFile.filename, - mimetype: taskFile.mimetype, - source_url: taskFile.source_url, - size: taskFile.size, - connector_type: taskFile.connector_type, - status: taskFile.status, - }; - }); + const { data: searchData = [], isFetching } = useGetSearchQuery( + queryOverride, + parsedFilterData + ); + // Convert TaskFiles to File format and merge with backend results + const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { + return { + filename: taskFile.filename, + mimetype: taskFile.mimetype, + source_url: taskFile.source_url, + size: taskFile.size, + connector_type: taskFile.connector_type, + status: taskFile.status, + }; + }); - // Create a map of task files by filename for quick lookup - const taskFileMap = new Map( - taskFilesAsFiles.map((file) => [file.filename, file]), - ); + // Create a map of task files by filename for quick lookup + const taskFileMap = new Map( + taskFilesAsFiles.map((file) => [file.filename, file]) + ); - // Override backend files with task file status if they exist - const backendFiles = (searchData as File[]) - .map((file) => { - const taskFile = taskFileMap.get(file.filename); - if (taskFile) { - // Override backend file with task file data (includes status) - return { ...file, ...taskFile }; - } - return file; - }) - .filter((file) => { - // Only filter out files that are currently processing AND in taskFiles - const taskFile = taskFileMap.get(file.filename); - return !taskFile || taskFile.status !== "processing"; - }); + // Override backend files with task file status if they exist + const backendFiles = (searchData as File[]) + .map((file) => { + const taskFile = taskFileMap.get(file.filename); + if (taskFile) { + // Override backend file with task file data (includes status) + return { ...file, ...taskFile }; + } + return file; + }) + .filter((file) => { + // Only filter out files that are currently processing AND in taskFiles + const taskFile = taskFileMap.get(file.filename); + return !taskFile || taskFile.status !== "processing"; + }); - const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { - return ( - taskFile.status !== "active" && - !backendFiles.some( - (backendFile) => backendFile.filename === taskFile.filename, - ) - ); - }); + const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { + return ( + taskFile.status !== "active" && + !backendFiles.some( + (backendFile) => backendFile.filename === taskFile.filename + ) + ); + }); - // Combine task files first, then backend files - const fileResults = [...backendFiles, ...filteredTaskFiles]; + // Combine task files first, then backend files + const fileResults = [...backendFiles, ...filteredTaskFiles]; - const handleTableSearch = (e: ChangeEvent) => { - gridRef.current?.api.setGridOption("quickFilterText", e.target.value); - }; + const gridRef = useRef(null); - const gridRef = useRef(null); + const columnDefs = [ + { + field: "filename", + headerName: "Source", + checkboxSelection: (params: CustomCellRendererProps) => + (params?.data?.status || "active") === "active", + headerCheckboxSelection: true, + initialFlex: 2, + minWidth: 220, + cellRenderer: ({ data, value }: CustomCellRendererProps) => { + // Read status directly from data on each render + const status = data?.status || "active"; + const isActive = status === "active"; + console.log(data?.filename, status, "a"); + return ( +
+
+ +
+ ); + }, + }, + { + field: "size", + headerName: "Size", + valueFormatter: (params: CustomCellRendererProps) => + params.value ? `${Math.round(params.value / 1024)} KB` : "-", + }, + { + field: "mimetype", + headerName: "Type", + }, + { + field: "owner", + headerName: "Owner", + valueFormatter: (params: CustomCellRendererProps) => + params.data?.owner_name || params.data?.owner_email || "—", + }, + { + field: "chunkCount", + headerName: "Chunks", + valueFormatter: (params: CustomCellRendererProps) => + params.data?.chunkCount?.toString() || "-", + }, + { + field: "avgScore", + headerName: "Avg score", + cellRenderer: ({ value }: CustomCellRendererProps) => { + return ( + + {value?.toFixed(2) ?? "-"} + + ); + }, + }, + { + field: "status", + headerName: "Status", + cellRenderer: ({ data }: CustomCellRendererProps) => { + console.log(data?.filename, data?.status, "b"); + // Default to 'active' status if no status is provided + const status = data?.status || "active"; + return ; + }, + }, + { + cellRenderer: ({ data }: CustomCellRendererProps) => { + const status = data?.status || "active"; + if (status !== "active") { + return null; + } + return ; + }, + cellStyle: { + alignItems: "center", + display: "flex", + justifyContent: "center", + padding: 0, + }, + colId: "actions", + filter: false, + minWidth: 0, + width: 40, + resizable: false, + sortable: false, + initialFlex: 0, + }, + ]; - const columnDefs = [ - { - field: "filename", - headerName: "Source", - checkboxSelection: (params: CustomCellRendererProps) => - (params?.data?.status || "active") === "active", - headerCheckboxSelection: true, - initialFlex: 2, - minWidth: 220, - cellRenderer: ({ data, value }: CustomCellRendererProps) => { - // Read status directly from data on each render - const status = data?.status || "active"; - const isActive = status === "active"; - console.log(data?.filename, status, "a"); - return ( -
-
- -
- ); - }, - }, - { - field: "size", - headerName: "Size", - valueFormatter: (params: CustomCellRendererProps) => - params.value ? `${Math.round(params.value / 1024)} KB` : "-", - }, - { - field: "mimetype", - headerName: "Type", - }, - { - field: "owner", - headerName: "Owner", - valueFormatter: (params: CustomCellRendererProps) => - params.data?.owner_name || params.data?.owner_email || "—", - }, - { - field: "chunkCount", - headerName: "Chunks", - valueFormatter: (params: CustomCellRendererProps) => params.data?.chunkCount?.toString() || "-", - }, - { - field: "avgScore", - headerName: "Avg score", - initialFlex: 0.5, - cellRenderer: ({ value }: CustomCellRendererProps) => { - return ( - - {value?.toFixed(2) ?? "-"} - - ); - }, - }, - { - field: "status", - headerName: "Status", - cellRenderer: ({ data }: CustomCellRendererProps) => { - console.log(data?.filename, data?.status, "b"); - // Default to 'active' status if no status is provided - const status = data?.status || "active"; - return ; - }, - }, - { - cellRenderer: ({ data }: CustomCellRendererProps) => { - const status = data?.status || "active"; - if (status !== "active") { - return null; - } - return ; - }, - cellStyle: { - alignItems: "center", - display: "flex", - justifyContent: "center", - padding: 0, - }, - colId: "actions", - filter: false, - minWidth: 0, - width: 40, - resizable: false, - sortable: false, - initialFlex: 0, - }, - ]; + const defaultColDef: ColDef = { + resizable: false, + suppressMovable: true, + initialFlex: 1, + minWidth: 100, + }; - const defaultColDef: ColDef = { - resizable: false, - suppressMovable: true, - initialFlex: 1, - minWidth: 100, - }; + const onSelectionChanged = useCallback(() => { + if (gridRef.current) { + const selectedNodes = gridRef.current.api.getSelectedRows(); + setSelectedRows(selectedNodes); + } + }, []); - const onSelectionChanged = useCallback(() => { - if (gridRef.current) { - const selectedNodes = gridRef.current.api.getSelectedRows(); - setSelectedRows(selectedNodes); - } - }, []); + const handleBulkDelete = async () => { + if (selectedRows.length === 0) return; - const handleBulkDelete = async () => { - if (selectedRows.length === 0) return; + try { + // Delete each file individually since the API expects one filename at a time + const deletePromises = selectedRows.map((row) => + deleteDocumentMutation.mutateAsync({ filename: row.filename }) + ); - try { - // Delete each file individually since the API expects one filename at a time - const deletePromises = selectedRows.map((row) => - deleteDocumentMutation.mutateAsync({ filename: row.filename }), - ); + await Promise.all(deletePromises); - await Promise.all(deletePromises); + toast.success( + `Successfully deleted ${selectedRows.length} document${ + selectedRows.length > 1 ? "s" : "" + }` + ); + setSelectedRows([]); + setShowBulkDeleteDialog(false); - toast.success( - `Successfully deleted ${selectedRows.length} document${ - selectedRows.length > 1 ? "s" : "" - }`, - ); - setSelectedRows([]); - setShowBulkDeleteDialog(false); - - // Clear selection in the grid - if (gridRef.current) { - gridRef.current.api.deselectAll(); - } - } catch (error) { - toast.error( - error instanceof Error - ? error.message - : "Failed to delete some documents", - ); - } - }; + // Clear selection in the grid + if (gridRef.current) { + gridRef.current.api.deselectAll(); + } + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Failed to delete some documents" + ); + } + }; return ( -
-
+ <> +

Project Knowledge

-
- {/* Search Input Area */} -
-
-
- {selectedFilter?.name && ( -
- {selectedFilter?.name} - setSelectedFilter(null)} - /> -
- )} - - -
- {/* */} - {/* //TODO: Implement sync button */} - {/* */} - {selectedRows.length > 0 && ( - - )} -
-
- []} - defaultColDef={defaultColDef} - loading={isFetching} - ref={gridRef} - rowData={fileResults} - rowSelection="multiple" - rowMultiSelectWithClick={false} - suppressRowClickSelection={true} - getRowId={(params: GetRowIdParams) => params.data?.filename} - domLayout="normal" - onSelectionChanged={onSelectionChanged} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> -
+ {selectedRows.length > 0 && ( + + )} +
+ +
+
+ []} + defaultColDef={defaultColDef} + loading={isFetching} + ref={gridRef} + rowData={fileResults} + rowSelection="multiple" + rowMultiSelectWithClick={false} + suppressRowClickSelection={true} + getRowId={(params: GetRowIdParams) => params.data?.filename} + domLayout="normal" + onSelectionChanged={onSelectionChanged} + noRowsOverlayComponent={() => ( +
+
+ No knowledge +
+
+ Add files from local or your preferred cloud. +
+
+ )} + /> +
- {/* Bulk Delete Confirmation Dialog */} - 1 ? "s" : "" - }? This will remove all chunks and data associated with these documents. This action cannot be undone. + {/* Bulk Delete Confirmation Dialog */} + 1 ? "s" : "" + }? This will remove all chunks and data associated with these documents. This action cannot be undone. Documents to be deleted: ${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`} - confirmText="Delete All" - onConfirm={handleBulkDelete} - isLoading={deleteDocumentMutation.isPending} - /> -
- ); + confirmText="Delete All" + onConfirm={handleBulkDelete} + isLoading={deleteDocumentMutation.isPending} + /> + + ); } export default function ProtectedSearchPage() { - return ( - - - - ); + return ( + + + + ); } From 984b9c419529bf871a6553ad574eeb58c3b57252 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Mon, 6 Oct 2025 14:00:09 -0500 Subject: [PATCH 53/55] import --- frontend/src/app/knowledge/page.tsx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 2f9f42b2..d4aebec8 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -1,16 +1,14 @@ "use client"; -import type { ColDef, GetRowIdParams } from "ag-grid-community"; +import { + themeQuartz, + type ColDef, + type GetRowIdParams, +} from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; import { Cloud, FileIcon, Globe } from "lucide-react"; import { useRouter } from "next/navigation"; -import { - type ChangeEvent, - useCallback, - useEffect, - useRef, - useState, -} from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; @@ -314,6 +312,7 @@ function SearchPage() { defaultColDef={defaultColDef} loading={isFetching} ref={gridRef} + theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} rowData={fileResults} rowSelection="multiple" rowMultiSelectWithClick={false} From 80eac4824cdc0fc0ad37af749c5cc08b4f46b562 Mon Sep 17 00:00:00 2001 From: Cole Goldsmith Date: Mon, 6 Oct 2025 14:05:10 -0500 Subject: [PATCH 54/55] remove css --- frontend/src/app/globals.css | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css index 5a2ff074..56dc8dc8 100644 --- a/frontend/src/app/globals.css +++ b/frontend/src/app/globals.css @@ -53,8 +53,6 @@ --flow-icon: #2f67d0; --radius: 0.5rem; - - --app-header-height: 40px; } .dark { From 6f32890d76b8d174359461434d24f7b19144f4db Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 6 Oct 2025 15:54:21 -0400 Subject: [PATCH 55/55] docling backend check --- .../app/api/queries/useDoclingHealthQuery.ts | 3 +- src/main.py | 7 + src/tui/managers/docling_manager.py | 133 +---------------- src/utils/container_utils.py | 135 ++++++++++++++++++ 4 files changed, 146 insertions(+), 132 deletions(-) diff --git a/frontend/src/app/api/queries/useDoclingHealthQuery.ts b/frontend/src/app/api/queries/useDoclingHealthQuery.ts index 16ffc6c5..8db560d0 100644 --- a/frontend/src/app/api/queries/useDoclingHealthQuery.ts +++ b/frontend/src/app/api/queries/useDoclingHealthQuery.ts @@ -16,7 +16,8 @@ export const useDoclingHealthQuery = ( async function checkDoclingHealth(): Promise { try { - const response = await fetch("http://127.0.0.1:5001/health", { + // Call backend proxy endpoint instead of direct localhost + const response = await fetch("/api/docling/health", { method: "GET", headers: { "Content-Type": "application/json", diff --git a/src/main.py b/src/main.py index bf6da342..a09d2488 100644 --- a/src/main.py +++ b/src/main.py @@ -31,6 +31,7 @@ from api import ( auth, chat, connectors, + docling, documents, flows, knowledge_filter, @@ -1111,6 +1112,12 @@ async def create_app(): ), methods=["POST"], ), + # Docling service proxy + Route( + "/docling/health", + partial(docling.health), + methods=["GET"], + ), ] app = Starlette(debug=True, routes=routes) diff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.py index 6fecfff9..7cb5d1e8 100644 --- a/src/tui/managers/docling_manager.py +++ b/src/tui/managers/docling_manager.py @@ -8,6 +8,7 @@ import threading import time from typing import Optional, Tuple, Dict, Any, List, AsyncIterator from utils.logging_config import get_logger +from utils.container_utils import guess_host_ip_for_containers logger = get_logger(__name__) @@ -31,7 +32,7 @@ class DoclingManager: self._process: Optional[subprocess.Popen] = None self._port = 5001 - self._host = self._get_host_for_containers() # Get appropriate host IP based on runtime + self._host = guess_host_ip_for_containers(logger=logger) # Get appropriate host IP based on runtime self._running = False self._external_process = False @@ -49,136 +50,6 @@ class DoclingManager: # Try to recover existing process from PID file self._recover_from_pid_file() - def _get_host_for_containers(self) -> str: - """ - Return a host IP that containers can reach (a bridge/CNI gateway). - Prefers Docker/Podman network gateways; falls back to bridge interfaces. - """ - import subprocess, json, shutil, re, logging - logger = logging.getLogger(__name__) - - def run(cmd, timeout=2, text=True): - return subprocess.run(cmd, capture_output=True, text=text, timeout=timeout) - - gateways = [] - compose_gateways = [] # Highest priority - compose project networks - active_gateways = [] # Medium priority - networks with containers - - # ---- Docker: enumerate networks and collect gateways - if shutil.which("docker"): - try: - ls = run(["docker", "network", "ls", "--format", "{{.Name}}"]) - if ls.returncode == 0: - for name in filter(None, ls.stdout.splitlines()): - try: - insp = run(["docker", "network", "inspect", name, "--format", "{{json .}}"]) - if insp.returncode == 0 and insp.stdout.strip(): - nw = json.loads(insp.stdout)[0] if insp.stdout.strip().startswith("[") else json.loads(insp.stdout) - ipam = nw.get("IPAM", {}) - containers = nw.get("Containers", {}) - for cfg in ipam.get("Config", []) or []: - gw = cfg.get("Gateway") - if gw: - # Highest priority: compose networks (ending in _default) - if name.endswith("_default"): - compose_gateways.append(gw) - # Medium priority: networks with active containers - elif len(containers) > 0: - active_gateways.append(gw) - # Low priority: empty networks - else: - gateways.append(gw) - except Exception: - pass - except Exception: - pass - - # ---- Podman: enumerate networks and collect gateways (netavark) - if shutil.which("podman"): - try: - # modern podman supports JSON format - ls = run(["podman", "network", "ls", "--format", "json"]) - if ls.returncode == 0 and ls.stdout.strip(): - for net in json.loads(ls.stdout): - name = net.get("name") or net.get("Name") - if not name: - continue - try: - insp = run(["podman", "network", "inspect", name, "--format", "json"]) - if insp.returncode == 0 and insp.stdout.strip(): - arr = json.loads(insp.stdout) - for item in (arr if isinstance(arr, list) else [arr]): - for sn in item.get("subnets", []) or []: - gw = sn.get("gateway") - if gw: - # Prioritize compose/project networks - if name.endswith("_default") or "_" in name: - compose_gateways.append(gw) - else: - gateways.append(gw) - except Exception: - pass - except Exception: - pass - - # ---- Fallback: parse host interfaces for common bridges - if not gateways: - try: - if shutil.which("ip"): - show = run(["ip", "-o", "-4", "addr", "show"]) - if show.returncode == 0: - for line in show.stdout.splitlines(): - # e.g. "12: br-3f0f... inet 172.18.0.1/16 ..." - m = re.search(r"^\d+:\s+([a-zA-Z0-9_.:-]+)\s+.*\binet\s+(\d+\.\d+\.\d+\.\d+)/", line) - if not m: - continue - ifname, ip = m.group(1), m.group(2) - if ifname == "docker0" or ifname.startswith(("br-", "cni")): - gateways.append(ip) - else: - # As a last resort, try net-tools ifconfig output - if shutil.which("ifconfig"): - show = run(["ifconfig"]) - for block in show.stdout.split("\n\n"): - if any(block.strip().startswith(n) for n in ("docker0", "cni", "br-")): - m = re.search(r"inet (?:addr:)?(\d+\.\d+\.\d+\.\d+)", block) - if m: - gateways.append(m.group(1)) - except Exception: - pass - - # Dedup, prioritizing: 1) compose networks, 2) active networks, 3) all others - seen, uniq = set(), [] - # First: compose project networks (_default suffix) - for ip in compose_gateways: - if ip not in seen: - uniq.append(ip) - seen.add(ip) - # Second: networks with active containers - for ip in active_gateways: - if ip not in seen: - uniq.append(ip) - seen.add(ip) - # Third: all other gateways - for ip in gateways: - if ip not in seen: - uniq.append(ip) - seen.add(ip) - - if uniq: - if len(uniq) > 1: - logger.info("Container-reachable host IP candidates: %s", ", ".join(uniq)) - else: - logger.info("Container-reachable host IP: %s", uniq[0]) - return uniq[0] - - # Nothing found: warn clearly - logger.warning( - "No container bridge IP found. If using rootless Podman (slirp4netns), there is no host bridge; publish ports or use 10.0.2.2 from the container." - ) - # Returning localhost is honest only for same-namespace; keep it explicit: - return "127.0.0.1" - def cleanup(self): """Cleanup resources but keep docling-serve running across sessions.""" # Don't stop the process on exit - let it persist diff --git a/src/utils/container_utils.py b/src/utils/container_utils.py index a18d9f1c..14222c84 100644 --- a/src/utils/container_utils.py +++ b/src/utils/container_utils.py @@ -136,3 +136,138 @@ def transform_localhost_url(url: str) -> str: return url.replace(pattern, container_host) return url + + +def guess_host_ip_for_containers(logger=None) -> str: + """Best-effort detection of a host IP reachable from container networks. + + The logic mirrors what the TUI uses when launching docling-serve so that + both CLI and API use consistent addresses. Preference order: + 1. Docker/Podman compose networks (ended with ``_default``) + 2. Networks with active containers + 3. Any discovered bridge or CNI gateway interfaces + + Args: + logger: Optional logger to emit diagnostics; falls back to module logger. + + Returns: + The most appropriate host IP address if discovered, otherwise ``"127.0.0.1"``. + """ + import json + import logging + import re + import shutil + import subprocess + + log = logger or logging.getLogger(__name__) + + def run(cmd, timeout=2, text=True): + return subprocess.run(cmd, capture_output=True, text=text, timeout=timeout) + + gateways: list[str] = [] + compose_gateways: list[str] = [] + active_gateways: list[str] = [] + + # ---- Docker networks + if shutil.which("docker"): + try: + ls = run(["docker", "network", "ls", "--format", "{{.Name}}"]) + if ls.returncode == 0: + for name in filter(None, ls.stdout.splitlines()): + try: + insp = run(["docker", "network", "inspect", name, "--format", "{{json .}}"]) + if insp.returncode == 0 and insp.stdout.strip(): + payload = insp.stdout.strip() + nw = json.loads(payload)[0] if payload.startswith("[") else json.loads(payload) + ipam = nw.get("IPAM", {}) + containers = nw.get("Containers", {}) + for cfg in ipam.get("Config", []) or []: + gw = cfg.get("Gateway") + if not gw: + continue + if name.endswith("_default"): + compose_gateways.append(gw) + elif len(containers) > 0: + active_gateways.append(gw) + else: + gateways.append(gw) + except Exception: + continue + except Exception: + pass + + # ---- Podman networks + if shutil.which("podman"): + try: + ls = run(["podman", "network", "ls", "--format", "json"]) + if ls.returncode == 0 and ls.stdout.strip(): + for net in json.loads(ls.stdout): + name = net.get("name") or net.get("Name") + if not name: + continue + try: + insp = run(["podman", "network", "inspect", name, "--format", "json"]) + if insp.returncode == 0 and insp.stdout.strip(): + arr = json.loads(insp.stdout) + for item in (arr if isinstance(arr, list) else [arr]): + for sn in item.get("subnets", []) or []: + gw = sn.get("gateway") + if not gw: + continue + if name.endswith("_default") or "_" in name: + compose_gateways.append(gw) + else: + gateways.append(gw) + except Exception: + continue + except Exception: + pass + + # ---- Host bridge interfaces + if not gateways and not compose_gateways and not active_gateways: + try: + if shutil.which("ip"): + show = run(["ip", "-o", "-4", "addr", "show"]) + if show.returncode == 0: + for line in show.stdout.splitlines(): + match = re.search(r"^\d+:\s+([\w_.:-]+)\s+.*\binet\s+(\d+\.\d+\.\d+\.\d+)/", line) + if not match: + continue + ifname, ip_addr = match.group(1), match.group(2) + if ifname == "docker0" or ifname.startswith(("br-", "cni")): + gateways.append(ip_addr) + elif shutil.which("ifconfig"): + show = run(["ifconfig"]) + for block in show.stdout.split("\n\n"): + if any(block.strip().startswith(n) for n in ("docker0", "cni", "br-")): + match = re.search(r"inet (?:addr:)?(\d+\.\d+\.\d+\.\d+)", block) + if match: + gateways.append(match.group(1)) + except Exception: + pass + + seen: set[str] = set() + ordered_candidates: list[str] = [] + + for collection in (compose_gateways, active_gateways, gateways): + for ip_addr in collection: + if ip_addr not in seen: + ordered_candidates.append(ip_addr) + seen.add(ip_addr) + + if ordered_candidates: + if len(ordered_candidates) > 1: + log.info( + "Container-reachable host IP candidates: %s", + ", ".join(ordered_candidates), + ) + else: + log.info("Container-reachable host IP: %s", ordered_candidates[0]) + + return ordered_candidates[0] + + log.warning( + "No container bridge IP found. For rootless Podman (slirp4netns) there may be no host bridge; publish ports or use 10.0.2.2 from the container." + ) + + return "127.0.0.1"