diff --git a/.gitignore b/.gitignore index 9c99e617..484db58d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ wheels/ .DS_Store config/ + +.docling.pid diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml index 570bc3b8..0c09254a 100644 --- a/docker-compose-cpu.yml +++ b/docker-compose-cpu.yml @@ -43,7 +43,7 @@ services: # build: # context: . # dockerfile: Dockerfile.backend - # container_name: openrag-backend + container_name: openrag-backend depends_on: - langflow environment: diff --git a/docker-compose.yml b/docker-compose.yml index b97f7cca..be9bcbc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: # build: # context: . # dockerfile: Dockerfile.backend - # container_name: openrag-backend + container_name: openrag-backend depends_on: - langflow environment: diff --git a/frontend/components/duplicate-handling-dialog.tsx b/frontend/components/duplicate-handling-dialog.tsx new file mode 100644 index 00000000..d5cb2edf --- /dev/null +++ b/frontend/components/duplicate-handling-dialog.tsx @@ -0,0 +1,66 @@ +"use client"; + +import { RotateCcw } from "lucide-react"; +import type React from "react"; +import { Button } from "./ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "./ui/dialog"; + +interface DuplicateHandlingDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + onOverwrite: () => void | Promise; + isLoading?: boolean; +} + +export const DuplicateHandlingDialog: React.FC< + DuplicateHandlingDialogProps +> = ({ open, onOpenChange, onOverwrite, isLoading = false }) => { + const handleOverwrite = async () => { + await onOverwrite(); + onOpenChange(false); + }; + + return ( + + + + Overwrite document + + Overwriting will replace the existing document with another version. + This can't be undone. + + + + + + + + + + ); +}; diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index 2d991e7b..19ddc387 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -1,17 +1,19 @@ "use client"; +import { useQueryClient } from "@tanstack/react-query"; import { ChevronDown, Cloud, FolderOpen, Loader2, PlugZap, - Plus, Upload, } from "lucide-react"; import { useRouter } from "next/navigation"; import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; +import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery"; +import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -24,21 +26,17 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { useTask } from "@/contexts/task-context"; import { cn } from "@/lib/utils"; +import type { File as SearchFile } from "@/src/app/api/queries/useGetSearchQuery"; -interface KnowledgeDropdownProps { - active?: boolean; - variant?: "navigation" | "button"; -} - -export function KnowledgeDropdown({ - active, - variant = "navigation", -}: KnowledgeDropdownProps) { +export function KnowledgeDropdown() { const { addTask } = useTask(); + const { refetch: refetchTasks } = useGetTasksQuery(); + const queryClient = useQueryClient(); const router = useRouter(); const [isOpen, setIsOpen] = useState(false); const [showFolderDialog, setShowFolderDialog] = useState(false); const [showS3Dialog, setShowS3Dialog] = useState(false); + const [showDuplicateDialog, setShowDuplicateDialog] = useState(false); const [awsEnabled, setAwsEnabled] = useState(false); const [folderPath, setFolderPath] = useState("/app/documents/"); const [bucketUrl, setBucketUrl] = useState("s3://"); @@ -46,6 +44,8 @@ export function KnowledgeDropdown({ const [s3Loading, setS3Loading] = useState(false); const [fileUploading, setFileUploading] = useState(false); const [isNavigatingToCloud, setIsNavigatingToCloud] = useState(false); + const [pendingFile, setPendingFile] = useState(null); + const [duplicateFilename, setDuplicateFilename] = useState(""); const [cloudConnectors, setCloudConnectors] = useState<{ [key: string]: { name: string; @@ -166,106 +166,54 @@ export function KnowledgeDropdown({ const handleFileChange = async (e: React.ChangeEvent) => { const files = e.target.files; if (files && files.length > 0) { - // Close dropdown and disable button immediately after file selection - setIsOpen(false); - setFileUploading(true); + const file = files[0]; - // Trigger the same file upload event as the chat page - window.dispatchEvent( - new CustomEvent("fileUploadStart", { - detail: { filename: files[0].name }, - }) - ); + // Close dropdown immediately after file selection + setIsOpen(false); try { - const formData = new FormData(); - formData.append("file", files[0]); + // Check if filename already exists (using ORIGINAL filename) + console.log("[Duplicate Check] Checking file:", file.name); + const checkResponse = await fetch( + `/api/documents/check-filename?filename=${encodeURIComponent( + file.name + )}` + ); - // Use router upload and ingest endpoint (automatically routes based on configuration) - const uploadIngestRes = await fetch("/api/router/upload_ingest", { - method: "POST", - body: formData, - }); + console.log("[Duplicate Check] Response status:", checkResponse.status); - const uploadIngestJson = await uploadIngestRes.json(); - - if (!uploadIngestRes.ok) { + if (!checkResponse.ok) { + const errorText = await checkResponse.text(); + console.error("[Duplicate Check] Error response:", errorText); throw new Error( - uploadIngestJson?.error || "Upload and ingest failed" + `Failed to check duplicates: ${checkResponse.statusText}` ); } - // Extract results from the response - handle both unified and simple formats - const fileId = uploadIngestJson?.upload?.id || uploadIngestJson?.id; - const filePath = - uploadIngestJson?.upload?.path || - uploadIngestJson?.path || - "uploaded"; - const runJson = uploadIngestJson?.ingestion; - const deleteResult = uploadIngestJson?.deletion; + const checkData = await checkResponse.json(); + console.log("[Duplicate Check] Result:", checkData); - if (!fileId) { - throw new Error("Upload successful but no file id returned"); - } - - // Check if ingestion actually succeeded - if ( - runJson && - runJson.status !== "COMPLETED" && - runJson.status !== "SUCCESS" - ) { - const errorMsg = runJson.error || "Ingestion pipeline failed"; - throw new Error( - `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.` - ); - } - - // Log deletion status if provided - if (deleteResult) { - if (deleteResult.status === "deleted") { - console.log( - "File successfully cleaned up from Langflow:", - deleteResult.file_id - ); - } else if (deleteResult.status === "delete_failed") { - console.warn( - "Failed to cleanup file from Langflow:", - deleteResult.error - ); + if (checkData.exists) { + // Show duplicate handling dialog + console.log("[Duplicate Check] Duplicate detected, showing dialog"); + setPendingFile(file); + setDuplicateFilename(file.name); + setShowDuplicateDialog(true); + // Reset file input + if (fileInputRef.current) { + fileInputRef.current.value = ""; } + return; } - // Notify UI - window.dispatchEvent( - new CustomEvent("fileUploaded", { - detail: { - file: files[0], - result: { - file_id: fileId, - file_path: filePath, - run: runJson, - deletion: deleteResult, - unified: true, - }, - }, - }) - ); - - // Trigger search refresh after successful ingestion - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); + // No duplicate, proceed with upload + console.log("[Duplicate Check] No duplicate, proceeding with upload"); + await uploadFile(file, false); } catch (error) { - window.dispatchEvent( - new CustomEvent("fileUploadError", { - detail: { - filename: files[0].name, - error: error instanceof Error ? error.message : "Upload failed", - }, - }) - ); - } finally { - window.dispatchEvent(new CustomEvent("fileUploadComplete")); - setFileUploading(false); - // Don't call refetchSearch() here - the knowledgeUpdated event will handle it + console.error("[Duplicate Check] Exception:", error); + toast.error("Failed to check for duplicates", { + description: error instanceof Error ? error.message : "Unknown error", + }); } } @@ -275,6 +223,120 @@ export function KnowledgeDropdown({ } }; + const uploadFile = async (file: File, replace: boolean) => { + setFileUploading(true); + + // Trigger the same file upload event as the chat page + window.dispatchEvent( + new CustomEvent("fileUploadStart", { + detail: { filename: file.name }, + }) + ); + + try { + const formData = new FormData(); + formData.append("file", file); + formData.append("replace_duplicates", replace.toString()); + + // Use router upload and ingest endpoint (automatically routes based on configuration) + const uploadIngestRes = await fetch("/api/router/upload_ingest", { + method: "POST", + body: formData, + }); + + const uploadIngestJson = await uploadIngestRes.json(); + + if (!uploadIngestRes.ok) { + throw new Error(uploadIngestJson?.error || "Upload and ingest failed"); + } + + // Extract results from the response - handle both unified and simple formats + const fileId = + uploadIngestJson?.upload?.id || + uploadIngestJson?.id || + uploadIngestJson?.task_id; + const filePath = + uploadIngestJson?.upload?.path || uploadIngestJson?.path || "uploaded"; + const runJson = uploadIngestJson?.ingestion; + const deleteResult = uploadIngestJson?.deletion; + console.log("c", uploadIngestJson); + if (!fileId) { + throw new Error("Upload successful but no file id returned"); + } + // Check if ingestion actually succeeded + if ( + runJson && + runJson.status !== "COMPLETED" && + runJson.status !== "SUCCESS" + ) { + const errorMsg = runJson.error || "Ingestion pipeline failed"; + throw new Error( + `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.` + ); + } + // Log deletion status if provided + if (deleteResult) { + if (deleteResult.status === "deleted") { + console.log( + "File successfully cleaned up from Langflow:", + deleteResult.file_id + ); + } else if (deleteResult.status === "delete_failed") { + console.warn( + "Failed to cleanup file from Langflow:", + deleteResult.error + ); + } + } + // Notify UI + window.dispatchEvent( + new CustomEvent("fileUploaded", { + detail: { + file: file, + result: { + file_id: fileId, + file_path: filePath, + run: runJson, + deletion: deleteResult, + unified: true, + }, + }, + }) + ); + + refetchTasks(); + } catch (error) { + window.dispatchEvent( + new CustomEvent("fileUploadError", { + detail: { + filename: file.name, + error: error instanceof Error ? error.message : "Upload failed", + }, + }) + ); + } finally { + window.dispatchEvent(new CustomEvent("fileUploadComplete")); + setFileUploading(false); + } + }; + + const handleOverwriteFile = async () => { + if (pendingFile) { + // Remove the old file from all search query caches before overwriting + queryClient.setQueriesData({ queryKey: ["search"] }, (oldData: []) => { + if (!oldData) return oldData; + // Filter out the file that's being overwritten + return oldData.filter( + (file: SearchFile) => file.filename !== pendingFile.name + ); + }); + + await uploadFile(pendingFile, true); + setPendingFile(null); + setDuplicateFilename(""); + } + }; + const handleFolderUpload = async () => { if (!folderPath.trim()) return; @@ -301,17 +363,12 @@ export function KnowledgeDropdown({ addTask(taskId); setFolderPath(""); - // Trigger search refresh after successful folder processing starts - console.log( - "Folder upload successful, dispatching knowledgeUpdated event" - ); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); + // Refetch tasks to show the new task + refetchTasks(); } else if (response.ok) { setFolderPath(""); - console.log( - "Folder upload successful (direct), dispatching knowledgeUpdated event" - ); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); + // Refetch tasks even for direct uploads in case tasks were created + refetchTasks(); } else { console.error("Folder upload failed:", result.error); if (response.status === 400) { @@ -324,7 +381,6 @@ export function KnowledgeDropdown({ console.error("Folder upload error:", error); } finally { setFolderLoading(false); - // Don't call refetchSearch() here - the knowledgeUpdated event will handle it } }; @@ -354,9 +410,8 @@ export function KnowledgeDropdown({ addTask(taskId); setBucketUrl("s3://"); - // Trigger search refresh after successful S3 processing starts - console.log("S3 upload successful, dispatching knowledgeUpdated event"); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); + // Refetch tasks to show the new task + refetchTasks(); } else { console.error("S3 upload failed:", result.error); if (response.status === 400) { @@ -369,7 +424,6 @@ export function KnowledgeDropdown({ console.error("S3 upload error:", error); } finally { setS3Loading(false); - // Don't call refetchSearch() here - the knowledgeUpdated event will handle it } }; @@ -437,84 +491,44 @@ export function KnowledgeDropdown({ return ( <>
- + /> + )} + + {isOpen && !isLoading && (
{menuItems.map((item, index) => ( + )} + +
+ + ); +}; diff --git a/frontend/components/logo/ibm-logo.tsx b/frontend/components/logo/ibm-logo.tsx index 158ffa3b..e37adec1 100644 --- a/frontend/components/logo/ibm-logo.tsx +++ b/frontend/components/logo/ibm-logo.tsx @@ -9,7 +9,7 @@ export default function IBMLogo(props: React.SVGProps) { {...props} > IBM watsonx.ai Logo - + ( placeholder={placeholder} className={cn( "primary-input", - icon && "pl-9", + icon && "!pl-9", type === "password" && "!pr-8", icon ? inputClassName : className )} diff --git a/frontend/src/app/api/mutations/useCancelTaskMutation.ts b/frontend/src/app/api/mutations/useCancelTaskMutation.ts new file mode 100644 index 00000000..1bf2faed --- /dev/null +++ b/frontend/src/app/api/mutations/useCancelTaskMutation.ts @@ -0,0 +1,47 @@ +import { + type UseMutationOptions, + useMutation, + useQueryClient, +} from "@tanstack/react-query"; + +export interface CancelTaskRequest { + taskId: string; +} + +export interface CancelTaskResponse { + status: string; + task_id: string; +} + +export const useCancelTaskMutation = ( + options?: Omit< + UseMutationOptions, + "mutationFn" + > +) => { + const queryClient = useQueryClient(); + + async function cancelTask( + variables: CancelTaskRequest, + ): Promise { + const response = await fetch(`/api/tasks/${variables.taskId}/cancel`, { + method: "POST", + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error || "Failed to cancel task"); + } + + return response.json(); + } + + return useMutation({ + mutationFn: cancelTask, + onSuccess: () => { + // Invalidate tasks query to refresh the list + queryClient.invalidateQueries({ queryKey: ["tasks"] }); + }, + ...options, + }); +}; diff --git a/frontend/src/app/api/queries/useDoclingHealthQuery.ts b/frontend/src/app/api/queries/useDoclingHealthQuery.ts index 16ffc6c5..8db560d0 100644 --- a/frontend/src/app/api/queries/useDoclingHealthQuery.ts +++ b/frontend/src/app/api/queries/useDoclingHealthQuery.ts @@ -16,7 +16,8 @@ export const useDoclingHealthQuery = ( async function checkDoclingHealth(): Promise { try { - const response = await fetch("http://127.0.0.1:5001/health", { + // Call backend proxy endpoint instead of direct localhost + const response = await fetch("/api/docling/health", { method: "GET", headers: { "Content-Type": "application/json", diff --git a/frontend/src/app/api/queries/useGetNudgesQuery.ts b/frontend/src/app/api/queries/useGetNudgesQuery.ts index a9fe37a4..2e313e0c 100644 --- a/frontend/src/app/api/queries/useGetNudgesQuery.ts +++ b/frontend/src/app/api/queries/useGetNudgesQuery.ts @@ -7,9 +7,6 @@ import { type Nudge = string; const DEFAULT_NUDGES = [ - "Show me this quarter's top 10 deals", - "Summarize recent client interactions", - "Search OpenSearch for mentions of our competitors", ]; export const useGetNudgesQuery = ( diff --git a/frontend/src/app/api/queries/useGetSearchQuery.ts b/frontend/src/app/api/queries/useGetSearchQuery.ts index 5383178d..50905fcc 100644 --- a/frontend/src/app/api/queries/useGetSearchQuery.ts +++ b/frontend/src/app/api/queries/useGetSearchQuery.ts @@ -29,6 +29,7 @@ export interface ChunkResult { owner_email?: string; file_size?: number; connector_type?: string; + index?: number; } export interface File { @@ -55,7 +56,7 @@ export interface File { export const useGetSearchQuery = ( query: string, queryData?: ParsedQueryData | null, - options?: Omit, + options?: Omit ) => { const queryClient = useQueryClient(); @@ -179,12 +180,12 @@ export const useGetSearchQuery = ( const queryResult = useQuery( { - queryKey: ["search", queryData], + queryKey: ["search", queryData, query], placeholderData: (prev) => prev, queryFn: getFiles, ...options, }, - queryClient, + queryClient ); return queryResult; diff --git a/frontend/src/app/api/queries/useGetTasksQuery.ts b/frontend/src/app/api/queries/useGetTasksQuery.ts new file mode 100644 index 00000000..1ea59d26 --- /dev/null +++ b/frontend/src/app/api/queries/useGetTasksQuery.ts @@ -0,0 +1,79 @@ +import { + type UseQueryOptions, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; + +export interface Task { + task_id: string; + status: + | "pending" + | "running" + | "processing" + | "completed" + | "failed" + | "error"; + total_files?: number; + processed_files?: number; + successful_files?: number; + failed_files?: number; + running_files?: number; + pending_files?: number; + created_at: string; + updated_at: string; + duration_seconds?: number; + result?: Record; + error?: string; + files?: Record>; +} + +export interface TasksResponse { + tasks: Task[]; +} + +export const useGetTasksQuery = ( + options?: Omit, "queryKey" | "queryFn"> +) => { + const queryClient = useQueryClient(); + + async function getTasks(): Promise { + const response = await fetch("/api/tasks"); + + if (!response.ok) { + throw new Error("Failed to fetch tasks"); + } + + const data: TasksResponse = await response.json(); + return data.tasks || []; + } + + const queryResult = useQuery( + { + queryKey: ["tasks"], + queryFn: getTasks, + refetchInterval: (query) => { + // Only poll if there are tasks with pending or running status + const data = query.state.data; + if (!data || data.length === 0) { + return false; // Stop polling if no tasks + } + + const hasActiveTasks = data.some( + (task: Task) => + task.status === "pending" || + task.status === "running" || + task.status === "processing" + ); + + return hasActiveTasks ? 3000 : false; // Poll every 3 seconds if active tasks exist + }, + refetchIntervalInBackground: true, + staleTime: 0, // Always consider data stale to ensure fresh updates + gcTime: 5 * 60 * 1000, // Keep in cache for 5 minutes + ...options, + }, + queryClient, + ); + + return queryResult; +}; diff --git a/frontend/src/app/chat/page.tsx b/frontend/src/app/chat/page.tsx index 84de0cc8..8bae1e84 100644 --- a/frontend/src/app/chat/page.tsx +++ b/frontend/src/app/chat/page.tsx @@ -1,7 +1,6 @@ "use client"; import { - AtSign, Bot, Check, ChevronDown, @@ -11,7 +10,6 @@ import { Loader2, Plus, Settings, - Upload, User, X, Zap, @@ -31,7 +29,6 @@ import { import { useAuth } from "@/contexts/auth-context"; import { type EndpointType, useChat } from "@/contexts/chat-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; -import { useLayout } from "@/contexts/layout-context"; import { useTask } from "@/contexts/task-context"; import { useLoadingStore } from "@/stores/loadingStore"; import { useGetNudgesQuery } from "../api/queries/useGetNudgesQuery"; @@ -151,9 +148,8 @@ function ChatPage() { const streamAbortRef = useRef(null); const streamIdRef = useRef(0); const lastLoadedConversationRef = useRef(null); - const { addTask, isMenuOpen } = useTask(); - const { totalTopOffset } = useLayout(); - const { selectedFilter, parsedFilterData, isPanelOpen, setSelectedFilter } = + const { addTask } = useTask(); + const { selectedFilter, parsedFilterData, setSelectedFilter } = useKnowledgeFilter(); const scrollToBottom = () => { @@ -2047,10 +2043,10 @@ function ChatPage() { }; return ( -
+
{/* Debug header - only show in debug mode */} {isDebugMode && ( -
+
{/* Async Mode Toggle */} diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css index 150d1c56..56dc8dc8 100644 --- a/frontend/src/app/globals.css +++ b/frontend/src/app/globals.css @@ -108,8 +108,47 @@ } @layer components { + .app-grid-arrangement { + --sidebar-width: 0px; + --notifications-width: 0px; + --filters-width: 0px; + --app-header-height: 53px; + --top-banner-height: 0px; + + @media (width >= 48rem) { + --sidebar-width: 288px; + } + &.notifications-open { + --notifications-width: 320px; + } + &.filters-open { + --filters-width: 320px; + } + &.banner-visible { + --top-banner-height: 52px; + } + display: grid; + height: 100%; + width: 100%; + grid-template-rows: + var(--top-banner-height) + var(--app-header-height) + 1fr; + grid-template-columns: + var(--sidebar-width) + 1fr + var(--notifications-width) + var(--filters-width); + grid-template-areas: + "banner banner banner banner" + "header header header header" + "nav main notifications filters"; + transition: grid-template-columns 0.25s ease-in-out, + grid-template-rows 0.25s ease-in-out; + } + .header-arrangement { - @apply flex w-full h-[53px] items-center justify-between border-b border-border; + @apply flex w-full items-center justify-between border-b border-border; } .header-start-display { diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index b4823726..080120cc 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -1,21 +1,26 @@ "use client"; -import { ArrowLeft, Check, Copy, Loader2, Search } from "lucide-react"; -import { Suspense, useCallback, useEffect, useMemo, useState } from "react"; +import { ArrowLeft, Check, Copy, Loader2, Search, X } from "lucide-react"; import { useRouter, useSearchParams } from "next/navigation"; +import { Suspense, useCallback, useEffect, useMemo, useState } from "react"; +// import { Label } from "@/components/ui/label"; +// import { Checkbox } from "@/components/ui/checkbox"; +import { filterAccentClasses } from "@/components/knowledge-filter-panel"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; -import { useLayout } from "@/contexts/layout-context"; import { useTask } from "@/contexts/task-context"; import { type ChunkResult, type File, useGetSearchQuery, } from "../../api/queries/useGetSearchQuery"; -import { Label } from "@/components/ui/label"; -import { Checkbox } from "@/components/ui/checkbox"; -import { Input } from "@/components/ui/input"; +// import { Label } from "@/components/ui/label"; +// import { Checkbox } from "@/components/ui/checkbox"; +import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; const getFileTypeLabel = (mimetype: string) => { if (mimetype === "application/pdf") return "PDF"; @@ -27,16 +32,13 @@ const getFileTypeLabel = (mimetype: string) => { function ChunksPageContent() { const router = useRouter(); const searchParams = useSearchParams(); - const { isMenuOpen } = useTask(); - const { totalTopOffset } = useLayout(); - const { parsedFilterData, isPanelOpen } = useKnowledgeFilter(); - + const { parsedFilterData, queryOverride } = useKnowledgeFilter(); const filename = searchParams.get("filename"); const [chunks, setChunks] = useState([]); const [chunksFilteredByQuery, setChunksFilteredByQuery] = useState< ChunkResult[] >([]); - const [selectedChunks, setSelectedChunks] = useState>(new Set()); + // const [selectedChunks, setSelectedChunks] = useState>(new Set()); const [activeCopiedChunkIndex, setActiveCopiedChunkIndex] = useState< number | null >(null); @@ -49,25 +51,13 @@ function ChunksPageContent() { [chunks] ); - const [selectAll, setSelectAll] = useState(false); - const [queryInputText, setQueryInputText] = useState( - parsedFilterData?.query ?? "" - ); + // const [selectAll, setSelectAll] = useState(false); // Use the same search query as the knowledge page, but we'll filter for the specific file - const { data = [], isFetching } = useGetSearchQuery("*", parsedFilterData); - - useEffect(() => { - if (queryInputText === "") { - setChunksFilteredByQuery(chunks); - } else { - setChunksFilteredByQuery( - chunks.filter(chunk => - chunk.text.toLowerCase().includes(queryInputText.toLowerCase()) - ) - ); - } - }, [queryInputText, chunks]); + const { data = [], isFetching } = useGetSearchQuery( + queryOverride, + parsedFilterData + ); const handleCopy = useCallback((text: string, index: number) => { // Trim whitespace and remove new lines/tabs for cleaner copy @@ -87,7 +77,9 @@ function ChunksPageContent() { return; } - setChunks(fileData?.chunks || []); + setChunks( + fileData?.chunks?.map((chunk, i) => ({ ...chunk, index: i + 1 })) || [] + ); }, [data, filename]); // Set selected state for all checkboxes when selectAll changes @@ -103,20 +95,20 @@ function ChunksPageContent() { router.push("/knowledge"); }, [router]); - const handleChunkCardCheckboxChange = useCallback( - (index: number) => { - setSelectedChunks(prevSelected => { - const newSelected = new Set(prevSelected); - if (newSelected.has(index)) { - newSelected.delete(index); - } else { - newSelected.add(index); - } - return newSelected; - }); - }, - [setSelectedChunks] - ); + // const handleChunkCardCheckboxChange = useCallback( + // (index: number) => { + // setSelectedChunks((prevSelected) => { + // const newSelected = new Set(prevSelected); + // if (newSelected.has(index)) { + // newSelected.delete(index); + // } else { + // newSelected.add(index); + // } + // return newSelected; + // }); + // }, + // [setSelectedChunks] + // ); if (!filename) { return ( @@ -133,12 +125,17 @@ function ChunksPageContent() { } return ( -
-
+
+
{/* Header */}
-
-

@@ -146,20 +143,9 @@ function ChunksPageContent() { {filename.replace(/\.[^/.]+$/, "")}

-
-
- : null} - id="search-query" - type="text" - defaultValue={parsedFilterData?.query} - value={queryInputText} - onChange={e => setQueryInputText(e.target.value)} - placeholder="Search chunks..." - /> -
-
+
+ + {/*
Select all -
+
*/}
{/* Content Area - matches knowledge page structure */} -
+
{isFetching ? (
@@ -191,10 +177,9 @@ function ChunksPageContent() { ) : chunks.length === 0 ? (
- -

No chunks found

-

- This file may not have been indexed yet +

No knowledge

+

+ Clear the knowledge filter or return to the knowledge page

@@ -207,16 +192,16 @@ function ChunksPageContent() { >
-
+ {/*
handleChunkCardCheckboxChange(index) } /> -
+
*/} - Chunk {chunk.page} + Chunk {chunk.index} {chunk.text.length} chars @@ -236,6 +221,10 @@ function ChunksPageContent() {
+ + {chunk.score.toFixed(2)} score + + {/* TODO: Update to use active toggle */} {/* */}
-
+
{chunk.text}
@@ -255,24 +244,29 @@ function ChunksPageContent() {
{/* Right panel - Summary (TODO), Technical details, */} -
-
-

Technical details

-
-
-
Total chunks
-
- {chunks.length} -
-
-
-
Avg length
-
- {averageChunkLength.toFixed(0)} chars -
-
- {/* TODO: Uncomment after data is available */} - {/*
+ {chunks.length > 0 && ( +
+
+

+ Technical details +

+
+
+
+ Total chunks +
+
+ {chunks.length} +
+
+
+
Avg length
+
+ {averageChunkLength.toFixed(0)} chars +
+
+ {/* TODO: Uncomment after data is available */} + {/*
Process time
@@ -282,51 +276,54 @@ function ChunksPageContent() {
*/} -
-
-
-

Original document

-
-
+
+
+
+

+ Original document +

+
+ {/*
Name
{fileData?.filename}
-
-
-
Type
-
- {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} -
-
-
-
Size
-
- {fileData?.size - ? `${Math.round(fileData.size / 1024)} KB` - : "Unknown"} -
-
-
+
*/} +
+
Type
+
+ {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} +
+
+
+
Size
+
+ {fileData?.size + ? `${Math.round(fileData.size / 1024)} KB` + : "Unknown"} +
+
+ {/*
Uploaded
N/A
-
- {/* TODO: Uncomment after data is available */} - {/*
+
*/} + {/* TODO: Uncomment after data is available */} + {/*
Source
*/} -
+ {/*
Updated
N/A
-
-
+
*/} +
+
-
+ )}
); } diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 692f8a10..334f8e6f 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -1,12 +1,14 @@ "use client"; -import type { ColDef } from "ag-grid-community"; +import { + themeQuartz, + type ColDef, + type GetRowIdParams, +} from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; -import { Building2, Cloud, HardDrive, Search, Trash2, X } from "lucide-react"; +import { Cloud, FileIcon, Globe } from "lucide-react"; import { useRouter } from "next/navigation"; -import { type ChangeEvent, useCallback, useRef, useState } from "react"; -import { SiGoogledrive } from "react-icons/si"; -import { TbBrandOnedrive } from "react-icons/tb"; +import { useCallback, useEffect, useRef, useState } from "react"; import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; @@ -20,49 +22,52 @@ import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdow import { StatusBadge } from "@/components/ui/status-badge"; import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog"; import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; -import { filterAccentClasses } from "@/components/knowledge-filter-panel"; +import GoogleDriveIcon from "../settings/icons/google-drive-icon"; +import OneDriveIcon from "../settings/icons/one-drive-icon"; +import SharePointIcon from "../settings/icons/share-point-icon"; +import { KnowledgeSearchInput } from "@/components/knowledge-search-input"; // Function to get the appropriate icon for a connector type function getSourceIcon(connectorType?: string) { switch (connectorType) { case "google_drive": return ( - + ); case "onedrive": - return ( - - ); + return ; case "sharepoint": - return ; + return ( + + ); + case "url": + return ; case "s3": return ; default: return ( - + ); } } function SearchPage() { const router = useRouter(); - const { files: taskFiles } = useTask(); - const { selectedFilter, setSelectedFilter, parsedFilterData } = - useKnowledgeFilter(); + const { files: taskFiles, refreshTasks } = useTask(); + const { parsedFilterData, queryOverride } = useKnowledgeFilter(); const [selectedRows, setSelectedRows] = useState([]); const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); const deleteDocumentMutation = useDeleteDocument(); - const { data = [], isFetching } = useGetSearchQuery( - parsedFilterData?.query || "*", + useEffect(() => { + refreshTasks(); + }, [refreshTasks]); + + const { data: searchData = [], isFetching } = useGetSearchQuery( + queryOverride, parsedFilterData ); - - const handleTableSearch = (e: ChangeEvent) => { - gridRef.current?.api.setGridOption("quickFilterText", e.target.value); - }; - // Convert TaskFiles to File format and merge with backend results const taskFilesAsFiles: File[] = taskFiles.map(taskFile => { return { @@ -75,7 +80,26 @@ function SearchPage() { }; }); - const backendFiles = data as File[]; + // Create a map of task files by filename for quick lookup + const taskFileMap = new Map( + taskFilesAsFiles.map(file => [file.filename, file]) + ); + + // Override backend files with task file status if they exist + const backendFiles = (searchData as File[]) + .map(file => { + const taskFile = taskFileMap.get(file.filename); + if (taskFile) { + // Override backend file with task file data (includes status) + return { ...file, ...taskFile }; + } + return file; + }) + .filter(file => { + // Only filter out files that are currently processing AND in taskFiles + const taskFile = taskFileMap.get(file.filename); + return !taskFile || taskFile.status !== "processing"; + }); const filteredTaskFiles = taskFilesAsFiles.filter(taskFile => { return ( @@ -91,39 +115,54 @@ function SearchPage() { const gridRef = useRef(null); - const [columnDefs] = useState[]>([ + const columnDefs = [ { field: "filename", headerName: "Source", - checkboxSelection: true, + checkboxSelection: (params: CustomCellRendererProps) => + (params?.data?.status || "active") === "active", headerCheckboxSelection: true, initialFlex: 2, minWidth: 220, cellRenderer: ({ data, value }: CustomCellRendererProps) => { + // Read status directly from data on each render + const status = data?.status || "active"; + const isActive = status === "active"; + console.log(data?.filename, status, "a"); return ( - +
+
+ +
); }, }, { field: "size", headerName: "Size", - valueFormatter: params => + valueFormatter: (params: CustomCellRendererProps) => params.value ? `${Math.round(params.value / 1024)} KB` : "-", }, { @@ -133,13 +172,14 @@ function SearchPage() { { field: "owner", headerName: "Owner", - valueFormatter: params => + valueFormatter: (params: CustomCellRendererProps) => params.data?.owner_name || params.data?.owner_email || "—", }, { field: "chunkCount", headerName: "Chunks", - valueFormatter: params => params.data?.chunkCount?.toString() || "-", + valueFormatter: (params: CustomCellRendererProps) => + params.data?.chunkCount?.toString() || "-", }, { field: "avgScore", @@ -156,6 +196,7 @@ function SearchPage() { field: "status", headerName: "Status", cellRenderer: ({ data }: CustomCellRendererProps) => { + console.log(data?.filename, data?.status, "b"); // Default to 'active' status if no status is provided const status = data?.status || "active"; return ; @@ -163,6 +204,10 @@ function SearchPage() { }, { cellRenderer: ({ data }: CustomCellRendererProps) => { + const status = data?.status || "active"; + if (status !== "active") { + return null; + } return ; }, cellStyle: { @@ -179,7 +224,7 @@ function SearchPage() { sortable: false, initialFlex: 0, }, - ]); + ]; const defaultColDef: ColDef = { resizable: false, @@ -228,45 +273,24 @@ function SearchPage() { }; return ( -
-
-

- Project Knowledge -

- -
+ <> +
+
+

Project Knowledge

+
- {/* Search Input Area */} -
-
-
- {selectedFilter?.name && ( -
- {selectedFilter?.name} - setSelectedFilter(null)} - /> -
- )} - - -
+ {/* Search Input Area */} +
+ + {/* //TODO: Implement sync button */} + {/* */} {selectedRows.length > 0 && ( )} - -
- -
+
+ +
+
[]} defaultColDef={defaultColDef} loading={isFetching} ref={gridRef} + theme={themeQuartz.withParams({ browserColorScheme: "inherit" })} rowData={fileResults} rowSelection="multiple" rowMultiSelectWithClick={false} suppressRowClickSelection={true} - getRowId={params => params.data.filename} + getRowId={(params: GetRowIdParams) => params.data?.filename} domLayout="normal" onSelectionChanged={onSelectionChanged} noRowsOverlayComponent={() => ( @@ -324,7 +349,7 @@ ${selectedRows.map(row => `• ${row.filename}`).join("\n")}`} onConfirm={handleBulkDelete} isLoading={deleteDocumentMutation.isPending} /> -
+ ); } diff --git a/frontend/src/app/layout.tsx b/frontend/src/app/layout.tsx index baf9f2d5..10ed826b 100644 --- a/frontend/src/app/layout.tsx +++ b/frontend/src/app/layout.tsx @@ -38,7 +38,7 @@ export default function RootLayout({ return ( ( +const GoogleDriveIcon = ({ className }: { className?: string }) => ( ( +const OneDriveIcon = ({ className }: { className?: string }) => ( ( +const SharePointIcon = ({ className }: { className?: string }) => ( ([]); + const [isConnecting, setIsConnecting] = useState(null); const [isSyncing, setIsSyncing] = useState(null); const [syncResults, setSyncResults] = useState<{ [key: string]: SyncResult | null; }>({}); + const [maxFiles, setMaxFiles] = useState(10); + const [syncAllFiles, setSyncAllFiles] = useState(false); // Only keep systemPrompt state since it needs manual save button const [systemPrompt, setSystemPrompt] = useState(""); @@ -377,6 +380,58 @@ function KnowledgeSourcesPage() { } }, [getConnectorIcon]); + const handleConnect = async (connector: Connector) => { + setIsConnecting(connector.id); + setSyncResults(prev => ({ ...prev, [connector.id]: null })); + + try { + // Use the shared auth callback URL, same as connectors page + const redirectUri = `${window.location.origin}/auth/callback`; + + const response = await fetch("/api/auth/init", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + connector_type: connector.type, + purpose: "data_source", + name: `${connector.name} Connection`, + redirect_uri: redirectUri, + }), + }); + + if (response.ok) { + const result = await response.json(); + + if (result.oauth_config) { + localStorage.setItem("connecting_connector_id", result.connection_id); + localStorage.setItem("connecting_connector_type", connector.type); + + const authUrl = + `${result.oauth_config.authorization_endpoint}?` + + `client_id=${result.oauth_config.client_id}&` + + `response_type=code&` + + `scope=${result.oauth_config.scopes.join(" ")}&` + + `redirect_uri=${encodeURIComponent( + result.oauth_config.redirect_uri + )}&` + + `access_type=offline&` + + `prompt=consent&` + + `state=${result.connection_id}`; + + window.location.href = authUrl; + } + } else { + console.error("Failed to initiate connection"); + setIsConnecting(null); + } + } catch (error) { + console.error("Connection error:", error); + setIsConnecting(null); + } + }; + // const handleSync = async (connector: Connector) => { // if (!connector.connectionId) return; @@ -568,556 +623,614 @@ function KnowledgeSourcesPage() { }; return ( -
-
-

- Cloud Connectors -

-
- - {/* Conditional Sync Settings or No-Auth Message */} - {isNoAuthMode ? ( -
- - - - Cloud connectors are only available with auth mode enabled - - - Please provide the following environment variables and restart: - - - -
-
- # make here https://console.cloud.google.com/apis/credentials -
-
GOOGLE_OAUTH_CLIENT_ID=
-
GOOGLE_OAUTH_CLIENT_SECRET=
-
- - -
- ) : null} - +
{/* Connectors Section */} -
- - -
- Cloud Connectors - - Connect to cloud storage providers to sync your knowledge. - -
- - -
- {DEFAULT_CONNECTORS.map(connector => { - const actualConnector = connectors.find( - c => c.id === connector.id - ); - return ( - - -
-
-
-
- {connector.icon} -
-
- - {connector.name} - {actualConnector && - getStatusBadge(actualConnector.status)} - - - {actualConnector?.description - ? `${actualConnector.name} is configured.` - : connector.description} - +
+
+

+ Cloud Connectors +

+
+ + {/* Conditional Sync Settings or No-Auth Message */} + { + isNoAuthMode ? ( + + + + Cloud connectors require authentication + + + Add the Google OAuth variables below to your .env{" "} + then restart the OpenRAG containers. + + + +
+
+
+ + 27 + + # Google OAuth +
+
+ + 28 + + # Create credentials here: +
+
+ + 29 + + + # https://console.cloud.google.com/apis/credentials + +
+
+
+ 30 + GOOGLE_OAUTH_CLIENT_ID= +
+
+ 31 + GOOGLE_OAUTH_CLIENT_SECRET= +
+
+ + + ) : null + //
+ //
+ //

Sync Settings

+ //

+ // Configure how many files to sync when manually triggering a sync + //

+ //
+ //
+ //
+ // { + // setSyncAllFiles(!!checked); + // if (checked) { + // setMaxFiles(0); + // } else { + // setMaxFiles(10); + // } + // }} + // /> + // + //
+ // + //
+ // setMaxFiles(parseInt(e.target.value) || 10)} + // disabled={syncAllFiles} + // className="w-16 min-w-16 max-w-16 flex-shrink-0 disabled:opacity-50 disabled:cursor-not-allowed" + // min="1" + // max="100" + // title={ + // syncAllFiles + // ? "Disabled when 'Sync all files' is checked" + // : "Leave blank or set to 0 for unlimited" + // } + // /> + //
+ //
+ //
+ } + + {/* Connectors Grid */} +
+ {DEFAULT_CONNECTORS.map(connector => { + const actualConnector = connectors.find(c => c.id === connector.id); + return ( + + +
+
+
+
+ {connector.icon}
- - - {actualConnector?.status === "connected" ? ( -
- + + {connector.name} + {actualConnector && + getStatusBadge(actualConnector.status)} + + + {actualConnector?.description + ? `${actualConnector.name} is configured.` + : connector.description} + +
+
+ + + {actualConnector?.status === "connected" ? ( +
+ - {syncResults[connector.id] && ( -
-
- Processed:{" "} - {syncResults[connector.id]?.processed || 0} -
-
- Added: {syncResults[connector.id]?.added || 0} -
- {syncResults[connector.id]?.errors && ( -
- Errors: {syncResults[connector.id]?.errors} -
- )} + {syncResults[connector.id] && ( +
+
+ Processed:{" "} + {syncResults[connector.id]?.processed || 0} +
+
+ Added: {syncResults[connector.id]?.added || 0} +
+ {syncResults[connector.id]?.errors && ( +
+ Errors: {syncResults[connector.id]?.errors}
)}
- ) : ( -
-

- See our{" "} - - Cloud Connectors installation guide - {" "} - for more detail. -

-
)} - - - ); - })} -
- - -
- - {/* Agent Behavior Section */} -
- - -
-
- Agent - - Quick Agent settings. Edit in Langflow for full control. - -
-
- - Restore flow - - } - title="Restore default Retrieval flow" - description="This restores defaults and discards all custom settings and overrides. This can’t be undone." - confirmText="Restore" - variant="destructive" - onConfirm={handleRestoreRetrievalFlow} - /> - - - Langflow icon - - - - - Edit in Langflow - - } - title="Edit Retrieval flow in Langflow" - description={ - <> -

- You're entering Langflow. You can edit the{" "} - Retrieval flow and other underlying flows. Manual - changes to components, wiring, or I/O can break this - experience. -

-

You can restore this flow from Settings.

- - } - confirmText="Proceed" - confirmIcon={} - onConfirm={closeDialog => - handleEditInLangflow("chat", closeDialog) - } - variant="warning" - /> -
-
- - -
-
- - } - value={modelsData ? settings.agent?.llm_model || "" : ""} - onValueChange={handleModelChange} - /> - -
-
- -