From bf04d7ea600cc554a8b2151ce7e8848e824355e4 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Thu, 2 Oct 2025 14:51:33 -0600 Subject: [PATCH 1/8] [design sweep]: update no auth cloud connector warning message --- frontend/src/app/settings/page.tsx | 46 +++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index 6017ab5b..aca2d293 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -628,25 +628,49 @@ function KnowledgeSourcesPage() { {/* Conditional Sync Settings or No-Auth Message */} { - isNoAuthMode ? ( - + true ? ( + - - Cloud connectors are only available with auth mode enabled + + Cloud connectors require authentication - Please provide the following environment variables and - restart: + Add the Google OAuth variables below to your .env{" "} + then restart the OpenRAG containers.
-
- # make here - https://console.cloud.google.com/apis/credentials +
+
+ + 18 + + # Google OAuth +
+
+ + 19 + + # Create credentials here: +
+
+ + 20 + + + # https://console.cloud.google.com/apis/credentials + +
+
+
+ 21 + GOOGLE_OAUTH_CLIENT_ID= +
+
+ 22 + GOOGLE_OAUTH_CLIENT_SECRET=
-
GOOGLE_OAUTH_CLIENT_ID=
-
GOOGLE_OAUTH_CLIENT_SECRET=
From 892dcb3d74561671924c452d8439b8e89699e6df Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Thu, 2 Oct 2025 14:53:00 -0600 Subject: [PATCH 2/8] revert --- frontend/src/app/settings/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index aca2d293..71c0c8e4 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -628,7 +628,7 @@ function KnowledgeSourcesPage() { {/* Conditional Sync Settings or No-Auth Message */} { - true ? ( + isNoAuthMode ? ( From ae02fc00e8b90c95e3dbcdb0dd0a4d205aa26f6d Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 10:48:42 -0600 Subject: [PATCH 3/8] updated numbers to match env.example --- frontend/src/app/settings/page.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index 71c0c8e4..07aeb5f8 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -644,19 +644,19 @@ function KnowledgeSourcesPage() {
- 18 + 27 # Google OAuth
- 19 + 28 # Create credentials here:
- 20 + 29 # https://console.cloud.google.com/apis/credentials @@ -664,11 +664,11 @@ function KnowledgeSourcesPage() {
- 21 + 30 GOOGLE_OAUTH_CLIENT_ID=
- 22 + 31 GOOGLE_OAUTH_CLIENT_SECRET=
From ca8c16bd8b04f36f5f281842ace828bed0c343eb Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 6 Oct 2025 12:11:56 -0400 Subject: [PATCH 4/8] set backend container name --- docker-compose-cpu.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml index 570bc3b8..0c09254a 100644 --- a/docker-compose-cpu.yml +++ b/docker-compose-cpu.yml @@ -43,7 +43,7 @@ services: # build: # context: . # dockerfile: Dockerfile.backend - # container_name: openrag-backend + container_name: openrag-backend depends_on: - langflow environment: diff --git a/docker-compose.yml b/docker-compose.yml index b97f7cca..be9bcbc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: # build: # context: . # dockerfile: Dockerfile.backend - # container_name: openrag-backend + container_name: openrag-backend depends_on: - langflow environment: From ee0d58a627548dfa02113b2719bb175235111189 Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 6 Oct 2025 12:13:09 -0400 Subject: [PATCH 5/8] clear default nudges --- frontend/src/app/api/queries/useGetNudgesQuery.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/frontend/src/app/api/queries/useGetNudgesQuery.ts b/frontend/src/app/api/queries/useGetNudgesQuery.ts index a9fe37a4..2e313e0c 100644 --- a/frontend/src/app/api/queries/useGetNudgesQuery.ts +++ b/frontend/src/app/api/queries/useGetNudgesQuery.ts @@ -7,9 +7,6 @@ import { type Nudge = string; const DEFAULT_NUDGES = [ - "Show me this quarter's top 10 deals", - "Summarize recent client interactions", - "Search OpenSearch for mentions of our competitors", ]; export const useGetNudgesQuery = ( From 14c3a8f3d1d0e14e4bdead50ae9d9f041f5afeb4 Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 6 Oct 2025 12:14:03 -0400 Subject: [PATCH 6/8] tui: start/stop all services --- src/tui/screens/welcome.py | 176 ++++++++++++++++++------------------- 1 file changed, 86 insertions(+), 90 deletions(-) diff --git a/src/tui/screens/welcome.py b/src/tui/screens/welcome.py index c93f5561..9c121022 100644 --- a/src/tui/screens/welcome.py +++ b/src/tui/screens/welcome.py @@ -118,9 +118,16 @@ class WelcomeScreen(Screen): welcome_text.append(ascii_art, style="bold white") welcome_text.append("Terminal User Interface for OpenRAG\n\n", style="dim") - if self.services_running: + # Check if all services are running + all_services_running = self.services_running and self.docling_running + + if all_services_running: welcome_text.append( - "✓ Services are currently running\n\n", style="bold green" + "✓ All services are running\n\n", style="bold green" + ) + elif self.services_running or self.docling_running: + welcome_text.append( + "⚠ Some services are running\n\n", style="bold yellow" ) elif self.has_oauth_config: welcome_text.append( @@ -140,16 +147,19 @@ class WelcomeScreen(Screen): buttons = [] - if self.services_running: - # Services running - show app link first, then stop services + # Check if all services (native + container) are running + all_services_running = self.services_running and self.docling_running + + if all_services_running: + # All services running - show app link first, then stop all buttons.append( Button("Launch OpenRAG", variant="success", id="open-app-btn") ) buttons.append( - Button("Stop Container Services", variant="error", id="stop-services-btn") + Button("Stop All Services", variant="error", id="stop-all-services-btn") ) else: - # Services not running - show setup options and start services + # Some or no services running - show setup options and start all if has_oauth: # If OAuth is configured, only show advanced setup buttons.append( @@ -165,25 +175,7 @@ class WelcomeScreen(Screen): ) buttons.append( - Button("Start Container Services", variant="primary", id="start-services-btn") - ) - - # Native services controls - if self.docling_running: - buttons.append( - Button( - "Stop Native Services", - variant="warning", - id="stop-native-services-btn", - ) - ) - else: - buttons.append( - Button( - "Start Native Services", - variant="primary", - id="start-native-services-btn", - ) + Button("Start All Services", variant="primary", id="start-all-services-btn") ) # Always show status option @@ -213,7 +205,7 @@ class WelcomeScreen(Screen): ) # Set default button focus - if self.services_running: + if self.services_running and self.docling_running: self.default_button_id = "open-app-btn" elif self.has_oauth_config: self.default_button_id = "advanced-setup-btn" @@ -234,7 +226,7 @@ class WelcomeScreen(Screen): def _focus_appropriate_button(self) -> None: """Focus the appropriate button based on current state.""" try: - if self.services_running: + if self.services_running and self.docling_running: self.query_one("#open-app-btn").focus() elif self.has_oauth_config: self.query_one("#advanced-setup-btn").focus() @@ -253,20 +245,16 @@ class WelcomeScreen(Screen): self.action_monitor() elif event.button.id == "diagnostics-btn": self.action_diagnostics() - elif event.button.id == "start-services-btn": - self.action_start_stop_services() - elif event.button.id == "stop-services-btn": - self.action_start_stop_services() - elif event.button.id == "start-native-services-btn": - self.action_start_native_services() - elif event.button.id == "stop-native-services-btn": - self.action_stop_native_services() + elif event.button.id == "start-all-services-btn": + self.action_start_all_services() + elif event.button.id == "stop-all-services-btn": + self.action_stop_all_services() elif event.button.id == "open-app-btn": self.action_open_app() def action_default_action(self) -> None: """Handle Enter key - go to default action based on state.""" - if self.services_running: + if self.services_running and self.docling_running: self.action_open_app() elif self.has_oauth_config: self.action_full_setup() @@ -297,28 +285,13 @@ class WelcomeScreen(Screen): self.app.push_screen(DiagnosticsScreen()) - def action_start_stop_services(self) -> None: - """Start or stop all services (containers + docling).""" - if self.services_running: - # Stop services - show modal with progress - if self.container_manager.is_available(): - command_generator = self.container_manager.stop_services() - modal = CommandOutputModal( - "Stopping Services", - command_generator, - on_complete=self._on_services_operation_complete, - ) - self.app.push_screen(modal) - else: - # Start services - show modal with progress - if self.container_manager.is_available(): - command_generator = self.container_manager.start_services() - modal = CommandOutputModal( - "Starting Services", - command_generator, - on_complete=self._on_services_operation_complete, - ) - self.app.push_screen(modal) + def action_start_all_services(self) -> None: + """Start all services (native first, then containers).""" + self.run_worker(self._start_all_services()) + + def action_stop_all_services(self) -> None: + """Stop all services (containers first, then native).""" + self.run_worker(self._stop_all_services()) async def _on_services_operation_complete(self) -> None: """Handle completion of services start/stop operation.""" @@ -334,7 +307,7 @@ class WelcomeScreen(Screen): def _update_default_button(self) -> None: """Update the default button target based on state.""" - if self.services_running: + if self.services_running and self.docling_running: self.default_button_id = "open-app-btn" elif self.has_oauth_config: self.default_button_id = "advanced-setup-btn" @@ -362,51 +335,74 @@ class WelcomeScreen(Screen): self.call_after_refresh(self._focus_appropriate_button) - def action_start_native_services(self) -> None: - """Start native services (docling).""" - if self.docling_running: - self.notify("Native services are already running.", severity="warning") - return - - self.run_worker(self._start_native_services()) - - async def _start_native_services(self) -> None: - """Worker task to start native services.""" - try: + async def _start_all_services(self) -> None: + """Start all services: native first, then containers.""" + # Step 1: Start native services (docling-serve) + if not self.docling_manager.is_running(): + self.notify("Starting native services...", severity="information") success, message = await self.docling_manager.start() if success: - self.docling_running = True self.notify(message, severity="information") else: self.notify(f"Failed to start native services: {message}", severity="error") - except Exception as exc: - self.notify(f"Error starting native services: {exc}", severity="error") - finally: - self.docling_running = self.docling_manager.is_running() + # Continue anyway - user might want containers even if native fails + else: + self.notify("Native services already running", severity="information") + + # Update state + self.docling_running = self.docling_manager.is_running() + + # Step 2: Start container services + if self.container_manager.is_available(): + command_generator = self.container_manager.start_services() + modal = CommandOutputModal( + "Starting Container Services", + command_generator, + on_complete=self._on_services_operation_complete, + ) + self.app.push_screen(modal) + else: + self.notify("No container runtime available", severity="warning") await self._refresh_welcome_content() - def action_stop_native_services(self) -> None: - """Stop native services (docling).""" - if not self.docling_running and not self.docling_manager.is_running(): - self.notify("Native services are not running.", severity="warning") - return + async def _stop_all_services(self) -> None: + """Stop all services: containers first, then native.""" + # Step 1: Stop container services + if self.container_manager.is_available() and self.services_running: + command_generator = self.container_manager.stop_services() + modal = CommandOutputModal( + "Stopping Container Services", + command_generator, + on_complete=self._on_stop_containers_complete, + ) + self.app.push_screen(modal) + else: + # No containers to stop, go directly to stopping native services + await self._stop_native_services_after_containers() - self.run_worker(self._stop_native_services()) + async def _on_stop_containers_complete(self) -> None: + """Called after containers are stopped, now stop native services.""" + # Update container state + self._detect_services_sync() - async def _stop_native_services(self) -> None: - """Worker task to stop native services.""" - try: + # Now stop native services + await self._stop_native_services_after_containers() + + async def _stop_native_services_after_containers(self) -> None: + """Stop native services after containers have been stopped.""" + if self.docling_manager.is_running(): + self.notify("Stopping native services...", severity="information") success, message = await self.docling_manager.stop() if success: - self.docling_running = False self.notify(message, severity="information") else: self.notify(f"Failed to stop native services: {message}", severity="error") - except Exception as exc: - self.notify(f"Error stopping native services: {exc}", severity="error") - finally: - self.docling_running = self.docling_manager.is_running() - await self._refresh_welcome_content() + else: + self.notify("Native services already stopped", severity="information") + + # Update state + self.docling_running = self.docling_manager.is_running() + await self._refresh_welcome_content() def action_open_app(self) -> None: """Open the OpenRAG app in the default browser.""" From 41e2d39dd061d5582466ad1340755da255008ff6 Mon Sep 17 00:00:00 2001 From: Edwin Jose Date: Mon, 6 Oct 2025 14:36:17 -0400 Subject: [PATCH 7/8] feat: Show globe icon for HTML documents in knowledge page (#188) * Show globe icon for HTML documents in knowledge page Updated getSourceIcon to display a globe icon for documents with 'text/html' mimetype, regardless of connector type. This improves visual identification of web-based documents in the grid. * changed page to get connector type as url * removed wrong docs * fix formatting * format --------- Co-authored-by: Lucas Oliveira --- frontend/src/app/knowledge/page.tsx | 651 ++++++++++++++-------------- 1 file changed, 331 insertions(+), 320 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 64eeb49c..2cd6f382 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -2,14 +2,22 @@ import type { ColDef, GetRowIdParams } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; -import { Building2, Cloud, HardDrive, Search, Trash2, X } from "lucide-react"; +import { + Building2, + Cloud, + Globe, + HardDrive, + Search, + Trash2, + X, +} from "lucide-react"; import { useRouter } from "next/navigation"; import { - type ChangeEvent, - useCallback, - useEffect, - useRef, - useState, + type ChangeEvent, + useCallback, + useEffect, + useRef, + useState, } from "react"; import { SiGoogledrive } from "react-icons/si"; import { TbBrandOnedrive } from "react-icons/tb"; @@ -31,250 +39,255 @@ import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; // Function to get the appropriate icon for a connector type function getSourceIcon(connectorType?: string) { - switch (connectorType) { - case "google_drive": - return ( - - ); - case "onedrive": - return ( - - ); - case "sharepoint": - return ; - case "s3": - return ; - default: - return ( - - ); - } + switch (connectorType) { + case "url": + return ; + case "google_drive": + return ( + + ); + case "onedrive": + return ( + + ); + case "sharepoint": + return ; + case "s3": + return ; + default: + return ( + + ); + } } function SearchPage() { - const router = useRouter(); - const { isMenuOpen, files: taskFiles, refreshTasks } = useTask(); + const router = useRouter(); + const { isMenuOpen, files: taskFiles, refreshTasks } = useTask(); const { totalTopOffset } = useLayout(); - const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = - useKnowledgeFilter(); - const [selectedRows, setSelectedRows] = useState([]); - const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); + const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = + useKnowledgeFilter(); + const [selectedRows, setSelectedRows] = useState([]); + const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); - const deleteDocumentMutation = useDeleteDocument(); + const deleteDocumentMutation = useDeleteDocument(); - useEffect(() => { - refreshTasks(); - }, [refreshTasks]); + useEffect(() => { + refreshTasks(); + }, [refreshTasks]); - const { data: searchData = [], isFetching } = useGetSearchQuery( - parsedFilterData?.query || "*", - parsedFilterData, - ); - // Convert TaskFiles to File format and merge with backend results - const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { - return { - filename: taskFile.filename, - mimetype: taskFile.mimetype, - source_url: taskFile.source_url, - size: taskFile.size, - connector_type: taskFile.connector_type, - status: taskFile.status, - }; - }); + const { data: searchData = [], isFetching } = useGetSearchQuery( + parsedFilterData?.query || "*", + parsedFilterData, + ); + // Convert TaskFiles to File format and merge with backend results + const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { + return { + filename: taskFile.filename, + mimetype: taskFile.mimetype, + source_url: taskFile.source_url, + size: taskFile.size, + connector_type: taskFile.connector_type, + status: taskFile.status, + }; + }); - // Create a map of task files by filename for quick lookup - const taskFileMap = new Map( - taskFilesAsFiles.map((file) => [file.filename, file]), - ); + // Create a map of task files by filename for quick lookup + const taskFileMap = new Map( + taskFilesAsFiles.map((file) => [file.filename, file]), + ); - // Override backend files with task file status if they exist - const backendFiles = (searchData as File[]) - .map((file) => { - const taskFile = taskFileMap.get(file.filename); - if (taskFile) { - // Override backend file with task file data (includes status) - return { ...file, ...taskFile }; - } - return file; - }) - .filter((file) => { - // Only filter out files that are currently processing AND in taskFiles - const taskFile = taskFileMap.get(file.filename); - return !taskFile || taskFile.status !== "processing"; - }); + // Override backend files with task file status if they exist + const backendFiles = (searchData as File[]) + .map((file) => { + const taskFile = taskFileMap.get(file.filename); + if (taskFile) { + // Override backend file with task file data (includes status) + return { ...file, ...taskFile }; + } + return file; + }) + .filter((file) => { + // Only filter out files that are currently processing AND in taskFiles + const taskFile = taskFileMap.get(file.filename); + return !taskFile || taskFile.status !== "processing"; + }); - const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { - return ( - taskFile.status !== "active" && - !backendFiles.some( - (backendFile) => backendFile.filename === taskFile.filename, - ) - ); - }); + const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { + return ( + taskFile.status !== "active" && + !backendFiles.some( + (backendFile) => backendFile.filename === taskFile.filename, + ) + ); + }); - // Combine task files first, then backend files - const fileResults = [...backendFiles, ...filteredTaskFiles]; + // Combine task files first, then backend files + const fileResults = [...backendFiles, ...filteredTaskFiles]; - const handleTableSearch = (e: ChangeEvent) => { - gridRef.current?.api.setGridOption("quickFilterText", e.target.value); - }; + const handleTableSearch = (e: ChangeEvent) => { + gridRef.current?.api.setGridOption("quickFilterText", e.target.value); + }; - const gridRef = useRef(null); + const gridRef = useRef(null); - const columnDefs = [ - { - field: "filename", - headerName: "Source", - checkboxSelection: (params: CustomCellRendererProps) => - (params?.data?.status || "active") === "active", - headerCheckboxSelection: true, - initialFlex: 2, - minWidth: 220, - cellRenderer: ({ data, value }: CustomCellRendererProps) => { - // Read status directly from data on each render - const status = data?.status || "active"; - const isActive = status === "active"; - console.log(data?.filename, status, "a"); - return ( -
-
- -
- ); - }, - }, - { - field: "size", - headerName: "Size", - valueFormatter: (params: CustomCellRendererProps) => - params.value ? `${Math.round(params.value / 1024)} KB` : "-", - }, - { - field: "mimetype", - headerName: "Type", - }, - { - field: "owner", - headerName: "Owner", - valueFormatter: (params: CustomCellRendererProps) => - params.data?.owner_name || params.data?.owner_email || "—", - }, - { - field: "chunkCount", - headerName: "Chunks", - valueFormatter: (params: CustomCellRendererProps) => params.data?.chunkCount?.toString() || "-", - }, - { - field: "avgScore", - headerName: "Avg score", - initialFlex: 0.5, - cellRenderer: ({ value }: CustomCellRendererProps) => { - return ( - - {value?.toFixed(2) ?? "-"} - - ); - }, - }, - { - field: "status", - headerName: "Status", - cellRenderer: ({ data }: CustomCellRendererProps) => { - console.log(data?.filename, data?.status, "b"); - // Default to 'active' status if no status is provided - const status = data?.status || "active"; - return ; - }, - }, - { - cellRenderer: ({ data }: CustomCellRendererProps) => { - const status = data?.status || "active"; - if (status !== "active") { - return null; - } - return ; - }, - cellStyle: { - alignItems: "center", - display: "flex", - justifyContent: "center", - padding: 0, - }, - colId: "actions", - filter: false, - minWidth: 0, - width: 40, - resizable: false, - sortable: false, - initialFlex: 0, - }, - ]; + const columnDefs = [ + { + field: "filename", + headerName: "Source", + checkboxSelection: (params: CustomCellRendererProps) => + (params?.data?.status || "active") === "active", + headerCheckboxSelection: true, + initialFlex: 2, + minWidth: 220, + cellRenderer: ({ data, value }: CustomCellRendererProps) => { + // Read status directly from data on each render + const status = data?.status || "active"; + const isActive = status === "active"; + console.log(data?.filename, status, "a"); + return ( +
+
+ +
+ ); + }, + }, + { + field: "size", + headerName: "Size", + valueFormatter: (params: CustomCellRendererProps) => + params.value ? `${Math.round(params.value / 1024)} KB` : "-", + }, + { + field: "mimetype", + headerName: "Type", + }, + { + field: "owner", + headerName: "Owner", + valueFormatter: (params: CustomCellRendererProps) => + params.data?.owner_name || params.data?.owner_email || "—", + }, + { + field: "chunkCount", + headerName: "Chunks", + valueFormatter: (params: CustomCellRendererProps) => + params.data?.chunkCount?.toString() || "-", + }, + { + field: "avgScore", + headerName: "Avg score", + initialFlex: 0.5, + cellRenderer: ({ value }: CustomCellRendererProps) => { + return ( + + {value?.toFixed(2) ?? "-"} + + ); + }, + }, + { + field: "status", + headerName: "Status", + cellRenderer: ({ data }: CustomCellRendererProps) => { + console.log(data?.filename, data?.status, "b"); + // Default to 'active' status if no status is provided + const status = data?.status || "active"; + return ; + }, + }, + { + cellRenderer: ({ data }: CustomCellRendererProps) => { + const status = data?.status || "active"; + if (status !== "active") { + return null; + } + return ; + }, + cellStyle: { + alignItems: "center", + display: "flex", + justifyContent: "center", + padding: 0, + }, + colId: "actions", + filter: false, + minWidth: 0, + width: 40, + resizable: false, + sortable: false, + initialFlex: 0, + }, + ]; - const defaultColDef: ColDef = { - resizable: false, - suppressMovable: true, - initialFlex: 1, - minWidth: 100, - }; + const defaultColDef: ColDef = { + resizable: false, + suppressMovable: true, + initialFlex: 1, + minWidth: 100, + }; - const onSelectionChanged = useCallback(() => { - if (gridRef.current) { - const selectedNodes = gridRef.current.api.getSelectedRows(); - setSelectedRows(selectedNodes); - } - }, []); + const onSelectionChanged = useCallback(() => { + if (gridRef.current) { + const selectedNodes = gridRef.current.api.getSelectedRows(); + setSelectedRows(selectedNodes); + } + }, []); - const handleBulkDelete = async () => { - if (selectedRows.length === 0) return; + const handleBulkDelete = async () => { + if (selectedRows.length === 0) return; - try { - // Delete each file individually since the API expects one filename at a time - const deletePromises = selectedRows.map((row) => - deleteDocumentMutation.mutateAsync({ filename: row.filename }), - ); + try { + // Delete each file individually since the API expects one filename at a time + const deletePromises = selectedRows.map((row) => + deleteDocumentMutation.mutateAsync({ filename: row.filename }), + ); - await Promise.all(deletePromises); + await Promise.all(deletePromises); - toast.success( - `Successfully deleted ${selectedRows.length} document${ - selectedRows.length > 1 ? "s" : "" - }`, - ); - setSelectedRows([]); - setShowBulkDeleteDialog(false); + toast.success( + `Successfully deleted ${selectedRows.length} document${ + selectedRows.length > 1 ? "s" : "" + }`, + ); + setSelectedRows([]); + setShowBulkDeleteDialog(false); - // Clear selection in the grid - if (gridRef.current) { - gridRef.current.api.deselectAll(); - } - } catch (error) { - toast.error( - error instanceof Error - ? error.message - : "Failed to delete some documents", - ); - } - }; + // Clear selection in the grid + if (gridRef.current) { + gridRef.current.api.deselectAll(); + } + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Failed to delete some documents", + ); + } + }; return (
- {/* Search Input Area */} -
-
-
- {selectedFilter?.name && ( -
- {selectedFilter?.name} - setSelectedFilter(null)} - /> -
- )} - - -
- {/* */} - {/* //TODO: Implement sync button */} - {/* */} - {selectedRows.length > 0 && ( - - )} -
-
- []} - defaultColDef={defaultColDef} - loading={isFetching} - ref={gridRef} - rowData={fileResults} - rowSelection="multiple" - rowMultiSelectWithClick={false} - suppressRowClickSelection={true} - getRowId={(params: GetRowIdParams) => params.data?.filename} - domLayout="normal" - onSelectionChanged={onSelectionChanged} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> - + {selectedRows.length > 0 && ( + + )} + + + []} + defaultColDef={defaultColDef} + loading={isFetching} + ref={gridRef} + rowData={fileResults} + rowSelection="multiple" + rowMultiSelectWithClick={false} + suppressRowClickSelection={true} + getRowId={(params: GetRowIdParams) => params.data?.filename} + domLayout="normal" + onSelectionChanged={onSelectionChanged} + noRowsOverlayComponent={() => ( +
+
+ No knowledge +
+
+ Add files from local or your preferred cloud. +
+
+ )} + /> + - {/* Bulk Delete Confirmation Dialog */} - 1 ? "s" : "" - }? This will remove all chunks and data associated with these documents. This action cannot be undone. + {/* Bulk Delete Confirmation Dialog */} + 1 ? "s" : "" + }? This will remove all chunks and data associated with these documents. This action cannot be undone. Documents to be deleted: ${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`} - confirmText="Delete All" - onConfirm={handleBulkDelete} - isLoading={deleteDocumentMutation.isPending} - /> - - ); + confirmText="Delete All" + onConfirm={handleBulkDelete} + isLoading={deleteDocumentMutation.isPending} + /> + + ); } export default function ProtectedSearchPage() { - return ( - - - - ); + return ( + + + + ); } From c450ecc50afefcb262b445f08293b423e0daff0a Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 6 Oct 2025 14:46:51 -0400 Subject: [PATCH 8/8] tui: docling pid and host detection --- .gitignore | 2 + src/tui/managers/docling_manager.py | 331 +++++++++++++++++++++++----- src/tui/screens/welcome.py | 42 ++-- 3 files changed, 298 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 9c99e617..484db58d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ wheels/ .DS_Store config/ + +.docling.pid diff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.py index 5b1100cc..6fecfff9 100644 --- a/src/tui/managers/docling_manager.py +++ b/src/tui/managers/docling_manager.py @@ -31,10 +31,14 @@ class DoclingManager: self._process: Optional[subprocess.Popen] = None self._port = 5001 - self._host = "127.0.0.1" + self._host = self._get_host_for_containers() # Get appropriate host IP based on runtime self._running = False self._external_process = False + # PID file to track docling-serve across sessions (in current working directory) + from pathlib import Path + self._pid_file = Path.cwd() / ".docling.pid" + # Log storage - simplified, no queue self._log_buffer: List[str] = [] self._max_log_lines = 1000 @@ -42,22 +46,198 @@ class DoclingManager: self._initialized = True - def cleanup(self): - """Cleanup resources and stop any running processes.""" - if self._process and self._process.poll() is None: - self._add_log_entry("Cleaning up docling-serve process on exit") - try: - self._process.terminate() - self._process.wait(timeout=5) - except subprocess.TimeoutExpired: - self._process.kill() - self._process.wait() - except Exception as e: - self._add_log_entry(f"Error during cleanup: {e}") + # Try to recover existing process from PID file + self._recover_from_pid_file() - self._running = False - self._process = None + def _get_host_for_containers(self) -> str: + """ + Return a host IP that containers can reach (a bridge/CNI gateway). + Prefers Docker/Podman network gateways; falls back to bridge interfaces. + """ + import subprocess, json, shutil, re, logging + logger = logging.getLogger(__name__) + + def run(cmd, timeout=2, text=True): + return subprocess.run(cmd, capture_output=True, text=text, timeout=timeout) + + gateways = [] + compose_gateways = [] # Highest priority - compose project networks + active_gateways = [] # Medium priority - networks with containers + + # ---- Docker: enumerate networks and collect gateways + if shutil.which("docker"): + try: + ls = run(["docker", "network", "ls", "--format", "{{.Name}}"]) + if ls.returncode == 0: + for name in filter(None, ls.stdout.splitlines()): + try: + insp = run(["docker", "network", "inspect", name, "--format", "{{json .}}"]) + if insp.returncode == 0 and insp.stdout.strip(): + nw = json.loads(insp.stdout)[0] if insp.stdout.strip().startswith("[") else json.loads(insp.stdout) + ipam = nw.get("IPAM", {}) + containers = nw.get("Containers", {}) + for cfg in ipam.get("Config", []) or []: + gw = cfg.get("Gateway") + if gw: + # Highest priority: compose networks (ending in _default) + if name.endswith("_default"): + compose_gateways.append(gw) + # Medium priority: networks with active containers + elif len(containers) > 0: + active_gateways.append(gw) + # Low priority: empty networks + else: + gateways.append(gw) + except Exception: + pass + except Exception: + pass + + # ---- Podman: enumerate networks and collect gateways (netavark) + if shutil.which("podman"): + try: + # modern podman supports JSON format + ls = run(["podman", "network", "ls", "--format", "json"]) + if ls.returncode == 0 and ls.stdout.strip(): + for net in json.loads(ls.stdout): + name = net.get("name") or net.get("Name") + if not name: + continue + try: + insp = run(["podman", "network", "inspect", name, "--format", "json"]) + if insp.returncode == 0 and insp.stdout.strip(): + arr = json.loads(insp.stdout) + for item in (arr if isinstance(arr, list) else [arr]): + for sn in item.get("subnets", []) or []: + gw = sn.get("gateway") + if gw: + # Prioritize compose/project networks + if name.endswith("_default") or "_" in name: + compose_gateways.append(gw) + else: + gateways.append(gw) + except Exception: + pass + except Exception: + pass + + # ---- Fallback: parse host interfaces for common bridges + if not gateways: + try: + if shutil.which("ip"): + show = run(["ip", "-o", "-4", "addr", "show"]) + if show.returncode == 0: + for line in show.stdout.splitlines(): + # e.g. "12: br-3f0f... inet 172.18.0.1/16 ..." + m = re.search(r"^\d+:\s+([a-zA-Z0-9_.:-]+)\s+.*\binet\s+(\d+\.\d+\.\d+\.\d+)/", line) + if not m: + continue + ifname, ip = m.group(1), m.group(2) + if ifname == "docker0" or ifname.startswith(("br-", "cni")): + gateways.append(ip) + else: + # As a last resort, try net-tools ifconfig output + if shutil.which("ifconfig"): + show = run(["ifconfig"]) + for block in show.stdout.split("\n\n"): + if any(block.strip().startswith(n) for n in ("docker0", "cni", "br-")): + m = re.search(r"inet (?:addr:)?(\d+\.\d+\.\d+\.\d+)", block) + if m: + gateways.append(m.group(1)) + except Exception: + pass + + # Dedup, prioritizing: 1) compose networks, 2) active networks, 3) all others + seen, uniq = set(), [] + # First: compose project networks (_default suffix) + for ip in compose_gateways: + if ip not in seen: + uniq.append(ip) + seen.add(ip) + # Second: networks with active containers + for ip in active_gateways: + if ip not in seen: + uniq.append(ip) + seen.add(ip) + # Third: all other gateways + for ip in gateways: + if ip not in seen: + uniq.append(ip) + seen.add(ip) + + if uniq: + if len(uniq) > 1: + logger.info("Container-reachable host IP candidates: %s", ", ".join(uniq)) + else: + logger.info("Container-reachable host IP: %s", uniq[0]) + return uniq[0] + + # Nothing found: warn clearly + logger.warning( + "No container bridge IP found. If using rootless Podman (slirp4netns), there is no host bridge; publish ports or use 10.0.2.2 from the container." + ) + # Returning localhost is honest only for same-namespace; keep it explicit: + return "127.0.0.1" + + def cleanup(self): + """Cleanup resources but keep docling-serve running across sessions.""" + # Don't stop the process on exit - let it persist + # Just clean up our references + self._add_log_entry("TUI exiting - docling-serve will continue running") + # Note: We keep the PID file so we can reconnect in future sessions + def _save_pid(self, pid: int) -> None: + """Save the process PID to a file for persistence across sessions.""" + try: + self._pid_file.write_text(str(pid)) + self._add_log_entry(f"Saved PID {pid} to {self._pid_file}") + except Exception as e: + self._add_log_entry(f"Failed to save PID file: {e}") + + def _load_pid(self) -> Optional[int]: + """Load the process PID from file.""" + try: + if self._pid_file.exists(): + pid_str = self._pid_file.read_text().strip() + if pid_str.isdigit(): + return int(pid_str) + except Exception as e: + self._add_log_entry(f"Failed to load PID file: {e}") + return None + + def _clear_pid_file(self) -> None: + """Remove the PID file.""" + try: + if self._pid_file.exists(): + self._pid_file.unlink() + self._add_log_entry("Cleared PID file") + except Exception as e: + self._add_log_entry(f"Failed to clear PID file: {e}") + + def _is_process_running(self, pid: int) -> bool: + """Check if a process with the given PID is running.""" + try: + # Send signal 0 to check if process exists (doesn't actually send a signal) + os.kill(pid, 0) + return True + except OSError: + return False + + def _recover_from_pid_file(self) -> None: + """Try to recover connection to existing docling-serve process from PID file.""" + pid = self._load_pid() + if pid is not None: + if self._is_process_running(pid): + self._add_log_entry(f"Recovered existing docling-serve process (PID: {pid})") + # Mark as external process since we didn't start it in this session + self._external_process = True + self._running = True + # Note: We don't have a Popen object for this process, but that's OK + # We'll detect it's running via port check + else: + self._add_log_entry(f"Stale PID file found (PID: {pid} not running)") + self._clear_pid_file() + def _add_log_entry(self, message: str) -> None: """Add a log entry to the buffer (thread-safe).""" timestamp = time.strftime("%Y-%m-%d %H:%M:%S") @@ -70,43 +250,35 @@ class DoclingManager: self._log_buffer = self._log_buffer[-self._max_log_lines:] def is_running(self) -> bool: - """Check if docling serve is running.""" - # First check our internal state - internal_running = self._running and self._process is not None and self._process.poll() is None - - # If we think it's not running, check if something is listening on the port - # This handles cases where docling-serve was started outside the TUI - if not internal_running: - try: - import socket - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(0.5) - result = s.connect_ex((self._host, self._port)) - s.close() - - # If port is in use, something is running there - if result == 0: - # Only log this once when we first detect external process - if not self._external_process: - self._add_log_entry(f"Detected external docling-serve running on {self._host}:{self._port}") - # Set a flag to indicate this is an external process - self._external_process = True - return True - except Exception as e: - # Only log errors occasionally to avoid spam - if not hasattr(self, '_last_port_error') or self._last_port_error != str(e): - self._add_log_entry(f"Error checking port: {e}") - self._last_port_error = str(e) - else: - # If we started it, it's not external + """Check if docling serve is running (by PID only).""" + # Check if we have a direct process handle + if self._process is not None and self._process.poll() is None: + self._running = True self._external_process = False + return True - return internal_running + # Check if we have a PID from file + pid = self._load_pid() + if pid is not None and self._is_process_running(pid): + self._running = True + self._external_process = True + return True + + # No running process found + self._running = False + self._external_process = False + return False def get_status(self) -> Dict[str, Any]: """Get current status of docling serve.""" if self.is_running(): - pid = self._process.pid if self._process else None + # Try to get PID from process handle first, then from PID file + pid = None + if self._process: + pid = self._process.pid + else: + pid = self._load_pid() + return { "status": "running", "port": self._port, @@ -127,13 +299,28 @@ class DoclingManager: "pid": None } - async def start(self, port: int = 5001, host: str = "127.0.0.1", enable_ui: bool = False) -> Tuple[bool, str]: + async def start(self, port: int = 5001, host: Optional[str] = None, enable_ui: bool = False) -> Tuple[bool, str]: """Start docling serve as external process.""" if self.is_running(): return False, "Docling serve is already running" self._port = port - self._host = host + # Use provided host or the bridge IP we detected in __init__ + if host is not None: + self._host = host + # else: keep self._host as already set in __init__ + + # Check if port is already in use before trying to start + import socket + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(0.5) + result = s.connect_ex((self._host, self._port)) + s.close() + if result == 0: + return False, f"Port {self._port} on {self._host} is already in use by another process. Please stop it first." + except Exception as e: + self._add_log_entry(f"Error checking port availability: {e}") # Clear log buffer when starting self._log_buffer = [] @@ -146,14 +333,14 @@ class DoclingManager: if shutil.which("uv") and (os.path.exists("pyproject.toml") or os.getenv("VIRTUAL_ENV")): cmd = [ "uv", "run", "python", "-m", "docling_serve", "run", - "--host", host, - "--port", str(port), + "--host", self._host, + "--port", str(self._port), ] else: cmd = [ sys.executable, "-m", "docling_serve", "run", - "--host", host, - "--port", str(port), + "--host", self._host, + "--port", str(self._port), ] if enable_ui: @@ -173,6 +360,9 @@ class DoclingManager: self._running = True self._add_log_entry("External process started") + # Save the PID to file for persistence + self._save_pid(self._process.pid) + # Start a thread to capture output self._start_output_capture() @@ -192,11 +382,11 @@ class DoclingManager: import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(0.5) - result = s.connect_ex((host, port)) + result = s.connect_ex((self._host, self._port)) s.close() if result == 0: - self._add_log_entry(f"Docling-serve is now listening on {host}:{port}") + self._add_log_entry(f"Docling-serve is now listening on {self._host}:{self._port}") break except: pass @@ -298,9 +488,12 @@ class DoclingManager: try: self._add_log_entry("Stopping docling-serve process") + pid_to_stop = None + if self._process: - # We started this process, so we can stop it directly - self._add_log_entry(f"Terminating our process (PID: {self._process.pid})") + # We have a direct process handle + pid_to_stop = self._process.pid + self._add_log_entry(f"Terminating our process (PID: {pid_to_stop})") self._process.terminate() # Wait for it to stop @@ -315,16 +508,32 @@ class DoclingManager: self._add_log_entry("Process force killed") elif self._external_process: - # This is an external process, we can't stop it directly - self._add_log_entry("Cannot stop external docling-serve process - it was started outside the TUI") - self._running = False - self._external_process = False - return False, "Cannot stop external docling-serve process. Please stop it manually." + # This is a process we recovered from PID file + pid_to_stop = self._load_pid() + if pid_to_stop and self._is_process_running(pid_to_stop): + self._add_log_entry(f"Stopping process from PID file (PID: {pid_to_stop})") + try: + os.kill(pid_to_stop, 15) # SIGTERM + # Wait a bit for graceful shutdown + await asyncio.sleep(2) + if self._is_process_running(pid_to_stop): + # Still running, force kill + self._add_log_entry(f"Force killing process (PID: {pid_to_stop})") + os.kill(pid_to_stop, 9) # SIGKILL + except Exception as e: + self._add_log_entry(f"Error stopping external process: {e}") + return False, f"Error stopping external process: {str(e)}" + else: + self._add_log_entry("External process not found") + return False, "Process not found" self._running = False self._process = None self._external_process = False + # Clear the PID file since we intentionally stopped the service + self._clear_pid_file() + self._add_log_entry("Docling serve stopped successfully") return True, "Docling serve stopped successfully" diff --git a/src/tui/screens/welcome.py b/src/tui/screens/welcome.py index 9c121022..217b0611 100644 --- a/src/tui/screens/welcome.py +++ b/src/tui/screens/welcome.py @@ -336,8 +336,31 @@ class WelcomeScreen(Screen): self.call_after_refresh(self._focus_appropriate_button) async def _start_all_services(self) -> None: - """Start all services: native first, then containers.""" - # Step 1: Start native services (docling-serve) + """Start all services: containers first, then native services.""" + # Step 1: Start container services first (to create the network) + if self.container_manager.is_available(): + command_generator = self.container_manager.start_services() + modal = CommandOutputModal( + "Starting Container Services", + command_generator, + on_complete=self._on_containers_started_start_native, + ) + self.app.push_screen(modal) + else: + self.notify("No container runtime available", severity="warning") + # Still try to start native services + await self._start_native_services_after_containers() + + async def _on_containers_started_start_native(self) -> None: + """Called after containers start successfully, now start native services.""" + # Update container state + self._detect_services_sync() + + # Now start native services (docling-serve can now detect the compose network) + await self._start_native_services_after_containers() + + async def _start_native_services_after_containers(self) -> None: + """Start native services after containers have been started.""" if not self.docling_manager.is_running(): self.notify("Starting native services...", severity="information") success, message = await self.docling_manager.start() @@ -345,25 +368,12 @@ class WelcomeScreen(Screen): self.notify(message, severity="information") else: self.notify(f"Failed to start native services: {message}", severity="error") - # Continue anyway - user might want containers even if native fails else: self.notify("Native services already running", severity="information") # Update state self.docling_running = self.docling_manager.is_running() - - # Step 2: Start container services - if self.container_manager.is_available(): - command_generator = self.container_manager.start_services() - modal = CommandOutputModal( - "Starting Container Services", - command_generator, - on_complete=self._on_services_operation_complete, - ) - self.app.push_screen(modal) - else: - self.notify("No container runtime available", severity="warning") - await self._refresh_welcome_content() + await self._refresh_welcome_content() async def _stop_all_services(self) -> None: """Stop all services: containers first, then native."""