From b493dab318cba30b06e496cdfc0b52b3aac5ac44 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Thu, 4 Sep 2025 18:33:16 -0300 Subject: [PATCH] Enhance KnowledgeSourcesPage with ingestion settings and connector management This commit refactors the KnowledgeSourcesPage component to include a new ingestion settings section, allowing users to configure document processing parameters such as chunk size and overlap. It also improves the connector management interface by integrating async fetching of connector statuses and enhancing error handling. The changes aim to provide a more robust and user-friendly experience while maintaining well-documented code practices. --- frontend/src/app/settings/page.tsx | 1225 ++++++++++++++++------------ 1 file changed, 722 insertions(+), 503 deletions(-) diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index e1352434..1e88b12c 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -1,539 +1,758 @@ -"use client" - -import { useState, useEffect, useCallback, Suspense } from "react" -import { useSearchParams } from "next/navigation" -import { Button } from "@/components/ui/button" -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" -import { Badge } from "@/components/ui/badge" -import { Input } from "@/components/ui/input" -import { Label } from "@/components/ui/label" -import { Checkbox } from "@/components/ui/checkbox" -import { Loader2, PlugZap, RefreshCw } from "lucide-react" -import { ProtectedRoute } from "@/components/protected-route" -import { useTask } from "@/contexts/task-context" -import { useAuth } from "@/contexts/auth-context" +"use client"; +import { useState, useEffect, useCallback, Suspense } from "react"; +import { useSearchParams } from "next/navigation"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Loader2, PlugZap, RefreshCw } from "lucide-react"; +import { ProtectedRoute } from "@/components/protected-route"; +import { useTask } from "@/contexts/task-context"; +import { useAuth } from "@/contexts/auth-context"; interface Connector { - id: string - name: string - description: string - icon: React.ReactNode - status: "not_connected" | "connecting" | "connected" | "error" - type: string - connectionId?: string - access_token?: string + id: string; + name: string; + description: string; + icon: React.ReactNode; + status: "not_connected" | "connecting" | "connected" | "error"; + type: string; + connectionId?: string; + access_token?: string; } interface SyncResult { - processed?: number; - added?: number; - errors?: number; - skipped?: number; - total?: number; + processed?: number; + added?: number; + errors?: number; + skipped?: number; + total?: number; } interface Connection { - connection_id: string - is_active: boolean - created_at: string - last_sync?: string + connection_id: string; + is_active: boolean; + created_at: string; + last_sync?: string; } function KnowledgeSourcesPage() { - const { isAuthenticated, isNoAuthMode } = useAuth() - const { addTask, tasks } = useTask() - const searchParams = useSearchParams() - - - // Connectors state - const [connectors, setConnectors] = useState([]) - const [isConnecting, setIsConnecting] = useState(null) - const [isSyncing, setIsSyncing] = useState(null) - const [syncResults, setSyncResults] = useState<{[key: string]: SyncResult | null}>({}) - const [maxFiles, setMaxFiles] = useState(10) - const [syncAllFiles, setSyncAllFiles] = useState(false) - - // Settings state - // Note: backend internal Langflow URL is not needed on the frontend - const [flowId, setFlowId] = useState('1098eea1-6649-4e1d-aed1-b77249fb8dd0') - const [ingestFlowId, setIngestFlowId] = useState('') - const [langflowEditUrl, setLangflowEditUrl] = useState('') - const [langflowIngestEditUrl, setLangflowIngestEditUrl] = useState('') - const [publicLangflowUrl, setPublicLangflowUrl] = useState('') + const { isAuthenticated, isNoAuthMode } = useAuth(); + const { addTask, tasks } = useTask(); + const searchParams = useSearchParams(); - // Fetch settings from backend - const fetchSettings = useCallback(async () => { - try { - const response = await fetch('/api/settings') - if (response.ok) { - const settings = await response.json() - if (settings.flow_id) { - setFlowId(settings.flow_id) - } - if (settings.ingest_flow_id) { - console.log('Setting ingestFlowId to:', settings.ingest_flow_id) - setIngestFlowId(settings.ingest_flow_id) - } else { - console.log('No ingest_flow_id in settings:', settings) - } - if (settings.langflow_edit_url) { - setLangflowEditUrl(settings.langflow_edit_url) - } - if (settings.langflow_ingest_edit_url) { - setLangflowIngestEditUrl(settings.langflow_ingest_edit_url) - } - if (settings.langflow_public_url) { - setPublicLangflowUrl(settings.langflow_public_url) - } - } - } catch (error) { - console.error('Failed to fetch settings:', error) - } - }, []) + // Connectors state + const [connectors, setConnectors] = useState([]); + const [isConnecting, setIsConnecting] = useState(null); + const [isSyncing, setIsSyncing] = useState(null); + const [syncResults, setSyncResults] = useState<{ + [key: string]: SyncResult | null; + }>({}); + const [maxFiles, setMaxFiles] = useState(10); + const [syncAllFiles, setSyncAllFiles] = useState(false); - // Helper function to get connector icon - const getConnectorIcon = (iconName: string) => { - const iconMap: { [key: string]: React.ReactElement } = { - 'google-drive': ( -
- G -
- ), - 'sharepoint': ( -
- SP -
- ), - 'onedrive': ( -
- OD -
- ), - } - return iconMap[iconName] || ( -
- ? -
- ) - } + // Settings state + // Note: backend internal Langflow URL is not needed on the frontend + const [flowId, setFlowId] = useState( + "1098eea1-6649-4e1d-aed1-b77249fb8dd0", + ); + const [ingestFlowId, setIngestFlowId] = useState(""); + const [langflowEditUrl, setLangflowEditUrl] = useState(""); + const [langflowIngestEditUrl, setLangflowIngestEditUrl] = + useState(""); + const [publicLangflowUrl, setPublicLangflowUrl] = useState(""); - // Connector functions - const checkConnectorStatuses = useCallback(async () => { - try { - // Fetch available connectors from backend - const connectorsResponse = await fetch('/api/connectors') - if (!connectorsResponse.ok) { - throw new Error('Failed to load connectors') - } - - const connectorsResult = await connectorsResponse.json() - const connectorTypes = Object.keys(connectorsResult.connectors) - - // Initialize connectors list with metadata from backend - const initialConnectors = connectorTypes - .filter(type => connectorsResult.connectors[type].available) // Only show available connectors - .map(type => ({ - id: type, - name: connectorsResult.connectors[type].name, - description: connectorsResult.connectors[type].description, - icon: getConnectorIcon(connectorsResult.connectors[type].icon), - status: "not_connected" as const, - type: type - })) - - setConnectors(initialConnectors) + // Ingestion settings state - will be populated from Langflow flow defaults + const [ingestionSettings, setIngestionSettings] = useState({ + chunkSize: 1000, + chunkOverlap: 200, + separator: "\\n", + embeddingModel: "text-embedding-3-small", + }); + const [settingsLoaded, setSettingsLoaded] = useState(false); - // Check status for each connector type - - for (const connectorType of connectorTypes) { - const response = await fetch(`/api/connectors/${connectorType}/status`) - if (response.ok) { - const data = await response.json() - const connections = data.connections || [] - const activeConnection = connections.find((conn: Connection) => conn.is_active) - const isConnected = activeConnection !== undefined - - setConnectors(prev => prev.map(c => - c.type === connectorType - ? { - ...c, - status: isConnected ? "connected" : "not_connected", - connectionId: activeConnection?.connection_id - } - : c - )) - } - } - } catch (error) { - console.error('Failed to check connector statuses:', error) - } - }, []) + // Fetch settings from backend + const fetchSettings = useCallback(async () => { + try { + const response = await fetch("/api/settings"); + if (response.ok) { + const settings = await response.json(); + if (settings.flow_id) { + setFlowId(settings.flow_id); + } + if (settings.ingest_flow_id) { + console.log("Setting ingestFlowId to:", settings.ingest_flow_id); + setIngestFlowId(settings.ingest_flow_id); + } else { + console.log("No ingest_flow_id in settings:", settings); + } + if (settings.langflow_edit_url) { + setLangflowEditUrl(settings.langflow_edit_url); + } + if (settings.langflow_ingest_edit_url) { + setLangflowIngestEditUrl(settings.langflow_ingest_edit_url); + } + if (settings.langflow_public_url) { + setPublicLangflowUrl(settings.langflow_public_url); + } + if (settings.ingestion_defaults) { + console.log( + "Loading ingestion defaults from backend:", + settings.ingestion_defaults, + ); + setIngestionSettings(settings.ingestion_defaults); + setSettingsLoaded(true); + } + } + } catch (error) { + console.error("Failed to fetch settings:", error); + } + }, []); - const handleConnect = async (connector: Connector) => { - setIsConnecting(connector.id) - setSyncResults(prev => ({ ...prev, [connector.id]: null })) - - try { - // Use the shared auth callback URL, same as connectors page - const redirectUri = `${window.location.origin}/auth/callback` - - const response = await fetch('/api/auth/init', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - connector_type: connector.type, - purpose: "data_source", - name: `${connector.name} Connection`, - redirect_uri: redirectUri - }), - }) - - if (response.ok) { - const result = await response.json() - - if (result.oauth_config) { - localStorage.setItem('connecting_connector_id', result.connection_id) - localStorage.setItem('connecting_connector_type', connector.type) - - const authUrl = `${result.oauth_config.authorization_endpoint}?` + - `client_id=${result.oauth_config.client_id}&` + - `response_type=code&` + - `scope=${result.oauth_config.scopes.join(' ')}&` + - `redirect_uri=${encodeURIComponent(result.oauth_config.redirect_uri)}&` + - `access_type=offline&` + - `prompt=consent&` + - `state=${result.connection_id}` - - window.location.href = authUrl - } - } else { - console.error('Failed to initiate connection') - setIsConnecting(null) - } - } catch (error) { - console.error('Connection error:', error) - setIsConnecting(null) - } - } + // Helper function to get connector icon + const getConnectorIcon = (iconName: string) => { + const iconMap: { [key: string]: React.ReactElement } = { + "google-drive": ( +
+ G +
+ ), + sharepoint: ( +
+ SP +
+ ), + onedrive: ( +
+ OD +
+ ), + }; + return ( + iconMap[iconName] || ( +
+ ? +
+ ) + ); + }; - const handleSync = async (connector: Connector) => { - if (!connector.connectionId) return - - setIsSyncing(connector.id) - setSyncResults(prev => ({ ...prev, [connector.id]: null })) - - try { - const response = await fetch(`/api/connectors/${connector.type}/sync`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - connection_id: connector.connectionId, - max_files: syncAllFiles ? 0 : (maxFiles || undefined) - }), - }) - - const result = await response.json() - - if (response.status === 201) { - const taskId = result.task_id - if (taskId) { - addTask(taskId) - setSyncResults(prev => ({ - ...prev, - [connector.id]: { - processed: 0, - total: result.total_files || 0 - } - })) - } - } else if (response.ok) { - setSyncResults(prev => ({ ...prev, [connector.id]: result })) - // Note: Stats will auto-refresh via task completion watcher for async syncs - } else { - console.error('Sync failed:', result.error) - } - } catch (error) { - console.error('Sync error:', error) - } finally { - setIsSyncing(null) - } - } + // Connector functions + const checkConnectorStatuses = useCallback(async () => { + try { + // Fetch available connectors from backend + const connectorsResponse = await fetch("/api/connectors"); + if (!connectorsResponse.ok) { + throw new Error("Failed to load connectors"); + } - const getStatusBadge = (status: Connector["status"]) => { - switch (status) { - case "connected": - return Connected - case "connecting": - return Connecting... - case "error": - return Error - default: - return Not Connected - } - } + const connectorsResult = await connectorsResponse.json(); + const connectorTypes = Object.keys(connectorsResult.connectors); - // Fetch settings on mount when authenticated - useEffect(() => { - if (isAuthenticated) { - fetchSettings() - } - }, [isAuthenticated, fetchSettings]) + // Initialize connectors list with metadata from backend + const initialConnectors = connectorTypes + .filter((type) => connectorsResult.connectors[type].available) // Only show available connectors + .map((type) => ({ + id: type, + name: connectorsResult.connectors[type].name, + description: connectorsResult.connectors[type].description, + icon: getConnectorIcon(connectorsResult.connectors[type].icon), + status: "not_connected" as const, + type: type, + })); - // Check connector status on mount and when returning from OAuth - useEffect(() => { - if (isAuthenticated) { - checkConnectorStatuses() - } - - if (searchParams.get('oauth_success') === 'true') { - const url = new URL(window.location.href) - url.searchParams.delete('oauth_success') - window.history.replaceState({}, '', url.toString()) - } - }, [searchParams, isAuthenticated, checkConnectorStatuses]) + setConnectors(initialConnectors); + // Check status for each connector type + for (const connectorType of connectorTypes) { + const response = await fetch(`/api/connectors/${connectorType}/status`); + if (response.ok) { + const data = await response.json(); + const connections = data.connections || []; + const activeConnection = connections.find( + (conn: Connection) => conn.is_active, + ); + const isConnected = activeConnection !== undefined; + setConnectors((prev) => + prev.map((c) => + c.type === connectorType + ? { + ...c, + status: isConnected ? "connected" : "not_connected", + connectionId: activeConnection?.connection_id, + } + : c, + ), + ); + } + } + } catch (error) { + console.error("Failed to check connector statuses:", error); + } + }, []); - // Track previous tasks to detect new completions - const [prevTasks, setPrevTasks] = useState([]) - - // Watch for task completions and refresh stats - useEffect(() => { - // Find newly completed tasks by comparing with previous state - const newlyCompletedTasks = tasks.filter(task => { - const wasCompleted = prevTasks.find(prev => prev.task_id === task.task_id)?.status === 'completed' - return task.status === 'completed' && !wasCompleted - }) - - if (newlyCompletedTasks.length > 0) { - // Task completed - could refresh data here if needed - const timeoutId = setTimeout(() => { - // Stats refresh removed - }, 1000) - - // Update previous tasks state - setPrevTasks(tasks) - - return () => clearTimeout(timeoutId) - } else { - // Always update previous tasks state - setPrevTasks(tasks) - } - }, [tasks, prevTasks]) + const handleConnect = async (connector: Connector) => { + setIsConnecting(connector.id); + setSyncResults((prev) => ({ ...prev, [connector.id]: null })); - return ( -
- {/* Agent Behavior Section */} -
-
-

Agent behavior

-

Adjust your retrieval agent flow

-
- -
+ try { + // Use the shared auth callback URL, same as connectors page + const redirectUri = `${window.location.origin}/auth/callback`; - {/* Ingest Flow Section */} -
-
-

File ingestion

-

Customize your file processing and indexing pipeline

-
- -
+ const response = await fetch("/api/auth/init", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + connector_type: connector.type, + purpose: "data_source", + name: `${connector.name} Connection`, + redirect_uri: redirectUri, + }), + }); + if (response.ok) { + const result = await response.json(); - {/* Connectors Section */} -
-
-

Cloud Connectors

-
+ if (result.oauth_config) { + localStorage.setItem("connecting_connector_id", result.connection_id); + localStorage.setItem("connecting_connector_type", connector.type); - {/* Conditional Sync Settings or No-Auth Message */} - {isNoAuthMode ? ( - - - Cloud connectors are only available with auth mode enabled - - Please provide the following environment variables and restart: - - - -
-
# make here https://console.cloud.google.com/apis/credentials
-
GOOGLE_OAUTH_CLIENT_ID=
-
GOOGLE_OAUTH_CLIENT_SECRET=
-
- - - ) : ( -
-
-

Sync Settings

-

Configure how many files to sync when manually triggering a sync

-
-
-
- { - setSyncAllFiles(!!checked) - if (checked) { - setMaxFiles(0) - } else { - setMaxFiles(10) - } - }} - /> - -
- -
- setMaxFiles(parseInt(e.target.value) || 10)} - disabled={syncAllFiles} - className="w-16 min-w-16 max-w-16 flex-shrink-0 disabled:opacity-50 disabled:cursor-not-allowed" - min="1" - max="100" - title={syncAllFiles ? "Disabled when 'Sync all files' is checked" : "Leave blank or set to 0 for unlimited"} - /> -
-
-
- )} + const authUrl = + `${result.oauth_config.authorization_endpoint}?` + + `client_id=${result.oauth_config.client_id}&` + + `response_type=code&` + + `scope=${result.oauth_config.scopes.join(" ")}&` + + `redirect_uri=${encodeURIComponent(result.oauth_config.redirect_uri)}&` + + `access_type=offline&` + + `prompt=consent&` + + `state=${result.connection_id}`; - {/* Connectors Grid */} -
- {connectors.map((connector) => ( - - -
-
- {connector.icon} -
- {connector.name} - - {connector.description} - -
-
- {getStatusBadge(connector.status)} -
- - - {connector.status === "connected" ? ( -
- - - {syncResults[connector.id] && ( -
-
Processed: {syncResults[connector.id]?.processed || 0}
-
Added: {syncResults[connector.id]?.added || 0}
- {syncResults[connector.id]?.errors && ( -
Errors: {syncResults[connector.id]?.errors}
- )} -
- )} -
- ) : ( - - )} - - - ))} -
+ window.location.href = authUrl; + } + } else { + console.error("Failed to initiate connection"); + setIsConnecting(null); + } + } catch (error) { + console.error("Connection error:", error); + setIsConnecting(null); + } + }; -
-
- ) + const handleSync = async (connector: Connector) => { + if (!connector.connectionId) return; + + setIsSyncing(connector.id); + setSyncResults((prev) => ({ ...prev, [connector.id]: null })); + + try { + const response = await fetch(`/api/connectors/${connector.type}/sync`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + connection_id: connector.connectionId, + max_files: syncAllFiles ? 0 : maxFiles || undefined, + }), + }); + + const result = await response.json(); + + if (response.status === 201) { + const taskId = result.task_id; + if (taskId) { + addTask(taskId); + setSyncResults((prev) => ({ + ...prev, + [connector.id]: { + processed: 0, + total: result.total_files || 0, + }, + })); + } + } else if (response.ok) { + setSyncResults((prev) => ({ ...prev, [connector.id]: result })); + // Note: Stats will auto-refresh via task completion watcher for async syncs + } else { + console.error("Sync failed:", result.error); + } + } catch (error) { + console.error("Sync error:", error); + } finally { + setIsSyncing(null); + } + }; + + const getStatusBadge = (status: Connector["status"]) => { + switch (status) { + case "connected": + return ( + + Connected + + ); + case "connecting": + return ( + + Connecting... + + ); + case "error": + return Error; + default: + return ( + + Not Connected + + ); + } + }; + + // Fetch settings on mount when authenticated + useEffect(() => { + if (isAuthenticated) { + fetchSettings(); + } + }, [isAuthenticated, fetchSettings]); + + // Check connector status on mount and when returning from OAuth + useEffect(() => { + if (isAuthenticated) { + checkConnectorStatuses(); + } + + if (searchParams.get("oauth_success") === "true") { + const url = new URL(window.location.href); + url.searchParams.delete("oauth_success"); + window.history.replaceState({}, "", url.toString()); + } + }, [searchParams, isAuthenticated, checkConnectorStatuses]); + + // Track previous tasks to detect new completions + const [prevTasks, setPrevTasks] = useState([]); + + // Watch for task completions and refresh stats + useEffect(() => { + // Find newly completed tasks by comparing with previous state + const newlyCompletedTasks = tasks.filter((task) => { + const wasCompleted = + prevTasks.find((prev) => prev.task_id === task.task_id)?.status === + "completed"; + return task.status === "completed" && !wasCompleted; + }); + + if (newlyCompletedTasks.length > 0) { + // Task completed - could refresh data here if needed + const timeoutId = setTimeout(() => { + // Stats refresh removed + }, 1000); + + // Update previous tasks state + setPrevTasks(tasks); + + return () => clearTimeout(timeoutId); + } else { + // Always update previous tasks state + setPrevTasks(tasks); + } + }, [tasks, prevTasks]); + + return ( +
+ {/* Agent Behavior Section */} +
+
+

Agent behavior

+

+ Adjust your retrieval agent flow +

+
+ +
+ + {/* Ingest Flow Section */} +
+
+

File ingestion

+

+ Customize your file processing and indexing flow +

+
+ +
+ + {/* Ingestion Settings Section */} +
+
+

Ingestion settings

+

+ Configure how your documents are processed and indexed +

+
+ +
+ + + Document Processing + + Control how text is split and processed + + + +
+ + + setIngestionSettings((prev) => ({ + ...prev, + chunkSize: parseInt(e.target.value) || 1000, + })) + } + min="100" + max="4000" + /> +

+ Maximum characters per text chunk (100-4000) +

+
+ +
+ + + setIngestionSettings((prev) => ({ + ...prev, + chunkOverlap: parseInt(e.target.value) || 200, + })) + } + min="0" + max="500" + /> +

+ Character overlap between chunks (0-500) +

+
+ + + + + + Embeddings + + Configure embedding model and search behavior + + + +
+ + +
+ + + +
+
+ + {/* Connectors Section */} +
+
+

+ Cloud Connectors +

+
+ + {/* Conditional Sync Settings or No-Auth Message */} + {isNoAuthMode ? ( + + + + Cloud connectors are only available with auth mode enabled + + + Please provide the following environment variables and restart: + + + +
+
+ # make here https://console.cloud.google.com/apis/credentials +
+
GOOGLE_OAUTH_CLIENT_ID=
+
GOOGLE_OAUTH_CLIENT_SECRET=
+
+ + + ) : ( +
+
+

Sync Settings

+

+ Configure how many files to sync when manually triggering a sync +

+
+
+
+ { + setSyncAllFiles(!!checked); + if (checked) { + setMaxFiles(0); + } else { + setMaxFiles(10); + } + }} + /> + +
+ +
+ setMaxFiles(parseInt(e.target.value) || 10)} + disabled={syncAllFiles} + className="w-16 min-w-16 max-w-16 flex-shrink-0 disabled:opacity-50 disabled:cursor-not-allowed" + min="1" + max="100" + title={ + syncAllFiles + ? "Disabled when 'Sync all files' is checked" + : "Leave blank or set to 0 for unlimited" + } + /> +
+
+
+ )} + + {/* Connectors Grid */} +
+ {connectors.map((connector) => ( + + +
+
+ {connector.icon} +
+ + {connector.name} + + + {connector.description} + +
+
+ {getStatusBadge(connector.status)} +
+ + + {connector.status === "connected" ? ( +
+ + + {syncResults[connector.id] && ( +
+
+ Processed: {syncResults[connector.id]?.processed || 0} +
+
+ Added: {syncResults[connector.id]?.added || 0} +
+ {syncResults[connector.id]?.errors && ( +
Errors: {syncResults[connector.id]?.errors}
+ )} +
+ )} +
+ ) : ( + + )} + + + ))} +
+
+
+ ); } export default function ProtectedKnowledgeSourcesPage() { - return ( - - Loading knowledge sources...}> - - - - ) + return ( + + Loading knowledge sources...}> + + + + ); }