From ba22091f31b82f833933523f2d893a89d87ee8f3 Mon Sep 17 00:00:00 2001 From: Mike Fortman Date: Fri, 26 Sep 2025 16:44:15 -0500 Subject: [PATCH 01/55] add banner for docling serve --- frontend/components/docling-health-banner.tsx | 134 +++++++++++++++++ frontend/components/ui/banner.tsx | 141 ++++++++++++++++++ .../app/api/queries/useDoclingHealthQuery.ts | 55 +++++++ frontend/src/app/chat/page.tsx | 5 +- frontend/src/app/knowledge/chunks/page.tsx | 5 +- frontend/src/app/knowledge/page.tsx | 5 +- frontend/src/components/layout-wrapper.tsx | 26 +++- frontend/src/contexts/layout-context.tsx | 34 +++++ src/tui/managers/env_manager.py | 2 +- 9 files changed, 400 insertions(+), 7 deletions(-) create mode 100644 frontend/components/docling-health-banner.tsx create mode 100644 frontend/components/ui/banner.tsx create mode 100644 frontend/src/app/api/queries/useDoclingHealthQuery.ts create mode 100644 frontend/src/contexts/layout-context.tsx diff --git a/frontend/components/docling-health-banner.tsx b/frontend/components/docling-health-banner.tsx new file mode 100644 index 00000000..c65a93cc --- /dev/null +++ b/frontend/components/docling-health-banner.tsx @@ -0,0 +1,134 @@ +"use client"; + +import { AlertTriangle, ExternalLink, Copy } from "lucide-react"; +import { useDoclingHealthQuery } from "@/src/app/api/queries/useDoclingHealthQuery"; +import { Banner, BannerIcon, BannerTitle, BannerAction } from "@/components/ui/banner"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter +} from "@/components/ui/dialog"; +import { cn } from "@/lib/utils"; +import { useState } from "react"; + +interface DoclingHealthBannerProps { + className?: string; +} + +// DoclingSetupDialog component +interface DoclingSetupDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + className?: string; +} + +function DoclingSetupDialog({ + open, + onOpenChange, + className +}: DoclingSetupDialogProps) { + const [copied, setCopied] = useState(false); + + const handleCopy = async () => { + await navigator.clipboard.writeText("uv run openrag"); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }; + + return ( + + + + + + docling-serve is stopped. Knowledge ingest is unavailable. + + + Start docling-serve by running: + + + +
+
+ + uv run openrag + + +
+ + + Then, select Start Native Services in the TUI. Once docling-serve is running, refresh OpenRAG. + +
+ + + + + + + ); +} + +export function DoclingHealthBanner({ className }: DoclingHealthBannerProps) { + const { data: health, isLoading, isError } = useDoclingHealthQuery(); + const [showDialog, setShowDialog] = useState(false); + + const isHealthy = health?.status === "healthy" && !isError; + const isUnhealthy = health?.status === "unhealthy" || isError; + + // Only show banner when service is unhealthy + if (isLoading || isHealthy) { + return null; + } + + if (isUnhealthy) { + return ( + <> + + + + docling-serve native service is stopped. Knowledge ingest is unavailable. + + setShowDialog(true)} + className="bg-foreground text-background hover:bg-primary/90" + > + Setup Docling Serve + + + + + + + ); + } + + return null; +} \ No newline at end of file diff --git a/frontend/components/ui/banner.tsx b/frontend/components/ui/banner.tsx new file mode 100644 index 00000000..3a1ea9f5 --- /dev/null +++ b/frontend/components/ui/banner.tsx @@ -0,0 +1,141 @@ +'use client'; +import { useControllableState } from '@radix-ui/react-use-controllable-state'; +import { type LucideIcon, XIcon } from 'lucide-react'; +import { + type ComponentProps, + createContext, + type HTMLAttributes, + type MouseEventHandler, + useContext, +} from 'react'; +import { Button } from '@/components/ui/button'; +import { cn } from '@/lib/utils'; + +type BannerContextProps = { + show: boolean; + setShow: (show: boolean) => void; +}; + +export const BannerContext = createContext({ + show: true, + setShow: () => {}, +}); + +export type BannerProps = HTMLAttributes & { + visible?: boolean; + defaultVisible?: boolean; + onClose?: () => void; + inset?: boolean; +}; + +export const Banner = ({ + children, + visible, + defaultVisible = true, + onClose, + className, + inset = false, + ...props +}: BannerProps) => { + const [show, setShow] = useControllableState({ + defaultProp: defaultVisible, + prop: visible, + onChange: onClose, + }); + + if (!show) { + return null; + } + + return ( + +
+ {children} +
+ + ); +}; + +export type BannerIconProps = HTMLAttributes & { + icon: LucideIcon; +}; + +export const BannerIcon = ({ + icon: Icon, + className, + ...props +}: BannerIconProps) => ( +
+ +
+); + +export type BannerTitleProps = HTMLAttributes; + +export const BannerTitle = ({ className, ...props }: BannerTitleProps) => ( +

+); + +export type BannerActionProps = ComponentProps; + +export const BannerAction = ({ + variant = 'outline', + size = 'sm', + className, + ...props +}: BannerActionProps) => ( + + ); +}; \ No newline at end of file diff --git a/frontend/src/app/api/queries/useDoclingHealthQuery.ts b/frontend/src/app/api/queries/useDoclingHealthQuery.ts new file mode 100644 index 00000000..16ffc6c5 --- /dev/null +++ b/frontend/src/app/api/queries/useDoclingHealthQuery.ts @@ -0,0 +1,55 @@ +import { + type UseQueryOptions, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; + +export interface DoclingHealthResponse { + status: "healthy" | "unhealthy"; + message?: string; +} + +export const useDoclingHealthQuery = ( + options?: Omit, "queryKey" | "queryFn">, +) => { + const queryClient = useQueryClient(); + + async function checkDoclingHealth(): Promise { + try { + const response = await fetch("http://127.0.0.1:5001/health", { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }); + + if (response.ok) { + return { status: "healthy" }; + } else { + return { + status: "unhealthy", + message: `Health check failed with status: ${response.status}`, + }; + } + } catch (error) { + return { + status: "unhealthy", + message: error instanceof Error ? error.message : "Connection failed", + }; + } + } + + const queryResult = useQuery( + { + queryKey: ["docling-health"], + queryFn: checkDoclingHealth, + retry: 1, + refetchInterval: 30000, // Check every 30 seconds + staleTime: 25000, // Consider data stale after 25 seconds + ...options, + }, + queryClient, + ); + + return queryResult; +}; \ No newline at end of file diff --git a/frontend/src/app/chat/page.tsx b/frontend/src/app/chat/page.tsx index 072567ba..bcb9d75f 100644 --- a/frontend/src/app/chat/page.tsx +++ b/frontend/src/app/chat/page.tsx @@ -22,6 +22,7 @@ import { Button } from "@/components/ui/button"; import { useAuth } from "@/contexts/auth-context"; import { type EndpointType, useChat } from "@/contexts/chat-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; +import { useLayout } from "@/contexts/layout-context"; import { useTask } from "@/contexts/task-context"; import { useLoadingStore } from "@/stores/loadingStore"; import { useGetNudgesQuery } from "../api/queries/useGetNudgesQuery"; @@ -140,6 +141,7 @@ function ChatPage() { const streamIdRef = useRef(0); const lastLoadedConversationRef = useRef(null); const { addTask, isMenuOpen } = useTask(); + const { totalTopOffset } = useLayout(); const { selectedFilter, parsedFilterData, isPanelOpen, setSelectedFilter } = useKnowledgeFilter(); @@ -1891,7 +1893,7 @@ function ChatPage() { return (

{/* Debug header - only show in debug mode */} {isDebugMode && ( diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index cdc9fcc3..73af7e95 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -12,6 +12,7 @@ import { useRouter, useSearchParams } from "next/navigation"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; +import { useLayout } from "@/contexts/layout-context"; import { useTask } from "@/contexts/task-context"; import { type ChunkResult, @@ -33,6 +34,7 @@ function ChunksPageContent() { const router = useRouter(); const searchParams = useSearchParams(); const { isMenuOpen } = useTask(); + const { totalTopOffset } = useLayout(); const { parsedFilterData, isPanelOpen } = useKnowledgeFilter(); const filename = searchParams.get("filename"); @@ -132,7 +134,7 @@ function ChunksPageContent() { return (
{/* Header */} diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 5155f4e2..dc275396 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -11,6 +11,7 @@ import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; +import { useLayout } from "@/contexts/layout-context"; import { useTask } from "@/contexts/task-context"; import { type File, useGetSearchQuery } from "../api/queries/useGetSearchQuery"; import "@/components/AgGrid/registerAgGridModules"; @@ -46,6 +47,7 @@ function getSourceIcon(connectorType?: string) { function SearchPage() { const router = useRouter(); const { isMenuOpen, files: taskFiles } = useTask(); + const { totalTopOffset } = useLayout(); const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = useKnowledgeFilter(); const [selectedRows, setSelectedRows] = useState([]); @@ -229,7 +231,7 @@ function SearchPage() { return (
diff --git a/frontend/src/components/layout-wrapper.tsx b/frontend/src/components/layout-wrapper.tsx index 5c2f0e13..5b598eb9 100644 --- a/frontend/src/components/layout-wrapper.tsx +++ b/frontend/src/components/layout-wrapper.tsx @@ -16,6 +16,9 @@ import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; // import { GitHubStarButton } from "@/components/github-star-button" // import { DiscordLink } from "@/components/discord-link" import { useTask } from "@/contexts/task-context"; +import { DoclingHealthBanner } from "@/components/docling-health-banner"; +import { useDoclingHealthQuery } from "@/src/app/api/queries/useDoclingHealthQuery"; +import { LayoutProvider } from "@/contexts/layout-context"; export function LayoutWrapper({ children }: { children: React.ReactNode }) { const pathname = usePathname(); @@ -31,6 +34,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { const { isLoading: isSettingsLoading, data: settings } = useGetSettingsQuery({ enabled: isAuthenticated || isNoAuthMode, }); + const { data: health, isLoading: isHealthLoading, isError } = useDoclingHealthQuery(); // Only fetch conversations on chat page const isOnChatPage = pathname === "/" || pathname === "/chat"; @@ -56,6 +60,15 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { task.status === "processing", ); + const isUnhealthy = health?.status === "unhealthy" || isError; + const isBannerVisible = !isHealthLoading && isUnhealthy; + + // Dynamic height calculations based on banner visibility + const headerHeight = 53; + const bannerHeight = 52; // Approximate banner height + const totalTopOffset = isBannerVisible ? headerHeight + bannerHeight : headerHeight; + const mainContentHeight = `calc(100vh - ${totalTopOffset}px)`; + // Show loading state when backend isn't ready if (isLoading || isSettingsLoading) { return ( @@ -76,6 +89,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { // For all other pages, render with Langflow-styled navigation and task menu return (
+
{/* Logo/Title */} @@ -118,7 +132,10 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
-
+
-
{children}
+ +
{children}
+
diff --git a/frontend/src/contexts/layout-context.tsx b/frontend/src/contexts/layout-context.tsx new file mode 100644 index 00000000..f40ea28c --- /dev/null +++ b/frontend/src/contexts/layout-context.tsx @@ -0,0 +1,34 @@ +"use client"; + +import { createContext, useContext } from "react"; + +interface LayoutContextType { + headerHeight: number; + totalTopOffset: number; +} + +const LayoutContext = createContext(undefined); + +export function useLayout() { + const context = useContext(LayoutContext); + if (context === undefined) { + throw new Error("useLayout must be used within a LayoutProvider"); + } + return context; +} + +export function LayoutProvider({ + children, + headerHeight, + totalTopOffset +}: { + children: React.ReactNode; + headerHeight: number; + totalTopOffset: number; +}) { + return ( + + {children} + + ); +} \ No newline at end of file diff --git a/src/tui/managers/env_manager.py b/src/tui/managers/env_manager.py index 9954b463..282f73d0 100644 --- a/src/tui/managers/env_manager.py +++ b/src/tui/managers/env_manager.py @@ -32,7 +32,7 @@ class EnvConfig: langflow_superuser: str = "admin" langflow_superuser_password: str = "" langflow_chat_flow_id: str = "1098eea1-6649-4e1d-aed1-b77249fb8dd0" - langflow_ingest_flow_id: str = "5488df7c-b93f-4f87-a446-b67028bc0813" + langflow_ingest_flow_id: str = "1402618b-e6d1-4ff2-9a11-d6ce71186915" # OAuth settings google_oauth_client_id: str = "" From c5c5ed46f489218bfc635fde29bdb6cd54a7c852 Mon Sep 17 00:00:00 2001 From: Mike Fortman Date: Fri, 26 Sep 2025 16:46:28 -0500 Subject: [PATCH 02/55] update example --- .env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index ee2a838c..7867ffb9 100644 --- a/.env.example +++ b/.env.example @@ -7,9 +7,9 @@ LANGFLOW_SECRET_KEY= # flow ids for chat and ingestion flows LANGFLOW_CHAT_FLOW_ID=1098eea1-6649-4e1d-aed1-b77249fb8dd0 -LANGFLOW_INGEST_FLOW_ID=5488df7c-b93f-4f87-a446-b67028bc0813 +# LANGFLOW_INGEST_FLOW_ID=5488df7c-b93f-4f87-a446-b67028bc0813 # Ingest flow using docling -# LANGFLOW_INGEST_FLOW_ID=1402618b-e6d1-4ff2-9a11-d6ce71186915 +LANGFLOW_INGEST_FLOW_ID=1402618b-e6d1-4ff2-9a11-d6ce71186915 NUDGES_FLOW_ID=ebc01d31-1976-46ce-a385-b0240327226c # Set a strong admin password for OpenSearch; a bcrypt hash is generated at From f51960476e0466c502176d7dee2d24dbac36b958 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Mon, 29 Sep 2025 14:52:21 -0300 Subject: [PATCH 03/55] fix layout not showing on chat --- frontend/src/components/layout-wrapper.tsx | 26 ++++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/layout-wrapper.tsx b/frontend/src/components/layout-wrapper.tsx index 5b598eb9..7f352601 100644 --- a/frontend/src/components/layout-wrapper.tsx +++ b/frontend/src/components/layout-wrapper.tsx @@ -2,8 +2,12 @@ import { Bell, Loader2 } from "lucide-react"; import { usePathname } from "next/navigation"; -import { useGetConversationsQuery, type ChatConversation } from "@/app/api/queries/useGetConversationsQuery"; +import { + type ChatConversation, + useGetConversationsQuery, +} from "@/app/api/queries/useGetConversationsQuery"; import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery"; +import { DoclingHealthBanner } from "@/components/docling-health-banner"; import { KnowledgeFilterPanel } from "@/components/knowledge-filter-panel"; import Logo from "@/components/logo/logo"; import { Navigation } from "@/components/navigation"; @@ -13,12 +17,11 @@ import { UserNav } from "@/components/user-nav"; import { useAuth } from "@/contexts/auth-context"; import { useChat } from "@/contexts/chat-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; +import { LayoutProvider } from "@/contexts/layout-context"; // import { GitHubStarButton } from "@/components/github-star-button" // import { DiscordLink } from "@/components/discord-link" import { useTask } from "@/contexts/task-context"; -import { DoclingHealthBanner } from "@/components/docling-health-banner"; import { useDoclingHealthQuery } from "@/src/app/api/queries/useDoclingHealthQuery"; -import { LayoutProvider } from "@/contexts/layout-context"; export function LayoutWrapper({ children }: { children: React.ReactNode }) { const pathname = usePathname(); @@ -34,7 +37,11 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { const { isLoading: isSettingsLoading, data: settings } = useGetSettingsQuery({ enabled: isAuthenticated || isNoAuthMode, }); - const { data: health, isLoading: isHealthLoading, isError } = useDoclingHealthQuery(); + const { + data: health, + isLoading: isHealthLoading, + isError, + } = useDoclingHealthQuery(); // Only fetch conversations on chat page const isOnChatPage = pathname === "/" || pathname === "/chat"; @@ -66,7 +73,9 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { // Dynamic height calculations based on banner visibility const headerHeight = 53; const bannerHeight = 52; // Approximate banner height - const totalTopOffset = isBannerVisible ? headerHeight + bannerHeight : headerHeight; + const totalTopOffset = isBannerVisible + ? headerHeight + bannerHeight + : headerHeight; const mainContentHeight = `calc(100vh - ${totalTopOffset}px)`; // Show loading state when backend isn't ready @@ -81,7 +90,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { ); } - if (isAuthPage || (settings && !settings.edited)) { + if (isAuthPage) { // For auth pages, render without navigation return
{children}
; } @@ -157,7 +166,10 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { }`} style={{ height: mainContentHeight }} > - +
{children}
 From 7dfe90c8a6cfb9236dc66bdd908af1345ef21050 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 15:49:21 -0300 Subject: [PATCH 04/55] fixed ibm logo --- frontend/components/logo/ibm-logo.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/components/logo/ibm-logo.tsx b/frontend/components/logo/ibm-logo.tsx index 158ffa3b..e37adec1 100644 --- a/frontend/components/logo/ibm-logo.tsx +++ b/frontend/components/logo/ibm-logo.tsx @@ -9,7 +9,7 @@ export default function IBMLogo(props: React.SVGProps) { {...props} > IBM watsonx.ai Logo - + Date: Thu, 2 Oct 2025 15:05:55 -0400 Subject: [PATCH 05/55] Add OpenRAG OpenSearch flow and update .gitignore Added a new flow definition for OpenRAG with OpenSearch hybrid search in flows/openrag_url_mcp.json. Updated .gitignore to allow tracked JSON files in the flows directory while ignoring others. --- .gitignore | 1 + flows/openrag_url_mcp.json | 3155 ++++++++++++++++++++++++++++++++++++ 2 files changed, 3156 insertions(+) create mode 100644 flows/openrag_url_mcp.json diff --git a/.gitignore b/.gitignore index 8bf471e7..32a23ff3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ wheels/ 1001*.pdf *.json +!flows/*.json .DS_Store config.yaml diff --git a/flows/openrag_url_mcp.json b/flows/openrag_url_mcp.json new file mode 100644 index 00000000..9bf255ff --- /dev/null +++ b/flows/openrag_url_mcp.json @@ -0,0 +1,3155 @@ +{ + "data": { + "edges": [ + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "SplitText", + "id": "SplitText-QIKhg", + "name": "dataframe", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "ingest_data", + "id": "OpenSearchHybrid-Ve6bS", + "inputTypes": [ + "Data", + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__SplitText-QIKhg{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-OpenSearchHybrid-Ve6bS{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "SplitText-QIKhg", + "sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "OpenSearchHybrid-Ve6bS", + "targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "OpenAIEmbeddings", + "id": "OpenAIEmbeddings-joRJ6", + "name": "embeddings", + "output_types": [ + "Embeddings" + ] + }, + "targetHandle": { + "fieldName": "embedding", + "id": "OpenSearchHybrid-Ve6bS", + "inputTypes": [ + "Embeddings" + ], + "type": "other" + } + }, + "id": "xy-edge__OpenAIEmbeddings-joRJ6{œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-joRJ6œ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}-OpenSearchHybrid-Ve6bS{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}", + "selected": false, + "source": "OpenAIEmbeddings-joRJ6", + "sourceHandle": "{œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-joRJ6œ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}", + "target": "OpenSearchHybrid-Ve6bS", + "targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "ChatInput", + "id": "ChatInput-WLvBD", + "name": "message", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "urls", + "id": "URLComponent-lnA0q", + "inputTypes": [ + "Message" + ], + "type": "str" + } + }, + "id": "xy-edge__ChatInput-WLvBD{œdataTypeœ:œChatInputœ,œidœ:œChatInput-WLvBDœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}-URLComponent-lnA0q{œfieldNameœ:œurlsœ,œidœ:œURLComponent-lnA0qœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}", + "selected": false, + "source": "ChatInput-WLvBD", + "sourceHandle": "{œdataTypeœ:œChatInputœ,œidœ:œChatInput-WLvBDœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}", + "target": "URLComponent-lnA0q", + "targetHandle": "{œfieldNameœ:œurlsœ,œidœ:œURLComponent-lnA0qœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "URLComponent", + "id": "URLComponent-lnA0q", + "name": "page_results", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "df", + "id": "DataFrameOperations-hqIoy", + "inputTypes": [ + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__URLComponent-lnA0q{œdataTypeœ:œURLComponentœ,œidœ:œURLComponent-lnA0qœ,œnameœ:œpage_resultsœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-hqIoy{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "URLComponent-lnA0q", + "sourceHandle": "{œdataTypeœ:œURLComponentœ,œidœ:œURLComponent-lnA0qœ,œnameœ:œpage_resultsœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "DataFrameOperations-hqIoy", + "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "ChatInput", + "id": "ChatInput-WLvBD", + "name": "message", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "new_column_value", + "id": "DataFrameOperations-hqIoy", + "inputTypes": [ + "Message" + ], + "type": "str" + } + }, + "id": "xy-edge__ChatInput-WLvBD{œdataTypeœ:œChatInputœ,œidœ:œChatInput-WLvBDœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}-DataFrameOperations-hqIoy{œfieldNameœ:œnew_column_valueœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}", + "selected": false, + "source": "ChatInput-WLvBD", + "sourceHandle": "{œdataTypeœ:œChatInputœ,œidœ:œChatInput-WLvBDœ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}", + "target": "DataFrameOperations-hqIoy", + "targetHandle": "{œfieldNameœ:œnew_column_valueœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DataFrameOperations", + "id": "DataFrameOperations-hqIoy", + "name": "output", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "df", + "id": "DataFrameOperations-A98BL", + "inputTypes": [ + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__DataFrameOperations-hqIoy{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-hqIoyœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-A98BL{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-A98BLœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DataFrameOperations-hqIoy", + "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-hqIoyœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "DataFrameOperations-A98BL", + "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-A98BLœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DataFrameOperations", + "id": "DataFrameOperations-A98BL", + "name": "output", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "data_inputs", + "id": "SplitText-QIKhg", + "inputTypes": [ + "Data", + "DataFrame", + "Message" + ], + "type": "other" + } + }, + "id": "xy-edge__DataFrameOperations-A98BL{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-A98BLœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-SplitText-QIKhg{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DataFrameOperations-A98BL", + "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-A98BLœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "SplitText-QIKhg", + "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" + } + ], + "nodes": [ + { + "data": { + "description": "Split text into chunks based on specified criteria.", + "display_name": "Split Text", + "id": "SplitText-QIKhg", + "node": { + "base_classes": [ + "DataFrame" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Split text into chunks based on specified criteria.", + "display_name": "Split Text", + "documentation": "https://docs.langflow.org/components-processing#split-text", + "edited": true, + "field_order": [ + "data_inputs", + "chunk_overlap", + "chunk_size", + "separator", + "text_key", + "keep_separator" + ], + "frozen": false, + "icon": "scissors-line-dashed", + "legacy": false, + "lf_version": "1.6.0", + "metadata": { + "code_hash": "f2867efda61f", + "dependencies": { + "dependencies": [ + { + "name": "langchain_text_splitters", + "version": "0.3.9" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 2 + }, + "module": "custom_components.split_text" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Chunks", + "group_outputs": false, + "hidden": null, + "method": "split_text", + "name": "dataframe", + "options": null, + "required_inputs": null, + "selected": "DataFrame", + "tool_mode": true, + "types": [ + "DataFrame" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "chunk_overlap": { + "_input_type": "IntInput", + "advanced": false, + "display_name": "Chunk Overlap", + "dynamic": false, + "info": "Number of characters to overlap between chunks.", + "list": false, + "list_add_label": "Add More", + "name": "chunk_overlap", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 200 + }, + "chunk_size": { + "_input_type": "IntInput", + "advanced": false, + "display_name": "Chunk Size", + "dynamic": false, + "info": "The maximum length of each chunk. Text is first split by separator, then chunks are merged up to this size. Individual splits larger than this won't be further divided.", + "list": false, + "list_add_label": "Add More", + "name": "chunk_size", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 1000 + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from langchain_text_splitters import CharacterTextSplitter\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import DropdownInput, HandleInput, IntInput, MessageTextInput, Output\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.util import unescape_string\n\n\nclass SplitTextComponent(Component):\n display_name: str = \"Split Text\"\n description: str = \"Split text into chunks based on specified criteria.\"\n documentation: str = \"https://docs.langflow.org/components-processing#split-text\"\n icon = \"scissors-line-dashed\"\n name = \"SplitText\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"Input\",\n info=\"The data with texts to split in chunks.\",\n input_types=[\"Data\", \"DataFrame\", \"Message\"],\n required=True,\n ),\n IntInput(\n name=\"chunk_overlap\",\n display_name=\"Chunk Overlap\",\n info=\"Number of characters to overlap between chunks.\",\n value=200,\n ),\n IntInput(\n name=\"chunk_size\",\n display_name=\"Chunk Size\",\n info=(\n \"The maximum length of each chunk. Text is first split by separator, \"\n \"then chunks are merged up to this size. \"\n \"Individual splits larger than this won't be further divided.\"\n ),\n value=1000,\n ),\n MessageTextInput(\n name=\"separator\",\n display_name=\"Separator\",\n info=(\n \"The character to split on. Use \\\\n for newline. \"\n \"Examples: \\\\n\\\\n for paragraphs, \\\\n for lines, . for sentences\"\n ),\n value=\"\\n\",\n ),\n MessageTextInput(\n name=\"text_key\",\n display_name=\"Text Key\",\n info=\"The key to use for the text column.\",\n value=\"text\",\n advanced=True,\n ),\n DropdownInput(\n name=\"keep_separator\",\n display_name=\"Keep Separator\",\n info=\"Whether to keep the separator in the output chunks and where to place it.\",\n options=[\"False\", \"True\", \"Start\", \"End\"],\n value=\"False\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Chunks\", name=\"dataframe\", method=\"split_text\"),\n ]\n\n def _docs_to_data(self, docs) -> list[Data]:\n return [Data(text=doc.page_content, data=doc.metadata) for doc in docs]\n\n def _fix_separator(self, separator: str) -> str:\n \"\"\"Fix common separator issues and convert to proper format.\"\"\"\n if separator == \"/n\":\n return \"\\n\"\n if separator == \"/t\":\n return \"\\t\"\n return separator\n\n def split_text_base(self):\n separator = self._fix_separator(self.separator)\n separator = unescape_string(separator)\n\n if isinstance(self.data_inputs, DataFrame):\n if not len(self.data_inputs):\n msg = \"DataFrame is empty\"\n raise TypeError(msg)\n\n self.data_inputs.text_key = self.text_key\n try:\n documents = self.data_inputs.to_lc_documents()\n except Exception as e:\n msg = f\"Error converting DataFrame to documents: {e}\"\n raise TypeError(msg) from e\n elif isinstance(self.data_inputs, Message):\n self.data_inputs = [self.data_inputs.to_data()]\n return self.split_text_base()\n else:\n if not self.data_inputs:\n msg = \"No data inputs provided\"\n raise TypeError(msg)\n\n documents = []\n if isinstance(self.data_inputs, Data):\n self.data_inputs.text_key = self.text_key\n documents = [self.data_inputs.to_lc_document()]\n else:\n try:\n documents = [input_.to_lc_document() for input_ in self.data_inputs if isinstance(input_, Data)]\n if not documents:\n msg = f\"No valid Data inputs found in {type(self.data_inputs)}\"\n raise TypeError(msg)\n except AttributeError as e:\n msg = f\"Invalid input type in collection: {e}\"\n raise TypeError(msg) from e\n try:\n # Convert string 'False'/'True' to boolean\n keep_sep = self.keep_separator\n if isinstance(keep_sep, str):\n if keep_sep.lower() == \"false\":\n keep_sep = False\n elif keep_sep.lower() == \"true\":\n keep_sep = True\n # 'start' and 'end' are kept as strings\n\n splitter = CharacterTextSplitter(\n chunk_overlap=self.chunk_overlap,\n chunk_size=self.chunk_size,\n separator=separator,\n keep_separator=keep_sep,\n )\n return splitter.split_documents(documents)\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n def split_text(self) -> DataFrame:\n return DataFrame(self._docs_to_data(self.split_text_base()))\n" + }, + "data_inputs": { + "_input_type": "HandleInput", + "advanced": false, + "display_name": "Input", + "dynamic": false, + "info": "The data with texts to split in chunks.", + "input_types": [ + "Data", + "DataFrame", + "Message" + ], + "list": false, + "list_add_label": "Add More", + "name": "data_inputs", + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "other", + "value": "" + }, + "keep_separator": { + "_input_type": "DropdownInput", + "advanced": true, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Keep Separator", + "dynamic": false, + "external_options": {}, + "info": "Whether to keep the separator in the output chunks and where to place it.", + "name": "keep_separator", + "options": [ + "False", + "True", + "Start", + "End" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "False" + }, + "separator": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Separator", + "dynamic": false, + "info": "The character to split on. Use \\n for newline. Examples: \\n\\n for paragraphs, \\n for lines, . for sentences", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "separator", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "\n" + }, + "text_key": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Text Key", + "dynamic": false, + "info": "The key to use for the text column.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "text_key", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "text" + } + }, + "tool_mode": false + }, + "selected_output": "chunks", + "type": "SplitText" + }, + "dragging": false, + "height": 475, + "id": "SplitText-QIKhg", + "measured": { + "height": 475, + "width": 320 + }, + "position": { + "x": 2299.485091096586, + "y": 1430.4506304359015 + }, + "positionAbsolute": { + "x": 1683.4543896546102, + "y": 1350.7871623588553 + }, + "selected": false, + "type": "genericNode", + "width": 320 + }, + { + "data": { + "description": "Generate embeddings using OpenAI models.", + "display_name": "OpenAI Embeddings", + "id": "OpenAIEmbeddings-joRJ6", + "node": { + "base_classes": [ + "Embeddings" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Generate embeddings using OpenAI models.", + "display_name": "OpenAI Embeddings", + "documentation": "", + "edited": false, + "field_order": [ + "default_headers", + "default_query", + "chunk_size", + "client", + "deployment", + "embedding_ctx_length", + "max_retries", + "model", + "model_kwargs", + "openai_api_key", + "openai_api_base", + "openai_api_type", + "openai_api_version", + "openai_organization", + "openai_proxy", + "request_timeout", + "show_progress_bar", + "skip_empty", + "tiktoken_model_name", + "tiktoken_enable", + "dimensions" + ], + "frozen": false, + "icon": "OpenAI", + "legacy": false, + "metadata": { + "code_hash": "8a658ed6d4c9", + "dependencies": { + "dependencies": [ + { + "name": "langchain_openai", + "version": "0.3.23" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 2 + }, + "module": "custom_components.openai_embeddings" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Embedding Model", + "group_outputs": false, + "method": "build_embeddings", + "name": "embeddings", + "options": null, + "required_inputs": null, + "selected": "Embeddings", + "tool_mode": true, + "types": [ + "Embeddings" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "chunk_size": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Chunk Size", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "chunk_size", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 1000 + }, + "client": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Client", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "client", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from langchain_openai import OpenAIEmbeddings\n\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import BoolInput, DictInput, DropdownInput, FloatInput, IntInput, MessageTextInput, SecretStrInput\n\n\nclass OpenAIEmbeddingsComponent(LCEmbeddingsModel):\n display_name = \"OpenAI Embeddings\"\n description = \"Generate embeddings using OpenAI models.\"\n icon = \"OpenAI\"\n name = \"OpenAIEmbeddings\"\n\n inputs = [\n DictInput(\n name=\"default_headers\",\n display_name=\"Default Headers\",\n advanced=True,\n info=\"Default headers to use for the API request.\",\n ),\n DictInput(\n name=\"default_query\",\n display_name=\"Default Query\",\n advanced=True,\n info=\"Default query parameters to use for the API request.\",\n ),\n IntInput(name=\"chunk_size\", display_name=\"Chunk Size\", advanced=True, value=1000),\n MessageTextInput(name=\"client\", display_name=\"Client\", advanced=True),\n MessageTextInput(name=\"deployment\", display_name=\"Deployment\", advanced=True),\n IntInput(name=\"embedding_ctx_length\", display_name=\"Embedding Context Length\", advanced=True, value=1536),\n IntInput(name=\"max_retries\", display_name=\"Max Retries\", value=3, advanced=True),\n DropdownInput(\n name=\"model\",\n display_name=\"Model\",\n advanced=False,\n options=OPENAI_EMBEDDING_MODEL_NAMES,\n value=\"text-embedding-3-small\",\n ),\n DictInput(name=\"model_kwargs\", display_name=\"Model Kwargs\", advanced=True),\n SecretStrInput(name=\"openai_api_key\", display_name=\"OpenAI API Key\", value=\"OPENAI_API_KEY\", required=True),\n MessageTextInput(name=\"openai_api_base\", display_name=\"OpenAI API Base\", advanced=True),\n MessageTextInput(name=\"openai_api_type\", display_name=\"OpenAI API Type\", advanced=True),\n MessageTextInput(name=\"openai_api_version\", display_name=\"OpenAI API Version\", advanced=True),\n MessageTextInput(\n name=\"openai_organization\",\n display_name=\"OpenAI Organization\",\n advanced=True,\n ),\n MessageTextInput(name=\"openai_proxy\", display_name=\"OpenAI Proxy\", advanced=True),\n FloatInput(name=\"request_timeout\", display_name=\"Request Timeout\", advanced=True),\n BoolInput(name=\"show_progress_bar\", display_name=\"Show Progress Bar\", advanced=True),\n BoolInput(name=\"skip_empty\", display_name=\"Skip Empty\", advanced=True),\n MessageTextInput(\n name=\"tiktoken_model_name\",\n display_name=\"TikToken Model Name\",\n advanced=True,\n ),\n BoolInput(\n name=\"tiktoken_enable\",\n display_name=\"TikToken Enable\",\n advanced=True,\n value=True,\n info=\"If False, you must have transformers installed.\",\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n ]\n\n def build_embeddings(self) -> Embeddings:\n return OpenAIEmbeddings(\n client=self.client or None,\n model=self.model,\n dimensions=self.dimensions or None,\n deployment=self.deployment or None,\n api_version=self.openai_api_version or None,\n base_url=self.openai_api_base or None,\n openai_api_type=self.openai_api_type or None,\n openai_proxy=self.openai_proxy or None,\n embedding_ctx_length=self.embedding_ctx_length,\n api_key=self.openai_api_key or None,\n organization=self.openai_organization or None,\n allowed_special=\"all\",\n disallowed_special=\"all\",\n chunk_size=self.chunk_size,\n max_retries=self.max_retries,\n timeout=self.request_timeout or None,\n tiktoken_enabled=self.tiktoken_enable,\n tiktoken_model_name=self.tiktoken_model_name or None,\n show_progress_bar=self.show_progress_bar,\n model_kwargs=self.model_kwargs,\n skip_empty=self.skip_empty,\n default_headers=self.default_headers or None,\n default_query=self.default_query or None,\n )\n" + }, + "default_headers": { + "_input_type": "DictInput", + "advanced": true, + "display_name": "Default Headers", + "dynamic": false, + "info": "Default headers to use for the API request.", + "list": false, + "list_add_label": "Add More", + "name": "default_headers", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "type": "dict", + "value": {} + }, + "default_query": { + "_input_type": "DictInput", + "advanced": true, + "display_name": "Default Query", + "dynamic": false, + "info": "Default query parameters to use for the API request.", + "list": false, + "list_add_label": "Add More", + "name": "default_query", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "type": "dict", + "value": {} + }, + "deployment": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Deployment", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "deployment", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "dimensions": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Dimensions", + "dynamic": false, + "info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.", + "list": false, + "list_add_label": "Add More", + "name": "dimensions", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": "" + }, + "embedding_ctx_length": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Embedding Context Length", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "embedding_ctx_length", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 1536 + }, + "max_retries": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Max Retries", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "max_retries", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 3 + }, + "model": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Model", + "dynamic": false, + "external_options": {}, + "info": "", + "name": "model", + "options": [ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "text-embedding-3-small" + }, + "model_kwargs": { + "_input_type": "DictInput", + "advanced": true, + "display_name": "Model Kwargs", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "model_kwargs", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "type": "dict", + "value": {} + }, + "openai_api_base": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI API Base", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "openai_api_base", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "openai_api_key": { + "_input_type": "SecretStrInput", + "advanced": false, + "display_name": "OpenAI API Key", + "dynamic": false, + "info": "", + "input_types": [], + "load_from_db": false, + "name": "openai_api_key", + "password": true, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "str", + "value": "" + }, + "openai_api_type": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI API Type", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "openai_api_type", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "openai_api_version": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI API Version", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "openai_api_version", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "openai_organization": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI Organization", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "openai_organization", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "openai_proxy": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "OpenAI Proxy", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "openai_proxy", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "request_timeout": { + "_input_type": "FloatInput", + "advanced": true, + "display_name": "Request Timeout", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "request_timeout", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "float", + "value": "" + }, + "show_progress_bar": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Show Progress Bar", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "show_progress_bar", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": false + }, + "skip_empty": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Skip Empty", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "skip_empty", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": false + }, + "tiktoken_enable": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "TikToken Enable", + "dynamic": false, + "info": "If False, you must have transformers installed.", + "list": false, + "list_add_label": "Add More", + "name": "tiktoken_enable", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "tiktoken_model_name": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "TikToken Model Name", + "dynamic": false, + "info": "", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "tiktoken_model_name", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + } + }, + "tool_mode": false + }, + "selected_output": "embeddings", + "type": "OpenAIEmbeddings" + }, + "dragging": false, + "height": 320, + "id": "OpenAIEmbeddings-joRJ6", + "measured": { + "height": 320, + "width": 320 + }, + "position": { + "x": 1870.4219509914485, + "y": 2193.4259215896764 + }, + "positionAbsolute": { + "x": 1690.9220896443658, + "y": 1866.483269483266 + }, + "selected": false, + "type": "genericNode", + "width": 320 + }, + { + "data": { + "id": "note-Bm5Xw", + "node": { + "description": "### 💡 Add your OpenAI API key here 👇", + "display_name": "", + "documentation": "", + "template": { + "backgroundColor": "transparent" + } + }, + "type": "note" + }, + "dragging": false, + "height": 324, + "id": "note-Bm5Xw", + "measured": { + "height": 324, + "width": 324 + }, + "position": { + "x": 1868.5365759938197, + "y": 2117.7924922977318 + }, + "positionAbsolute": { + "x": 1692.2322233423606, + "y": 1821.9077961087607 + }, + "selected": false, + "type": "noteNode", + "width": 324 + }, + { + "data": { + "id": "OpenSearchHybrid-Ve6bS", + "node": { + "base_classes": [ + "Data", + "DataFrame", + "VectorStore" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Store and search documents using OpenSearch with hybrid semantic and keyword search capabilities.", + "display_name": "OpenSearch", + "documentation": "", + "edited": true, + "field_order": [ + "docs_metadata", + "opensearch_url", + "index_name", + "engine", + "space_type", + "ef_construction", + "m", + "ingest_data", + "search_query", + "should_cache_vector_store", + "embedding", + "vector_field", + "number_of_results", + "filter_expression", + "auth_mode", + "username", + "password", + "jwt_token", + "jwt_header", + "bearer_prefix", + "use_ssl", + "verify_certs" + ], + "frozen": false, + "icon": "OpenSearch", + "legacy": false, + "metadata": { + "code_hash": "08d808984c3d", + "dependencies": { + "dependencies": [ + { + "name": "opensearchpy", + "version": "2.8.0" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 2 + }, + "module": "custom_components.opensearch" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Search Results", + "group_outputs": false, + "hidden": null, + "method": "search_documents", + "name": "search_results", + "options": null, + "required_inputs": null, + "selected": "Data", + "tool_mode": true, + "types": [ + "Data" + ], + "value": "__UNDEFINED__" + }, + { + "allows_loop": false, + "cache": true, + "display_name": "DataFrame", + "group_outputs": false, + "hidden": null, + "method": "as_dataframe", + "name": "dataframe", + "options": null, + "required_inputs": null, + "selected": "DataFrame", + "tool_mode": true, + "types": [ + "DataFrame" + ], + "value": "__UNDEFINED__" + }, + { + "allows_loop": false, + "cache": true, + "display_name": "Vector Store Connection", + "group_outputs": false, + "hidden": false, + "method": "as_vector_store", + "name": "vectorstoreconnection", + "options": null, + "required_inputs": null, + "selected": "VectorStore", + "tool_mode": true, + "types": [ + "VectorStore" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "auth_mode": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Authentication Mode", + "dynamic": false, + "external_options": {}, + "info": "Authentication method: 'basic' for username/password authentication, or 'jwt' for JSON Web Token (Bearer) authentication.", + "load_from_db": false, + "name": "auth_mode", + "options": [ + "basic", + "jwt" + ], + "options_metadata": [], + "placeholder": "", + "real_time_refresh": true, + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "jwt" + }, + "bearer_prefix": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Prefix 'Bearer '", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "bearer_prefix", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from __future__ import annotations\n\nimport json\nimport uuid\nfrom typing import Any\n\nfrom opensearchpy import OpenSearch, helpers\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, MultilineInput, SecretStrInput, StrInput, TableInput\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponent(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports document\n ingestion, vector embeddings, and advanced filtering with authentication options.\n\n Features:\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Hybrid search combining KNN vector similarity and keyword matching\n - Flexible authentication (Basic auth, JWT tokens)\n - Advanced filtering and aggregations\n - Metadata injection during document ingestion\n \"\"\"\n\n display_name: str = \"OpenSearch\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with hybrid semantic and keyword search capabilities.\"\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"docs_metadata\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n \"load_from_db\": True\n },\n ],\n value=[],\n # advanced=True,\n input_types=[\"Data\"]\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"jvector\", \"nmslib\", \"faiss\", \"lucene\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'jvector' is recommended for most use cases. \"\n \"Note: Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"]),\n StrInput(\n name=\"vector_field\",\n display_name=\"Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=\"Name of the field in OpenSearch documents that stores the vector embeddings for similarity search.\",\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"basic\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=False,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=False,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=True,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=True,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n ]\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n }\n }\n },\n }\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our “vector store.”\n self.log(self.ingest_data)\n client = self.build_client()\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings\n - Creates appropriate index mappings\n - Bulk inserts documents with vectors\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n docs = self.ingest_data or []\n if not docs:\n self.log(\"No documents to ingest.\")\n return\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Generate embeddings\n vectors = self.embedding.embed_documents(texts)\n\n if not vectors:\n self.log(\"No vectors generated from documents.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=self.vector_field,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with proper KNN mapping...\")\n\n # Use the LangChain-style bulk ingestion\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=self.vector_field,\n text_field=\"text\",\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n self.log(f\"Successfully indexed {len(return_ids)} documents.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n # ---------- search (single hybrid path matching your tool) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform hybrid search combining vector similarity and keyword matching.\n\n This method executes a sophisticated search that combines:\n - K-nearest neighbor (KNN) vector similarity search (70% weight)\n - Multi-field keyword search with fuzzy matching (30% weight)\n - Optional filtering and score thresholds\n - Aggregations for faceted search results\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression (can be either A or B shape; see _coerce_filter_clauses)\n filter_obj = None\n if getattr(self, \"filter_expression\", \"\") and self.filter_expression.strip():\n try:\n filter_obj = json.loads(self.filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Embed the query\n vec = self.embedding.embed_query(q)\n\n # Build filter clauses (accept both shapes)\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Respect the tool's limit/threshold defaults\n limit = (filter_obj or {}).get(\"limit\", self.number_of_results)\n score_threshold = (filter_obj or {}).get(\"score_threshold\", 0)\n\n # Build the same hybrid body as your SearchService\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"knn\": {\n self.vector_field: {\n \"vector\": vec,\n \"k\": 10, # fixed to match the tool\n \"boost\": 0.7,\n }\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3,\n }\n },\n ],\n \"minimum_should_match\": 1,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": \"filename\", \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n if filter_clauses:\n body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n\n if isinstance(score_threshold, (int, float)) and score_threshold > 0:\n # top-level min_score (matches your tool)\n body[\"min_score\"] = score_threshold\n\n resp = client.search(index=self.index_name, body=body)\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as Data objects.\n\n This is the main interface method that performs the search using the\n configured search_query and returns results in Langflow's Data format.\n\n Returns:\n List of Data objects containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n raw = self.search(self.search_query or \"\")\n return [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n self.log(self.ingest_data)\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n" + }, + "docs_metadata": { + "_input_type": "TableInput", + "advanced": false, + "display_name": "Document Metadata", + "dynamic": false, + "info": "Additional metadata key-value pairs to be added to all ingested documents. Useful for tagging documents with source information, categories, or other custom attributes.", + "input_types": [ + "Data" + ], + "is_list": true, + "list_add_label": "Add More", + "name": "docs_metadata", + "placeholder": "", + "required": false, + "show": true, + "table_icon": "Table", + "table_schema": [ + { + "description": "Key name", + "display_name": "Key", + "formatter": "text", + "name": "key", + "type": "str" + }, + { + "description": "Value of the metadata", + "display_name": "Value", + "formatter": "text", + "load_from_db": true, + "name": "value", + "type": "str" + } + ], + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "trigger_icon": "Table", + "trigger_text": "Open table", + "type": "table", + "value": [ + { + "key": "owner_name", + "value": "OWNER_NAME" + }, + { + "key": "owner", + "value": "OWNER" + } + ] + }, + "ef_construction": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "EF Construction", + "dynamic": false, + "info": "Size of the dynamic candidate list during index construction. Higher values improve recall but increase indexing time and memory usage.", + "list": false, + "list_add_label": "Add More", + "name": "ef_construction", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 512 + }, + "embedding": { + "_input_type": "HandleInput", + "advanced": false, + "display_name": "Embedding", + "dynamic": false, + "info": "", + "input_types": [ + "Embeddings" + ], + "list": false, + "list_add_label": "Add More", + "name": "embedding", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "other", + "value": "" + }, + "engine": { + "_input_type": "DropdownInput", + "advanced": true, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Vector Engine", + "dynamic": false, + "external_options": {}, + "info": "Vector search engine for similarity calculations. 'jvector' is recommended for most use cases. Note: Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.", + "load_from_db": false, + "name": "engine", + "options": [ + "jvector", + "nmslib", + "faiss", + "lucene" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "nmslib" + }, + "filter_expression": { + "_input_type": "MultilineInput", + "advanced": false, + "copy_field": false, + "display_name": "Search Filters (JSON)", + "dynamic": false, + "info": "Optional JSON configuration for search filtering, result limits, and score thresholds.\n\nFormat 1 - Explicit filters:\n{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, {\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\n\nFormat 2 - Context-style mapping:\n{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\n\nUse __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "multiline": true, + "name": "filter_expression", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "index_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Index Name", + "dynamic": false, + "info": "The OpenSearch index name where documents will be stored and searched. Will be created automatically if it doesn't exist.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "index_name", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "documents" + }, + "ingest_data": { + "_input_type": "HandleInput", + "advanced": false, + "display_name": "Ingest Data", + "dynamic": false, + "info": "", + "input_types": [ + "Data", + "DataFrame" + ], + "list": true, + "list_add_label": "Add More", + "name": "ingest_data", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "other", + "value": "" + }, + "jwt_header": { + "_input_type": "StrInput", + "advanced": true, + "display_name": "JWT Header Name", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "jwt_header", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "Authorization" + }, + "jwt_token": { + "_input_type": "SecretStrInput", + "advanced": false, + "display_name": "JWT Token", + "dynamic": false, + "info": "Valid JSON Web Token for authentication. Will be sent in the Authorization header (with optional 'Bearer ' prefix).", + "input_types": [], + "load_from_db": true, + "name": "jwt_token", + "password": true, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "type": "str", + "value": "JWT" + }, + "m": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "M Parameter", + "dynamic": false, + "info": "Number of bidirectional connections for each vector in the HNSW graph. Higher values improve search quality but increase memory usage and indexing time.", + "list": false, + "list_add_label": "Add More", + "name": "m", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 16 + }, + "number_of_results": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Default Result Limit", + "dynamic": false, + "info": "Default maximum number of search results to return when no limit is specified in the filter expression.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "number_of_results", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 15 + }, + "opensearch_url": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "OpenSearch URL", + "dynamic": false, + "info": "The connection URL for your OpenSearch cluster (e.g., http://localhost:9200 for local development or your cloud endpoint).", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "opensearch_url", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "https://opensearch:9200" + }, + "password": { + "_input_type": "SecretStrInput", + "advanced": false, + "display_name": "OpenSearch Password", + "dynamic": false, + "info": "", + "input_types": [], + "load_from_db": false, + "name": "password", + "password": true, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "type": "str", + "value": "" + }, + "search_query": { + "_input_type": "QueryInput", + "advanced": false, + "display_name": "Search Query", + "dynamic": false, + "info": "Enter a query to run a similarity search.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "search_query", + "placeholder": "Enter a query...", + "required": false, + "show": true, + "title_case": false, + "tool_mode": true, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "query", + "value": "" + }, + "should_cache_vector_store": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Cache Vector Store", + "dynamic": false, + "info": "If True, the vector store will be cached for the current build of the component. This is useful for components that have multiple output methods and want to share the same vector store.", + "list": false, + "list_add_label": "Add More", + "name": "should_cache_vector_store", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "space_type": { + "_input_type": "DropdownInput", + "advanced": true, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Distance Metric", + "dynamic": false, + "external_options": {}, + "info": "Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, 'cosinesimil' for cosine similarity, 'innerproduct' for dot product.", + "name": "space_type", + "options": [ + "l2", + "l1", + "cosinesimil", + "linf", + "innerproduct" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "l2" + }, + "use_ssl": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Use SSL/TLS", + "dynamic": false, + "info": "Enable SSL/TLS encryption for secure connections to OpenSearch.", + "list": false, + "list_add_label": "Add More", + "name": "use_ssl", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "username": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Username", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "username", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "admin" + }, + "vector_field": { + "_input_type": "StrInput", + "advanced": true, + "display_name": "Vector Field Name", + "dynamic": false, + "info": "Name of the field in OpenSearch documents that stores the vector embeddings for similarity search.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "vector_field", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "chunk_embedding" + }, + "verify_certs": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Verify SSL Certificates", + "dynamic": false, + "info": "Verify SSL certificates when connecting. Disable for self-signed certificates in development environments.", + "list": false, + "list_add_label": "Add More", + "name": "verify_certs", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": false + } + }, + "tool_mode": false + }, + "selected_output": "search_results", + "showNode": true, + "type": "OpenSearchVectorStoreComponent" + }, + "dragging": false, + "id": "OpenSearchHybrid-Ve6bS", + "measured": { + "height": 822, + "width": 320 + }, + "position": { + "x": 2694.183983837566, + "y": 1425.777807367294 + }, + "selected": true, + "type": "genericNode" + }, + { + "data": { + "id": "URLComponent-lnA0q", + "node": { + "base_classes": [ + "DataFrame", + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Fetch content from one or more web pages, following links recursively.", + "display_name": "URL", + "documentation": "https://docs.langflow.org/components-data#url", + "edited": true, + "field_order": [ + "urls", + "max_depth", + "prevent_outside", + "use_async", + "format", + "timeout", + "headers", + "filter_text_html", + "continue_on_failure", + "check_response_status", + "autoset_encoding" + ], + "frozen": false, + "icon": "layout-template", + "legacy": false, + "lf_version": "1.6.0", + "metadata": { + "code_hash": "4c72ce0f2e34", + "dependencies": { + "dependencies": [ + { + "name": "requests", + "version": "2.32.5" + }, + { + "name": "bs4", + "version": "4.12.3" + }, + { + "name": "langchain_community", + "version": "0.3.21" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 4 + }, + "module": "custom_components.url" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Extracted Pages", + "group_outputs": false, + "hidden": null, + "method": "fetch_content", + "name": "page_results", + "options": null, + "required_inputs": null, + "selected": "DataFrame", + "tool_mode": true, + "types": [ + "DataFrame" + ], + "value": "__UNDEFINED__" + }, + { + "allows_loop": false, + "cache": true, + "display_name": "Raw Content", + "group_outputs": false, + "hidden": null, + "method": "fetch_content_as_message", + "name": "raw_results", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": false, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "autoset_encoding": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Autoset Encoding", + "dynamic": false, + "info": "If enabled, automatically sets the encoding of the request.", + "list": false, + "list_add_label": "Add More", + "name": "autoset_encoding", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "check_response_status": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Check Response Status", + "dynamic": false, + "info": "If enabled, checks the response status of the request.", + "list": false, + "list_add_label": "Add More", + "name": "check_response_status", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": false + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "import importlib\nimport re\n\nimport requests\nfrom bs4 import BeautifulSoup\nfrom langchain_community.document_loaders import RecursiveUrlLoader\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.field_typing.range_spec import RangeSpec\nfrom lfx.helpers.data import safe_convert\nfrom lfx.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SliderInput, TableInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.request_utils import get_user_agent\n\n# Constants\nDEFAULT_TIMEOUT = 30\nDEFAULT_MAX_DEPTH = 1\nDEFAULT_FORMAT = \"Text\"\n\n\nURL_REGEX = re.compile(\n r\"^(https?:\\/\\/)?\" r\"(www\\.)?\" r\"([a-zA-Z0-9.-]+)\" r\"(\\.[a-zA-Z]{2,})?\" r\"(:\\d+)?\" r\"(\\/[^\\s]*)?$\",\n re.IGNORECASE,\n)\n\nUSER_AGENT = None\n# Check if langflow is installed using importlib.util.find_spec(name))\nif importlib.util.find_spec(\"langflow\"):\n langflow_installed = True\n USER_AGENT = get_user_agent()\nelse:\n langflow_installed = False\n USER_AGENT = \"lfx\"\n\n\nclass URLComponent(Component):\n \"\"\"A component that loads and parses content from web pages recursively.\n\n This component allows fetching content from one or more URLs, with options to:\n - Control crawl depth\n - Prevent crawling outside the root domain\n - Use async loading for better performance\n - Extract either raw HTML or clean text\n - Configure request headers and timeouts\n \"\"\"\n\n display_name = \"URL\"\n description = \"Fetch content from one or more web pages, following links recursively.\"\n documentation: str = \"https://docs.langflow.org/components-data#url\"\n icon = \"layout-template\"\n name = \"URLComponent\"\n\n inputs = [\n MessageTextInput(\n name=\"urls\",\n display_name=\"URLs\",\n info=\"Enter one or more URLs to crawl recursively, by clicking the '+' button.\",\n is_list=True,\n tool_mode=True,\n placeholder=\"Enter a URL...\",\n list_add_label=\"Add URL\",\n input_types=[\"Message\"],\n ),\n SliderInput(\n name=\"max_depth\",\n display_name=\"Depth\",\n info=(\n \"Controls how many 'clicks' away from the initial page the crawler will go:\\n\"\n \"- depth 1: only the initial page\\n\"\n \"- depth 2: initial page + all pages linked directly from it\\n\"\n \"- depth 3: initial page + direct links + links found on those direct link pages\\n\"\n \"Note: This is about link traversal, not URL path depth.\"\n ),\n value=DEFAULT_MAX_DEPTH,\n range_spec=RangeSpec(min=1, max=5, step=1),\n required=False,\n min_label=\" \",\n max_label=\" \",\n min_label_icon=\"None\",\n max_label_icon=\"None\",\n # slider_input=True\n ),\n BoolInput(\n name=\"prevent_outside\",\n display_name=\"Prevent Outside\",\n info=(\n \"If enabled, only crawls URLs within the same domain as the root URL. \"\n \"This helps prevent the crawler from going to external websites.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"use_async\",\n display_name=\"Use Async\",\n info=(\n \"If enabled, uses asynchronous loading which can be significantly faster \"\n \"but might use more system resources.\"\n ),\n value=True,\n required=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"format\",\n display_name=\"Output Format\",\n info=\"Output Format. Use 'Text' to extract the text from the HTML or 'HTML' for the raw HTML content.\",\n options=[\"Text\", \"HTML\"],\n value=DEFAULT_FORMAT,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Timeout for the request in seconds.\",\n value=DEFAULT_TIMEOUT,\n required=False,\n advanced=True,\n ),\n TableInput(\n name=\"headers\",\n display_name=\"Headers\",\n info=\"The headers to send with the request\",\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Header\",\n \"type\": \"str\",\n \"description\": \"Header name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Header value\",\n },\n ],\n value=[{\"key\": \"User-Agent\", \"value\": USER_AGENT}],\n advanced=True,\n input_types=[\"DataFrame\"],\n ),\n BoolInput(\n name=\"filter_text_html\",\n display_name=\"Filter Text/HTML\",\n info=\"If enabled, filters out text/css content type from the results.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"continue_on_failure\",\n display_name=\"Continue on Failure\",\n info=\"If enabled, continues crawling even if some requests fail.\",\n value=True,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"check_response_status\",\n display_name=\"Check Response Status\",\n info=\"If enabled, checks the response status of the request.\",\n value=False,\n required=False,\n advanced=True,\n ),\n BoolInput(\n name=\"autoset_encoding\",\n display_name=\"Autoset Encoding\",\n info=\"If enabled, automatically sets the encoding of the request.\",\n value=True,\n required=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Extracted Pages\", name=\"page_results\", method=\"fetch_content\"),\n Output(display_name=\"Raw Content\", name=\"raw_results\", method=\"fetch_content_as_message\", tool_mode=False),\n ]\n\n @staticmethod\n def validate_url(url: str) -> bool:\n \"\"\"Validates if the given string matches URL pattern.\n\n Args:\n url: The URL string to validate\n\n Returns:\n bool: True if the URL is valid, False otherwise\n \"\"\"\n return bool(URL_REGEX.match(url))\n\n def ensure_url(self, url: str) -> str:\n \"\"\"Ensures the given string is a valid URL.\n\n Args:\n url: The URL string to validate and normalize\n\n Returns:\n str: The normalized URL\n\n Raises:\n ValueError: If the URL is invalid\n \"\"\"\n url = url.strip()\n if not url.startswith((\"http://\", \"https://\")):\n url = \"https://\" + url\n\n if not self.validate_url(url):\n msg = f\"Invalid URL: {url}\"\n raise ValueError(msg)\n\n return url\n\n def _create_loader(self, url: str) -> RecursiveUrlLoader:\n \"\"\"Creates a RecursiveUrlLoader instance with the configured settings.\n\n Args:\n url: The URL to load\n\n Returns:\n RecursiveUrlLoader: Configured loader instance\n \"\"\"\n headers_dict = {header[\"key\"]: header[\"value\"] for header in self.headers if header[\"value\"] is not None}\n extractor = (lambda x: x) if self.format == \"HTML\" else (lambda x: BeautifulSoup(x, \"lxml\").get_text())\n\n return RecursiveUrlLoader(\n url=url,\n max_depth=self.max_depth,\n prevent_outside=self.prevent_outside,\n use_async=self.use_async,\n extractor=extractor,\n timeout=self.timeout,\n headers=headers_dict,\n check_response_status=self.check_response_status,\n continue_on_failure=self.continue_on_failure,\n base_url=url, # Add base_url to ensure consistent domain crawling\n autoset_encoding=self.autoset_encoding, # Enable automatic encoding detection\n exclude_dirs=[], # Allow customization of excluded directories\n link_regex=None, # Allow customization of link filtering\n )\n\n def fetch_url_contents(self) -> list[dict]:\n \"\"\"Load documents from the configured URLs.\n\n Returns:\n List[Data]: List of Data objects containing the fetched content\n\n Raises:\n ValueError: If no valid URLs are provided or if there's an error loading documents\n \"\"\"\n try:\n urls = list({self.ensure_url(url) for url in self.urls if url.strip()})\n logger.debug(f\"URLs: {urls}\")\n if not urls:\n msg = \"No valid URLs provided.\"\n raise ValueError(msg)\n\n all_docs = []\n for url in urls:\n logger.debug(f\"Loading documents from {url}\")\n\n try:\n loader = self._create_loader(url)\n docs = loader.load()\n\n if not docs:\n logger.warning(f\"No documents found for {url}\")\n continue\n\n logger.debug(f\"Found {len(docs)} documents from {url}\")\n all_docs.extend(docs)\n\n except requests.exceptions.RequestException as e:\n logger.exception(f\"Error loading documents from {url}: {e}\")\n continue\n\n if not all_docs:\n msg = \"No documents were successfully loaded from any URL\"\n raise ValueError(msg)\n\n # data = [Data(text=doc.page_content, **doc.metadata) for doc in all_docs]\n data = [\n {\n \"text\": safe_convert(doc.page_content, clean_data=True),\n \"url\": doc.metadata.get(\"source\", \"\"),\n \"title\": doc.metadata.get(\"title\", \"\"),\n \"description\": doc.metadata.get(\"description\", \"\"),\n \"content_type\": doc.metadata.get(\"content_type\", \"\"),\n \"language\": doc.metadata.get(\"language\", \"\"),\n }\n for doc in all_docs\n ]\n except Exception as e:\n error_msg = e.message if hasattr(e, \"message\") else e\n msg = f\"Error loading documents: {error_msg!s}\"\n logger.exception(msg)\n raise ValueError(msg) from e\n return data\n\n def fetch_content(self) -> DataFrame:\n \"\"\"Convert the documents to a DataFrame.\"\"\"\n return DataFrame(data=self.fetch_url_contents())\n\n def fetch_content_as_message(self) -> Message:\n \"\"\"Convert the documents to a Message.\"\"\"\n url_contents = self.fetch_url_contents()\n return Message(text=\"\\n\\n\".join([x[\"text\"] for x in url_contents]), data={\"data\": url_contents})\n" + }, + "continue_on_failure": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Continue on Failure", + "dynamic": false, + "info": "If enabled, continues crawling even if some requests fail.", + "list": false, + "list_add_label": "Add More", + "name": "continue_on_failure", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "filter_text_html": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Filter Text/HTML", + "dynamic": false, + "info": "If enabled, filters out text/css content type from the results.", + "list": false, + "list_add_label": "Add More", + "name": "filter_text_html", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "format": { + "_input_type": "DropdownInput", + "advanced": true, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Output Format", + "dynamic": false, + "external_options": {}, + "info": "Output Format. Use 'Text' to extract the text from the HTML or 'HTML' for the raw HTML content.", + "name": "format", + "options": [ + "Text", + "HTML" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "Text" + }, + "headers": { + "_input_type": "TableInput", + "advanced": true, + "display_name": "Headers", + "dynamic": false, + "info": "The headers to send with the request", + "input_types": [ + "DataFrame" + ], + "is_list": true, + "list_add_label": "Add More", + "load_from_db": false, + "name": "headers", + "placeholder": "", + "required": false, + "show": true, + "table_icon": "Table", + "table_schema": [ + { + "description": "Header name", + "display_name": "Header", + "name": "key", + "type": "str" + }, + { + "description": "Header value", + "display_name": "Value", + "name": "value", + "type": "str" + } + ], + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "trigger_icon": "Table", + "trigger_text": "Open table", + "type": "table", + "value": [ + { + "key": "User-Agent", + "value": "langflow" + } + ] + }, + "max_depth": { + "_input_type": "SliderInput", + "advanced": false, + "display_name": "Depth", + "dynamic": false, + "info": "Controls how many 'clicks' away from the initial page the crawler will go:\n- depth 1: only the initial page\n- depth 2: initial page + all pages linked directly from it\n- depth 3: initial page + direct links + links found on those direct link pages\nNote: This is about link traversal, not URL path depth.", + "max_label": " ", + "max_label_icon": "None", + "min_label": " ", + "min_label_icon": "None", + "name": "max_depth", + "placeholder": "", + "range_spec": { + "max": 5, + "min": 1, + "step": 1, + "step_type": "float" + }, + "required": false, + "show": true, + "slider_buttons": false, + "slider_buttons_options": [], + "slider_input": false, + "title_case": false, + "tool_mode": false, + "type": "slider", + "value": 1 + }, + "prevent_outside": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Prevent Outside", + "dynamic": false, + "info": "If enabled, only crawls URLs within the same domain as the root URL. This helps prevent the crawler from going to external websites.", + "list": false, + "list_add_label": "Add More", + "name": "prevent_outside", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "timeout": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Timeout", + "dynamic": false, + "info": "Timeout for the request in seconds.", + "list": false, + "list_add_label": "Add More", + "name": "timeout", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 30 + }, + "urls": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "URLs", + "dynamic": false, + "info": "Enter one or more URLs to crawl recursively, by clicking the '+' button.", + "input_types": [ + "Message" + ], + "list": true, + "list_add_label": "Add URL", + "load_from_db": false, + "name": "urls", + "placeholder": "Enter a URL...", + "required": false, + "show": true, + "title_case": false, + "tool_mode": true, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "use_async": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Use Async", + "dynamic": false, + "info": "If enabled, uses asynchronous loading which can be significantly faster but might use more system resources.", + "list": false, + "list_add_label": "Add More", + "name": "use_async", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + } + }, + "tool_mode": false + }, + "selected_output": "page_results", + "showNode": true, + "type": "URLComponent" + }, + "dragging": false, + "id": "URLComponent-lnA0q", + "measured": { + "height": 292, + "width": 320 + }, + "position": { + "x": 1249.8241608743583, + "y": 1270.7229143090308 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "description": "Get chat inputs from the Playground.", + "display_name": "Chat Input", + "id": "ChatInput-WLvBD", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Get chat inputs from the Playground.", + "display_name": "Chat Input", + "documentation": "https://docs.langflow.org/components-io#chat-input", + "edited": false, + "field_order": [ + "input_value", + "should_store_message", + "sender", + "sender_name", + "session_id", + "files" + ], + "frozen": false, + "icon": "MessagesSquare", + "legacy": false, + "lf_version": "1.6.0", + "metadata": { + "code_hash": "f701f686b325", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 1 + }, + "module": "custom_components.chat_input" + }, + "minimized": true, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Chat Message", + "group_outputs": false, + "method": "message_response", + "name": "message", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from lfx.base.data.utils import IMG_FILE_TYPES, TEXT_FILE_TYPES\nfrom lfx.base.io.chat import ChatComponent\nfrom lfx.inputs.inputs import BoolInput\nfrom lfx.io import (\n DropdownInput,\n FileInput,\n MessageTextInput,\n MultilineInput,\n Output,\n)\nfrom lfx.schema.message import Message\nfrom lfx.utils.constants import (\n MESSAGE_SENDER_AI,\n MESSAGE_SENDER_NAME_USER,\n MESSAGE_SENDER_USER,\n)\n\n\nclass ChatInput(ChatComponent):\n display_name = \"Chat Input\"\n description = \"Get chat inputs from the Playground.\"\n documentation: str = \"https://docs.langflow.org/components-io#chat-input\"\n icon = \"MessagesSquare\"\n name = \"ChatInput\"\n minimized = True\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Input Text\",\n value=\"\",\n info=\"Message to be passed as input.\",\n input_types=[],\n ),\n BoolInput(\n name=\"should_store_message\",\n display_name=\"Store Messages\",\n info=\"Store the message in the history.\",\n value=True,\n advanced=True,\n ),\n DropdownInput(\n name=\"sender\",\n display_name=\"Sender Type\",\n options=[MESSAGE_SENDER_AI, MESSAGE_SENDER_USER],\n value=MESSAGE_SENDER_USER,\n info=\"Type of sender.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"sender_name\",\n display_name=\"Sender Name\",\n info=\"Name of the sender.\",\n value=MESSAGE_SENDER_NAME_USER,\n advanced=True,\n ),\n MessageTextInput(\n name=\"session_id\",\n display_name=\"Session ID\",\n info=\"The session ID of the chat. If empty, the current session ID parameter will be used.\",\n advanced=True,\n ),\n FileInput(\n name=\"files\",\n display_name=\"Files\",\n file_types=TEXT_FILE_TYPES + IMG_FILE_TYPES,\n info=\"Files to be sent with the message.\",\n advanced=True,\n is_list=True,\n temp_file=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Chat Message\", name=\"message\", method=\"message_response\"),\n ]\n\n async def message_response(self) -> Message:\n # Ensure files is a list and filter out empty/None values\n files = self.files if self.files else []\n if files and not isinstance(files, list):\n files = [files]\n # Filter out None/empty values\n files = [f for f in files if f is not None and f != \"\"]\n\n message = await Message.create(\n text=self.input_value,\n sender=self.sender,\n sender_name=self.sender_name,\n session_id=self.session_id,\n files=files,\n )\n if self.session_id and isinstance(message, Message) and self.should_store_message:\n stored_message = await self.send_message(\n message,\n )\n self.message.value = stored_message\n message = stored_message\n\n self.status = message\n return message\n" + }, + "files": { + "_input_type": "FileInput", + "advanced": true, + "display_name": "Files", + "dynamic": false, + "fileTypes": [ + "csv", + "json", + "pdf", + "txt", + "md", + "mdx", + "yaml", + "yml", + "xml", + "html", + "htm", + "docx", + "py", + "sh", + "sql", + "js", + "ts", + "tsx", + "jpg", + "jpeg", + "png", + "bmp", + "image" + ], + "file_path": "", + "info": "Files to be sent with the message.", + "list": true, + "list_add_label": "Add More", + "name": "files", + "placeholder": "", + "required": false, + "show": true, + "temp_file": true, + "title_case": false, + "trace_as_metadata": true, + "type": "file", + "value": "" + }, + "input_value": { + "_input_type": "MultilineInput", + "advanced": false, + "copy_field": false, + "display_name": "Input Text", + "dynamic": false, + "info": "Message to be passed as input.", + "input_types": [], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "multiline": true, + "name": "input_value", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "www.langflow.org" + }, + "sender": { + "_input_type": "DropdownInput", + "advanced": true, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Sender Type", + "dynamic": false, + "external_options": {}, + "info": "Type of sender.", + "name": "sender", + "options": [ + "Machine", + "User" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "User" + }, + "sender_name": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Sender Name", + "dynamic": false, + "info": "Name of the sender.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "sender_name", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "User" + }, + "session_id": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Session ID", + "dynamic": false, + "info": "The session ID of the chat. If empty, the current session ID parameter will be used.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "session_id", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "should_store_message": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Store Messages", + "dynamic": false, + "info": "Store the message in the history.", + "list": false, + "list_add_label": "Add More", + "name": "should_store_message", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + } + }, + "tool_mode": false + }, + "showNode": true, + "type": "ChatInput" + }, + "dragging": false, + "id": "ChatInput-WLvBD", + "measured": { + "height": 204, + "width": 320 + }, + "position": { + "x": 884.822226410103, + "y": 1554.894873551992 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "DataFrameOperations-hqIoy", + "node": { + "base_classes": [ + "DataFrame" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Perform various operations on a DataFrame.", + "display_name": "DataFrame Operations", + "documentation": "https://docs.langflow.org/components-processing#dataframe-operations", + "edited": false, + "field_order": [ + "df", + "operation", + "column_name", + "filter_value", + "filter_operator", + "ascending", + "new_column_name", + "new_column_value", + "columns_to_select", + "num_rows", + "replace_value", + "replacement_value" + ], + "frozen": false, + "icon": "table", + "last_updated": "2025-10-02T18:55:27.635Z", + "legacy": false, + "lf_version": "1.6.0", + "metadata": { + "code_hash": "b4d6b19b6eef", + "dependencies": { + "dependencies": [ + { + "name": "pandas", + "version": "2.2.3" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 2 + }, + "module": "lfx.components.processing.dataframe_operations.DataFrameOperationsComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "DataFrame", + "group_outputs": false, + "method": "perform_operation", + "name": "output", + "options": null, + "required_inputs": null, + "selected": "DataFrame", + "tool_mode": true, + "types": [ + "DataFrame" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "ascending": { + "_input_type": "BoolInput", + "advanced": false, + "display_name": "Sort Ascending", + "dynamic": true, + "info": "Whether to sort in ascending order.", + "list": false, + "list_add_label": "Add More", + "name": "ascending", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "import pandas as pd\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs import SortableListInput\nfrom lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\n\n\nclass DataFrameOperationsComponent(Component):\n display_name = \"DataFrame Operations\"\n description = \"Perform various operations on a DataFrame.\"\n documentation: str = \"https://docs.langflow.org/components-processing#dataframe-operations\"\n icon = \"table\"\n name = \"DataFrameOperations\"\n\n OPERATION_CHOICES = [\n \"Add Column\",\n \"Drop Column\",\n \"Filter\",\n \"Head\",\n \"Rename Column\",\n \"Replace Value\",\n \"Select Columns\",\n \"Sort\",\n \"Tail\",\n \"Drop Duplicates\",\n ]\n\n inputs = [\n DataFrameInput(\n name=\"df\",\n display_name=\"DataFrame\",\n info=\"The input DataFrame to operate on.\",\n required=True,\n ),\n SortableListInput(\n name=\"operation\",\n display_name=\"Operation\",\n placeholder=\"Select Operation\",\n info=\"Select the DataFrame operation to perform.\",\n options=[\n {\"name\": \"Add Column\", \"icon\": \"plus\"},\n {\"name\": \"Drop Column\", \"icon\": \"minus\"},\n {\"name\": \"Filter\", \"icon\": \"filter\"},\n {\"name\": \"Head\", \"icon\": \"arrow-up\"},\n {\"name\": \"Rename Column\", \"icon\": \"pencil\"},\n {\"name\": \"Replace Value\", \"icon\": \"replace\"},\n {\"name\": \"Select Columns\", \"icon\": \"columns\"},\n {\"name\": \"Sort\", \"icon\": \"arrow-up-down\"},\n {\"name\": \"Tail\", \"icon\": \"arrow-down\"},\n {\"name\": \"Drop Duplicates\", \"icon\": \"copy-x\"},\n ],\n real_time_refresh=True,\n limit=1,\n ),\n StrInput(\n name=\"column_name\",\n display_name=\"Column Name\",\n info=\"The column name to use for the operation.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"filter_value\",\n display_name=\"Filter Value\",\n info=\"The value to filter rows by.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"filter_operator\",\n display_name=\"Filter Operator\",\n options=[\n \"equals\",\n \"not equals\",\n \"contains\",\n \"not contains\",\n \"starts with\",\n \"ends with\",\n \"greater than\",\n \"less than\",\n ],\n value=\"equals\",\n info=\"The operator to apply for filtering rows.\",\n advanced=False,\n dynamic=True,\n show=False,\n ),\n BoolInput(\n name=\"ascending\",\n display_name=\"Sort Ascending\",\n info=\"Whether to sort in ascending order.\",\n dynamic=True,\n show=False,\n value=True,\n ),\n StrInput(\n name=\"new_column_name\",\n display_name=\"New Column Name\",\n info=\"The new column name when renaming or adding a column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"new_column_value\",\n display_name=\"New Column Value\",\n info=\"The value to populate the new column with.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"columns_to_select\",\n display_name=\"Columns to Select\",\n dynamic=True,\n is_list=True,\n show=False,\n ),\n IntInput(\n name=\"num_rows\",\n display_name=\"Number of Rows\",\n info=\"Number of rows to return (for head/tail).\",\n dynamic=True,\n show=False,\n value=5,\n ),\n MessageTextInput(\n name=\"replace_value\",\n display_name=\"Value to Replace\",\n info=\"The value to replace in the column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"replacement_value\",\n display_name=\"Replacement Value\",\n info=\"The value to replace with.\",\n dynamic=True,\n show=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"DataFrame\",\n name=\"output\",\n method=\"perform_operation\",\n info=\"The resulting DataFrame after the operation.\",\n )\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n dynamic_fields = [\n \"column_name\",\n \"filter_value\",\n \"filter_operator\",\n \"ascending\",\n \"new_column_name\",\n \"new_column_value\",\n \"columns_to_select\",\n \"num_rows\",\n \"replace_value\",\n \"replacement_value\",\n ]\n for field in dynamic_fields:\n build_config[field][\"show\"] = False\n\n if field_name == \"operation\":\n # Handle SortableListInput format\n if isinstance(field_value, list):\n operation_name = field_value[0].get(\"name\", \"\") if field_value else \"\"\n else:\n operation_name = field_value or \"\"\n\n # If no operation selected, all dynamic fields stay hidden (already set to False above)\n if not operation_name:\n return build_config\n\n if operation_name == \"Filter\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"filter_value\"][\"show\"] = True\n build_config[\"filter_operator\"][\"show\"] = True\n elif operation_name == \"Sort\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"ascending\"][\"show\"] = True\n elif operation_name == \"Drop Column\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Rename Column\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"new_column_name\"][\"show\"] = True\n elif operation_name == \"Add Column\":\n build_config[\"new_column_name\"][\"show\"] = True\n build_config[\"new_column_value\"][\"show\"] = True\n elif operation_name == \"Select Columns\":\n build_config[\"columns_to_select\"][\"show\"] = True\n elif operation_name in {\"Head\", \"Tail\"}:\n build_config[\"num_rows\"][\"show\"] = True\n elif operation_name == \"Replace Value\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"replace_value\"][\"show\"] = True\n build_config[\"replacement_value\"][\"show\"] = True\n elif operation_name == \"Drop Duplicates\":\n build_config[\"column_name\"][\"show\"] = True\n\n return build_config\n\n def perform_operation(self) -> DataFrame:\n df_copy = self.df.copy()\n\n # Handle SortableListInput format for operation\n operation_input = getattr(self, \"operation\", [])\n if isinstance(operation_input, list) and len(operation_input) > 0:\n op = operation_input[0].get(\"name\", \"\")\n else:\n op = \"\"\n\n # If no operation selected, return original DataFrame\n if not op:\n return df_copy\n\n if op == \"Filter\":\n return self.filter_rows_by_value(df_copy)\n if op == \"Sort\":\n return self.sort_by_column(df_copy)\n if op == \"Drop Column\":\n return self.drop_column(df_copy)\n if op == \"Rename Column\":\n return self.rename_column(df_copy)\n if op == \"Add Column\":\n return self.add_column(df_copy)\n if op == \"Select Columns\":\n return self.select_columns(df_copy)\n if op == \"Head\":\n return self.head(df_copy)\n if op == \"Tail\":\n return self.tail(df_copy)\n if op == \"Replace Value\":\n return self.replace_values(df_copy)\n if op == \"Drop Duplicates\":\n return self.drop_duplicates(df_copy)\n msg = f\"Unsupported operation: {op}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def filter_rows_by_value(self, df: DataFrame) -> DataFrame:\n column = df[self.column_name]\n filter_value = self.filter_value\n\n # Handle regular DropdownInput format (just a string value)\n operator = getattr(self, \"filter_operator\", \"equals\") # Default to equals for backward compatibility\n\n if operator == \"equals\":\n mask = column == filter_value\n elif operator == \"not equals\":\n mask = column != filter_value\n elif operator == \"contains\":\n mask = column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"not contains\":\n mask = ~column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"starts with\":\n mask = column.astype(str).str.startswith(str(filter_value), na=False)\n elif operator == \"ends with\":\n mask = column.astype(str).str.endswith(str(filter_value), na=False)\n elif operator == \"greater than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column > numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) > str(filter_value)\n elif operator == \"less than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column < numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) < str(filter_value)\n else:\n mask = column == filter_value # Fallback to equals\n\n return DataFrame(df[mask])\n\n def sort_by_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))\n\n def drop_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop(columns=[self.column_name]))\n\n def rename_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))\n\n def add_column(self, df: DataFrame) -> DataFrame:\n df[self.new_column_name] = [self.new_column_value] * len(df)\n return DataFrame(df)\n\n def select_columns(self, df: DataFrame) -> DataFrame:\n columns = [col.strip() for col in self.columns_to_select]\n return DataFrame(df[columns])\n\n def head(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.head(self.num_rows))\n\n def tail(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.tail(self.num_rows))\n\n def replace_values(self, df: DataFrame) -> DataFrame:\n df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)\n return DataFrame(df)\n\n def drop_duplicates(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop_duplicates(subset=self.column_name))\n" + }, + "column_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Column Name", + "dynamic": true, + "info": "The column name to use for the operation.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "column_name", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "columns_to_select": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Columns to Select", + "dynamic": true, + "info": "", + "list": true, + "list_add_label": "Add More", + "load_from_db": false, + "name": "columns_to_select", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "df": { + "_input_type": "DataFrameInput", + "advanced": false, + "display_name": "DataFrame", + "dynamic": false, + "info": "The input DataFrame to operate on.", + "input_types": [ + "DataFrame" + ], + "list": false, + "list_add_label": "Add More", + "name": "df", + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "other", + "value": "" + }, + "filter_operator": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Filter Operator", + "dynamic": true, + "external_options": {}, + "info": "The operator to apply for filtering rows.", + "name": "filter_operator", + "options": [ + "equals", + "not equals", + "contains", + "not contains", + "starts with", + "ends with", + "greater than", + "less than" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "equals" + }, + "filter_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Filter Value", + "dynamic": true, + "info": "The value to filter rows by.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "filter_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "new_column_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "New Column Name", + "dynamic": true, + "info": "The new column name when renaming or adding a column.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "new_column_name", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "filename" + }, + "new_column_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "New Column Value", + "dynamic": true, + "info": "The value to populate the new column with.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "new_column_value", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "num_rows": { + "_input_type": "IntInput", + "advanced": false, + "display_name": "Number of Rows", + "dynamic": true, + "info": "Number of rows to return (for head/tail).", + "list": false, + "list_add_label": "Add More", + "name": "num_rows", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 5 + }, + "operation": { + "_input_type": "SortableListInput", + "advanced": false, + "display_name": "Operation", + "dynamic": false, + "info": "Select the DataFrame operation to perform.", + "limit": 1, + "name": "operation", + "options": [ + { + "icon": "plus", + "name": "Add Column" + }, + { + "icon": "minus", + "name": "Drop Column" + }, + { + "icon": "filter", + "name": "Filter" + }, + { + "icon": "arrow-up", + "name": "Head" + }, + { + "icon": "pencil", + "name": "Rename Column" + }, + { + "icon": "replace", + "name": "Replace Value" + }, + { + "icon": "columns", + "name": "Select Columns" + }, + { + "icon": "arrow-up-down", + "name": "Sort" + }, + { + "icon": "arrow-down", + "name": "Tail" + }, + { + "icon": "copy-x", + "name": "Drop Duplicates" + } + ], + "placeholder": "Select Operation", + "real_time_refresh": true, + "required": false, + "search_category": [], + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "sortableList", + "value": [ + { + "chosen": false, + "icon": "plus", + "name": "Add Column", + "selected": false + } + ] + }, + "replace_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Value to Replace", + "dynamic": true, + "info": "The value to replace in the column.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "replace_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "replacement_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Replacement Value", + "dynamic": true, + "info": "The value to replace with.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "replacement_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + } + }, + "tool_mode": false + }, + "showNode": true, + "type": "DataFrameOperations" + }, + "dragging": false, + "id": "DataFrameOperations-hqIoy", + "measured": { + "height": 399, + "width": 320 + }, + "position": { + "x": 1601.8752590736613, + "y": 1442.944202002645 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "DataFrameOperations-A98BL", + "node": { + "base_classes": [ + "DataFrame" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Perform various operations on a DataFrame.", + "display_name": "DataFrame Operations", + "documentation": "https://docs.langflow.org/components-processing#dataframe-operations", + "edited": false, + "field_order": [ + "df", + "operation", + "column_name", + "filter_value", + "filter_operator", + "ascending", + "new_column_name", + "new_column_value", + "columns_to_select", + "num_rows", + "replace_value", + "replacement_value" + ], + "frozen": false, + "icon": "table", + "last_updated": "2025-10-02T18:55:27.636Z", + "legacy": false, + "lf_version": "1.6.0", + "metadata": { + "code_hash": "b4d6b19b6eef", + "dependencies": { + "dependencies": [ + { + "name": "pandas", + "version": "2.2.3" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 2 + }, + "module": "lfx.components.processing.dataframe_operations.DataFrameOperationsComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "DataFrame", + "group_outputs": false, + "method": "perform_operation", + "name": "output", + "options": null, + "required_inputs": null, + "selected": "DataFrame", + "tool_mode": true, + "types": [ + "DataFrame" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "ascending": { + "_input_type": "BoolInput", + "advanced": false, + "display_name": "Sort Ascending", + "dynamic": true, + "info": "Whether to sort in ascending order.", + "list": false, + "list_add_label": "Add More", + "name": "ascending", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "import pandas as pd\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs import SortableListInput\nfrom lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\n\n\nclass DataFrameOperationsComponent(Component):\n display_name = \"DataFrame Operations\"\n description = \"Perform various operations on a DataFrame.\"\n documentation: str = \"https://docs.langflow.org/components-processing#dataframe-operations\"\n icon = \"table\"\n name = \"DataFrameOperations\"\n\n OPERATION_CHOICES = [\n \"Add Column\",\n \"Drop Column\",\n \"Filter\",\n \"Head\",\n \"Rename Column\",\n \"Replace Value\",\n \"Select Columns\",\n \"Sort\",\n \"Tail\",\n \"Drop Duplicates\",\n ]\n\n inputs = [\n DataFrameInput(\n name=\"df\",\n display_name=\"DataFrame\",\n info=\"The input DataFrame to operate on.\",\n required=True,\n ),\n SortableListInput(\n name=\"operation\",\n display_name=\"Operation\",\n placeholder=\"Select Operation\",\n info=\"Select the DataFrame operation to perform.\",\n options=[\n {\"name\": \"Add Column\", \"icon\": \"plus\"},\n {\"name\": \"Drop Column\", \"icon\": \"minus\"},\n {\"name\": \"Filter\", \"icon\": \"filter\"},\n {\"name\": \"Head\", \"icon\": \"arrow-up\"},\n {\"name\": \"Rename Column\", \"icon\": \"pencil\"},\n {\"name\": \"Replace Value\", \"icon\": \"replace\"},\n {\"name\": \"Select Columns\", \"icon\": \"columns\"},\n {\"name\": \"Sort\", \"icon\": \"arrow-up-down\"},\n {\"name\": \"Tail\", \"icon\": \"arrow-down\"},\n {\"name\": \"Drop Duplicates\", \"icon\": \"copy-x\"},\n ],\n real_time_refresh=True,\n limit=1,\n ),\n StrInput(\n name=\"column_name\",\n display_name=\"Column Name\",\n info=\"The column name to use for the operation.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"filter_value\",\n display_name=\"Filter Value\",\n info=\"The value to filter rows by.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"filter_operator\",\n display_name=\"Filter Operator\",\n options=[\n \"equals\",\n \"not equals\",\n \"contains\",\n \"not contains\",\n \"starts with\",\n \"ends with\",\n \"greater than\",\n \"less than\",\n ],\n value=\"equals\",\n info=\"The operator to apply for filtering rows.\",\n advanced=False,\n dynamic=True,\n show=False,\n ),\n BoolInput(\n name=\"ascending\",\n display_name=\"Sort Ascending\",\n info=\"Whether to sort in ascending order.\",\n dynamic=True,\n show=False,\n value=True,\n ),\n StrInput(\n name=\"new_column_name\",\n display_name=\"New Column Name\",\n info=\"The new column name when renaming or adding a column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"new_column_value\",\n display_name=\"New Column Value\",\n info=\"The value to populate the new column with.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"columns_to_select\",\n display_name=\"Columns to Select\",\n dynamic=True,\n is_list=True,\n show=False,\n ),\n IntInput(\n name=\"num_rows\",\n display_name=\"Number of Rows\",\n info=\"Number of rows to return (for head/tail).\",\n dynamic=True,\n show=False,\n value=5,\n ),\n MessageTextInput(\n name=\"replace_value\",\n display_name=\"Value to Replace\",\n info=\"The value to replace in the column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"replacement_value\",\n display_name=\"Replacement Value\",\n info=\"The value to replace with.\",\n dynamic=True,\n show=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"DataFrame\",\n name=\"output\",\n method=\"perform_operation\",\n info=\"The resulting DataFrame after the operation.\",\n )\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n dynamic_fields = [\n \"column_name\",\n \"filter_value\",\n \"filter_operator\",\n \"ascending\",\n \"new_column_name\",\n \"new_column_value\",\n \"columns_to_select\",\n \"num_rows\",\n \"replace_value\",\n \"replacement_value\",\n ]\n for field in dynamic_fields:\n build_config[field][\"show\"] = False\n\n if field_name == \"operation\":\n # Handle SortableListInput format\n if isinstance(field_value, list):\n operation_name = field_value[0].get(\"name\", \"\") if field_value else \"\"\n else:\n operation_name = field_value or \"\"\n\n # If no operation selected, all dynamic fields stay hidden (already set to False above)\n if not operation_name:\n return build_config\n\n if operation_name == \"Filter\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"filter_value\"][\"show\"] = True\n build_config[\"filter_operator\"][\"show\"] = True\n elif operation_name == \"Sort\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"ascending\"][\"show\"] = True\n elif operation_name == \"Drop Column\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Rename Column\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"new_column_name\"][\"show\"] = True\n elif operation_name == \"Add Column\":\n build_config[\"new_column_name\"][\"show\"] = True\n build_config[\"new_column_value\"][\"show\"] = True\n elif operation_name == \"Select Columns\":\n build_config[\"columns_to_select\"][\"show\"] = True\n elif operation_name in {\"Head\", \"Tail\"}:\n build_config[\"num_rows\"][\"show\"] = True\n elif operation_name == \"Replace Value\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"replace_value\"][\"show\"] = True\n build_config[\"replacement_value\"][\"show\"] = True\n elif operation_name == \"Drop Duplicates\":\n build_config[\"column_name\"][\"show\"] = True\n\n return build_config\n\n def perform_operation(self) -> DataFrame:\n df_copy = self.df.copy()\n\n # Handle SortableListInput format for operation\n operation_input = getattr(self, \"operation\", [])\n if isinstance(operation_input, list) and len(operation_input) > 0:\n op = operation_input[0].get(\"name\", \"\")\n else:\n op = \"\"\n\n # If no operation selected, return original DataFrame\n if not op:\n return df_copy\n\n if op == \"Filter\":\n return self.filter_rows_by_value(df_copy)\n if op == \"Sort\":\n return self.sort_by_column(df_copy)\n if op == \"Drop Column\":\n return self.drop_column(df_copy)\n if op == \"Rename Column\":\n return self.rename_column(df_copy)\n if op == \"Add Column\":\n return self.add_column(df_copy)\n if op == \"Select Columns\":\n return self.select_columns(df_copy)\n if op == \"Head\":\n return self.head(df_copy)\n if op == \"Tail\":\n return self.tail(df_copy)\n if op == \"Replace Value\":\n return self.replace_values(df_copy)\n if op == \"Drop Duplicates\":\n return self.drop_duplicates(df_copy)\n msg = f\"Unsupported operation: {op}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def filter_rows_by_value(self, df: DataFrame) -> DataFrame:\n column = df[self.column_name]\n filter_value = self.filter_value\n\n # Handle regular DropdownInput format (just a string value)\n operator = getattr(self, \"filter_operator\", \"equals\") # Default to equals for backward compatibility\n\n if operator == \"equals\":\n mask = column == filter_value\n elif operator == \"not equals\":\n mask = column != filter_value\n elif operator == \"contains\":\n mask = column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"not contains\":\n mask = ~column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"starts with\":\n mask = column.astype(str).str.startswith(str(filter_value), na=False)\n elif operator == \"ends with\":\n mask = column.astype(str).str.endswith(str(filter_value), na=False)\n elif operator == \"greater than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column > numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) > str(filter_value)\n elif operator == \"less than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column < numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) < str(filter_value)\n else:\n mask = column == filter_value # Fallback to equals\n\n return DataFrame(df[mask])\n\n def sort_by_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))\n\n def drop_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop(columns=[self.column_name]))\n\n def rename_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))\n\n def add_column(self, df: DataFrame) -> DataFrame:\n df[self.new_column_name] = [self.new_column_value] * len(df)\n return DataFrame(df)\n\n def select_columns(self, df: DataFrame) -> DataFrame:\n columns = [col.strip() for col in self.columns_to_select]\n return DataFrame(df[columns])\n\n def head(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.head(self.num_rows))\n\n def tail(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.tail(self.num_rows))\n\n def replace_values(self, df: DataFrame) -> DataFrame:\n df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)\n return DataFrame(df)\n\n def drop_duplicates(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop_duplicates(subset=self.column_name))\n" + }, + "column_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Column Name", + "dynamic": true, + "info": "The column name to use for the operation.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "column_name", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "columns_to_select": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Columns to Select", + "dynamic": true, + "info": "", + "list": true, + "list_add_label": "Add More", + "load_from_db": false, + "name": "columns_to_select", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "df": { + "_input_type": "DataFrameInput", + "advanced": false, + "display_name": "DataFrame", + "dynamic": false, + "info": "The input DataFrame to operate on.", + "input_types": [ + "DataFrame" + ], + "list": false, + "list_add_label": "Add More", + "name": "df", + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "other", + "value": "" + }, + "filter_operator": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Filter Operator", + "dynamic": true, + "external_options": {}, + "info": "The operator to apply for filtering rows.", + "name": "filter_operator", + "options": [ + "equals", + "not equals", + "contains", + "not contains", + "starts with", + "ends with", + "greater than", + "less than" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "equals" + }, + "filter_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Filter Value", + "dynamic": true, + "info": "The value to filter rows by.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "filter_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "new_column_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "New Column Name", + "dynamic": true, + "info": "The new column name when renaming or adding a column.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "new_column_name", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "mimetype" + }, + "new_column_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "New Column Value", + "dynamic": true, + "info": "The value to populate the new column with.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "new_column_value", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "text/html" + }, + "num_rows": { + "_input_type": "IntInput", + "advanced": false, + "display_name": "Number of Rows", + "dynamic": true, + "info": "Number of rows to return (for head/tail).", + "list": false, + "list_add_label": "Add More", + "name": "num_rows", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 5 + }, + "operation": { + "_input_type": "SortableListInput", + "advanced": false, + "display_name": "Operation", + "dynamic": false, + "info": "Select the DataFrame operation to perform.", + "limit": 1, + "name": "operation", + "options": [ + { + "icon": "plus", + "name": "Add Column" + }, + { + "icon": "minus", + "name": "Drop Column" + }, + { + "icon": "filter", + "name": "Filter" + }, + { + "icon": "arrow-up", + "name": "Head" + }, + { + "icon": "pencil", + "name": "Rename Column" + }, + { + "icon": "replace", + "name": "Replace Value" + }, + { + "icon": "columns", + "name": "Select Columns" + }, + { + "icon": "arrow-up-down", + "name": "Sort" + }, + { + "icon": "arrow-down", + "name": "Tail" + }, + { + "icon": "copy-x", + "name": "Drop Duplicates" + } + ], + "placeholder": "Select Operation", + "real_time_refresh": true, + "required": false, + "search_category": [], + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "sortableList", + "value": [ + { + "chosen": false, + "icon": "plus", + "name": "Add Column", + "selected": false + } + ] + }, + "replace_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Value to Replace", + "dynamic": true, + "info": "The value to replace in the column.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "replace_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "replacement_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Replacement Value", + "dynamic": true, + "info": "The value to replace with.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "replacement_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + } + }, + "tool_mode": false + }, + "showNode": true, + "type": "DataFrameOperations" + }, + "dragging": false, + "id": "DataFrameOperations-A98BL", + "measured": { + "height": 399, + "width": 320 + }, + "position": { + "x": 1946.8185577395595, + "y": 1432.2126327108165 + }, + "selected": false, + "type": "genericNode" + } + ], + "viewport": { + "x": -455.58644249058534, + "y": -635.4279993708312, + "zoom": 0.5860911118774581 + } + }, + "description": "This flow is to ingest the URL to open search.", + "endpoint_name": null, + "id": "72c3d17c-2dac-4a73-b48a-6518473d7830", + "is_component": false, + "last_tested_version": "1.6.0", + "mcp_enabled": true, + "name": "OpenSearch URL Ingestion Flow", + "tags": [ + "openai", + "astradb", + "rag", + "q-a" + ] +} \ No newline at end of file From 41e4ecefdb987191a6444c8502ff2698f9a3149f Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 16:50:34 -0300 Subject: [PATCH 06/55] implement tasks fetching and cancelling on useQuery --- .../api/mutations/useCancelTaskMutation.ts | 47 +++++++++++ .../app/api/queries/useGetTaskStatusQuery.ts | 80 +++++++++++++++++++ .../src/app/api/queries/useGetTasksQuery.ts | 79 ++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 frontend/src/app/api/mutations/useCancelTaskMutation.ts create mode 100644 frontend/src/app/api/queries/useGetTaskStatusQuery.ts create mode 100644 frontend/src/app/api/queries/useGetTasksQuery.ts diff --git a/frontend/src/app/api/mutations/useCancelTaskMutation.ts b/frontend/src/app/api/mutations/useCancelTaskMutation.ts new file mode 100644 index 00000000..1bf2faed --- /dev/null +++ b/frontend/src/app/api/mutations/useCancelTaskMutation.ts @@ -0,0 +1,47 @@ +import { + type UseMutationOptions, + useMutation, + useQueryClient, +} from "@tanstack/react-query"; + +export interface CancelTaskRequest { + taskId: string; +} + +export interface CancelTaskResponse { + status: string; + task_id: string; +} + +export const useCancelTaskMutation = ( + options?: Omit< + UseMutationOptions, + "mutationFn" + > +) => { + const queryClient = useQueryClient(); + + async function cancelTask( + variables: CancelTaskRequest, + ): Promise { + const response = await fetch(`/api/tasks/${variables.taskId}/cancel`, { + method: "POST", + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error || "Failed to cancel task"); + } + + return response.json(); + } + + return useMutation({ + mutationFn: cancelTask, + onSuccess: () => { + // Invalidate tasks query to refresh the list + queryClient.invalidateQueries({ queryKey: ["tasks"] }); + }, + ...options, + }); +}; diff --git a/frontend/src/app/api/queries/useGetTaskStatusQuery.ts b/frontend/src/app/api/queries/useGetTaskStatusQuery.ts new file mode 100644 index 00000000..17cd2d16 --- /dev/null +++ b/frontend/src/app/api/queries/useGetTaskStatusQuery.ts @@ -0,0 +1,80 @@ +import { + type UseQueryOptions, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; + +export interface TaskStatus { + task_id: string; + status: + | "pending" + | "running" + | "processing" + | "completed" + | "failed" + | "error"; + total_files?: number; + processed_files?: number; + successful_files?: number; + failed_files?: number; + running_files?: number; + pending_files?: number; + created_at: string; + updated_at: string; + duration_seconds?: number; + result?: Record; + error?: string; + files?: Record>; +} + +export const useGetTaskStatusQuery = ( + taskId: string, + options?: Omit, "queryKey" | "queryFn"> +) => { + const queryClient = useQueryClient(); + + async function getTaskStatus(): Promise { + if (!taskId) { + return null; + } + + const response = await fetch(`/api/tasks/${taskId}`); + + if (!response.ok) { + if (response.status === 404) { + return null; // Task not found + } + throw new Error("Failed to fetch task status"); + } + + return response.json(); + } + + const queryResult = useQuery( + { + queryKey: ["task-status", taskId], + queryFn: getTaskStatus, + refetchInterval: (data) => { + // Only poll if the task is still active + if (!data) { + return false; // Stop polling if no data + } + + const isActive = + data.status === "pending" || + data.status === "running" || + data.status === "processing"; + + return isActive ? 3000 : false; // Poll every 3 seconds if active + }, + refetchIntervalInBackground: true, + staleTime: 0, // Always consider data stale to ensure fresh updates + gcTime: 5 * 60 * 1000, // Keep in cache for 5 minutes + enabled: !!taskId, // Only run if taskId is provided + ...options, + }, + queryClient, + ); + + return queryResult; +}; diff --git a/frontend/src/app/api/queries/useGetTasksQuery.ts b/frontend/src/app/api/queries/useGetTasksQuery.ts new file mode 100644 index 00000000..1ea59d26 --- /dev/null +++ b/frontend/src/app/api/queries/useGetTasksQuery.ts @@ -0,0 +1,79 @@ +import { + type UseQueryOptions, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; + +export interface Task { + task_id: string; + status: + | "pending" + | "running" + | "processing" + | "completed" + | "failed" + | "error"; + total_files?: number; + processed_files?: number; + successful_files?: number; + failed_files?: number; + running_files?: number; + pending_files?: number; + created_at: string; + updated_at: string; + duration_seconds?: number; + result?: Record; + error?: string; + files?: Record>; +} + +export interface TasksResponse { + tasks: Task[]; +} + +export const useGetTasksQuery = ( + options?: Omit, "queryKey" | "queryFn"> +) => { + const queryClient = useQueryClient(); + + async function getTasks(): Promise { + const response = await fetch("/api/tasks"); + + if (!response.ok) { + throw new Error("Failed to fetch tasks"); + } + + const data: TasksResponse = await response.json(); + return data.tasks || []; + } + + const queryResult = useQuery( + { + queryKey: ["tasks"], + queryFn: getTasks, + refetchInterval: (query) => { + // Only poll if there are tasks with pending or running status + const data = query.state.data; + if (!data || data.length === 0) { + return false; // Stop polling if no tasks + } + + const hasActiveTasks = data.some( + (task: Task) => + task.status === "pending" || + task.status === "running" || + task.status === "processing" + ); + + return hasActiveTasks ? 3000 : false; // Poll every 3 seconds if active tasks exist + }, + refetchIntervalInBackground: true, + staleTime: 0, // Always consider data stale to ensure fresh updates + gcTime: 5 * 60 * 1000, // Keep in cache for 5 minutes + ...options, + }, + queryClient, + ); + + return queryResult; +}; From aa61ba265c3a9744873c817c75779a15910fe9e7 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 16:50:47 -0300 Subject: [PATCH 07/55] made files and toast appear just once, use new queries --- frontend/components/knowledge-dropdown.tsx | 1117 ++++++++++---------- frontend/src/contexts/task-context.tsx | 427 ++++---- 2 files changed, 733 insertions(+), 811 deletions(-) diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index ee49fc3a..9b71ee81 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -1,24 +1,25 @@ "use client"; import { - ChevronDown, - Cloud, - FolderOpen, - Loader2, - PlugZap, - Plus, - Upload, + ChevronDown, + Cloud, + FolderOpen, + Loader2, + PlugZap, + Plus, + Upload, } from "lucide-react"; import { useRouter } from "next/navigation"; import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; +import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery"; import { Button } from "@/components/ui/button"; import { - Dialog, - DialogContent, - DialogDescription, - DialogHeader, - DialogTitle, + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; @@ -26,600 +27,590 @@ import { useTask } from "@/contexts/task-context"; import { cn } from "@/lib/utils"; interface KnowledgeDropdownProps { - active?: boolean; - variant?: "navigation" | "button"; + active?: boolean; + variant?: "navigation" | "button"; } export function KnowledgeDropdown({ - active, - variant = "navigation", + active, + variant = "navigation", }: KnowledgeDropdownProps) { - const { addTask } = useTask(); - const router = useRouter(); - const [isOpen, setIsOpen] = useState(false); - const [showFolderDialog, setShowFolderDialog] = useState(false); - const [showS3Dialog, setShowS3Dialog] = useState(false); - const [awsEnabled, setAwsEnabled] = useState(false); - const [folderPath, setFolderPath] = useState("/app/documents/"); - const [bucketUrl, setBucketUrl] = useState("s3://"); - const [folderLoading, setFolderLoading] = useState(false); - const [s3Loading, setS3Loading] = useState(false); - const [fileUploading, setFileUploading] = useState(false); - const [isNavigatingToCloud, setIsNavigatingToCloud] = useState(false); - const [cloudConnectors, setCloudConnectors] = useState<{ - [key: string]: { - name: string; - available: boolean; - connected: boolean; - hasToken: boolean; - }; - }>({}); - const fileInputRef = useRef(null); - const dropdownRef = useRef(null); + const { addTask } = useTask(); + const { refetch: refetchTasks } = useGetTasksQuery(); + const router = useRouter(); + const [isOpen, setIsOpen] = useState(false); + const [showFolderDialog, setShowFolderDialog] = useState(false); + const [showS3Dialog, setShowS3Dialog] = useState(false); + const [awsEnabled, setAwsEnabled] = useState(false); + const [folderPath, setFolderPath] = useState("/app/documents/"); + const [bucketUrl, setBucketUrl] = useState("s3://"); + const [folderLoading, setFolderLoading] = useState(false); + const [s3Loading, setS3Loading] = useState(false); + const [fileUploading, setFileUploading] = useState(false); + const [isNavigatingToCloud, setIsNavigatingToCloud] = useState(false); + const [cloudConnectors, setCloudConnectors] = useState<{ + [key: string]: { + name: string; + available: boolean; + connected: boolean; + hasToken: boolean; + }; + }>({}); + const fileInputRef = useRef(null); + const dropdownRef = useRef(null); - // Check AWS availability and cloud connectors on mount - useEffect(() => { - const checkAvailability = async () => { - try { - // Check AWS - const awsRes = await fetch("/api/upload_options"); - if (awsRes.ok) { - const awsData = await awsRes.json(); - setAwsEnabled(Boolean(awsData.aws)); - } + // Check AWS availability and cloud connectors on mount + useEffect(() => { + const checkAvailability = async () => { + try { + // Check AWS + const awsRes = await fetch("/api/upload_options"); + if (awsRes.ok) { + const awsData = await awsRes.json(); + setAwsEnabled(Boolean(awsData.aws)); + } - // Check cloud connectors - const connectorsRes = await fetch("/api/connectors"); - if (connectorsRes.ok) { - const connectorsResult = await connectorsRes.json(); - const cloudConnectorTypes = [ - "google_drive", - "onedrive", - "sharepoint", - ]; - const connectorInfo: { - [key: string]: { - name: string; - available: boolean; - connected: boolean; - hasToken: boolean; - }; - } = {}; + // Check cloud connectors + const connectorsRes = await fetch("/api/connectors"); + if (connectorsRes.ok) { + const connectorsResult = await connectorsRes.json(); + const cloudConnectorTypes = [ + "google_drive", + "onedrive", + "sharepoint", + ]; + const connectorInfo: { + [key: string]: { + name: string; + available: boolean; + connected: boolean; + hasToken: boolean; + }; + } = {}; - for (const type of cloudConnectorTypes) { - if (connectorsResult.connectors[type]) { - connectorInfo[type] = { - name: connectorsResult.connectors[type].name, - available: connectorsResult.connectors[type].available, - connected: false, - hasToken: false, - }; + for (const type of cloudConnectorTypes) { + if (connectorsResult.connectors[type]) { + connectorInfo[type] = { + name: connectorsResult.connectors[type].name, + available: connectorsResult.connectors[type].available, + connected: false, + hasToken: false, + }; - // Check connection status - try { - const statusRes = await fetch(`/api/connectors/${type}/status`); - if (statusRes.ok) { - const statusData = await statusRes.json(); - const connections = statusData.connections || []; - const activeConnection = connections.find( - (conn: { is_active: boolean; connection_id: string }) => - conn.is_active - ); - const isConnected = activeConnection !== undefined; + // Check connection status + try { + const statusRes = await fetch(`/api/connectors/${type}/status`); + if (statusRes.ok) { + const statusData = await statusRes.json(); + const connections = statusData.connections || []; + const activeConnection = connections.find( + (conn: { is_active: boolean; connection_id: string }) => + conn.is_active, + ); + const isConnected = activeConnection !== undefined; - if (isConnected && activeConnection) { - connectorInfo[type].connected = true; + if (isConnected && activeConnection) { + connectorInfo[type].connected = true; - // Check token availability - try { - const tokenRes = await fetch( - `/api/connectors/${type}/token?connection_id=${activeConnection.connection_id}` - ); - if (tokenRes.ok) { - const tokenData = await tokenRes.json(); - if (tokenData.access_token) { - connectorInfo[type].hasToken = true; - } - } - } catch { - // Token check failed - } - } - } - } catch { - // Status check failed - } - } - } + // Check token availability + try { + const tokenRes = await fetch( + `/api/connectors/${type}/token?connection_id=${activeConnection.connection_id}`, + ); + if (tokenRes.ok) { + const tokenData = await tokenRes.json(); + if (tokenData.access_token) { + connectorInfo[type].hasToken = true; + } + } + } catch { + // Token check failed + } + } + } + } catch { + // Status check failed + } + } + } - setCloudConnectors(connectorInfo); - } - } catch (err) { - console.error("Failed to check availability", err); - } - }; - checkAvailability(); - }, []); + setCloudConnectors(connectorInfo); + } + } catch (err) { + console.error("Failed to check availability", err); + } + }; + checkAvailability(); + }, []); - // Handle click outside to close dropdown - useEffect(() => { - const handleClickOutside = (event: MouseEvent) => { - if ( - dropdownRef.current && - !dropdownRef.current.contains(event.target as Node) - ) { - setIsOpen(false); - } - }; + // Handle click outside to close dropdown + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + dropdownRef.current && + !dropdownRef.current.contains(event.target as Node) + ) { + setIsOpen(false); + } + }; - if (isOpen) { - document.addEventListener("mousedown", handleClickOutside); - return () => - document.removeEventListener("mousedown", handleClickOutside); - } - }, [isOpen]); + if (isOpen) { + document.addEventListener("mousedown", handleClickOutside); + return () => + document.removeEventListener("mousedown", handleClickOutside); + } + }, [isOpen]); - const handleFileUpload = () => { - fileInputRef.current?.click(); - }; + const handleFileUpload = () => { + fileInputRef.current?.click(); + }; - const handleFileChange = async (e: React.ChangeEvent) => { - const files = e.target.files; - if (files && files.length > 0) { - // Close dropdown and disable button immediately after file selection - setIsOpen(false); - setFileUploading(true); + const handleFileChange = async (e: React.ChangeEvent) => { + const files = e.target.files; + if (files && files.length > 0) { + // Close dropdown and disable button immediately after file selection + setIsOpen(false); + setFileUploading(true); - // Trigger the same file upload event as the chat page - window.dispatchEvent( - new CustomEvent("fileUploadStart", { - detail: { filename: files[0].name }, - }) - ); + // Trigger the same file upload event as the chat page + window.dispatchEvent( + new CustomEvent("fileUploadStart", { + detail: { filename: files[0].name }, + }), + ); - try { - const formData = new FormData(); - formData.append("file", files[0]); + try { + const formData = new FormData(); + formData.append("file", files[0]); - // Use router upload and ingest endpoint (automatically routes based on configuration) - const uploadIngestRes = await fetch("/api/router/upload_ingest", { - method: "POST", - body: formData, - }); + // Use router upload and ingest endpoint (automatically routes based on configuration) + const uploadIngestRes = await fetch("/api/router/upload_ingest", { + method: "POST", + body: formData, + }); - const uploadIngestJson = await uploadIngestRes.json(); + const uploadIngestJson = await uploadIngestRes.json(); - if (!uploadIngestRes.ok) { - throw new Error( - uploadIngestJson?.error || "Upload and ingest failed" - ); - } + if (!uploadIngestRes.ok) { + throw new Error( + uploadIngestJson?.error || "Upload and ingest failed", + ); + } - // Extract results from the response - handle both unified and simple formats - const fileId = uploadIngestJson?.upload?.id || uploadIngestJson?.id; - const filePath = - uploadIngestJson?.upload?.path || - uploadIngestJson?.path || - "uploaded"; - const runJson = uploadIngestJson?.ingestion; - const deleteResult = uploadIngestJson?.deletion; + // Extract results from the response - handle both unified and simple formats + const fileId = uploadIngestJson?.upload?.id || uploadIngestJson?.id || uploadIngestJson?.task_id; + const filePath = + uploadIngestJson?.upload?.path || + uploadIngestJson?.path || + "uploaded"; + const runJson = uploadIngestJson?.ingestion; + const deleteResult = uploadIngestJson?.deletion; + console.log("c", uploadIngestJson ) + if (!fileId) { + throw new Error("Upload successful but no file id returned"); + } + // Check if ingestion actually succeeded + if ( + runJson && + runJson.status !== "COMPLETED" && + runJson.status !== "SUCCESS" + ) { + const errorMsg = runJson.error || "Ingestion pipeline failed"; + throw new Error( + `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.`, + ); + } + // Log deletion status if provided + if (deleteResult) { + if (deleteResult.status === "deleted") { + console.log( + "File successfully cleaned up from Langflow:", + deleteResult.file_id, + ); + } else if (deleteResult.status === "delete_failed") { + console.warn( + "Failed to cleanup file from Langflow:", + deleteResult.error, + ); + } + } + // Notify UI + window.dispatchEvent( + new CustomEvent("fileUploaded", { + detail: { + file: files[0], + result: { + file_id: fileId, + file_path: filePath, + run: runJson, + deletion: deleteResult, + unified: true, + }, + }, + }), + ); - if (!fileId) { - throw new Error("Upload successful but no file id returned"); - } + refetchTasks(); + } catch (error) { + window.dispatchEvent( + new CustomEvent("fileUploadError", { + detail: { + filename: files[0].name, + error: error instanceof Error ? error.message : "Upload failed", + }, + }), + ); + } finally { + window.dispatchEvent(new CustomEvent("fileUploadComplete")); + setFileUploading(false); + } + } - // Check if ingestion actually succeeded - if ( - runJson && - runJson.status !== "COMPLETED" && - runJson.status !== "SUCCESS" - ) { - const errorMsg = runJson.error || "Ingestion pipeline failed"; - throw new Error( - `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.` - ); - } + // Reset file input + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } + }; - // Log deletion status if provided - if (deleteResult) { - if (deleteResult.status === "deleted") { - console.log( - "File successfully cleaned up from Langflow:", - deleteResult.file_id - ); - } else if (deleteResult.status === "delete_failed") { - console.warn( - "Failed to cleanup file from Langflow:", - deleteResult.error - ); - } - } + const handleFolderUpload = async () => { + if (!folderPath.trim()) return; - // Notify UI - window.dispatchEvent( - new CustomEvent("fileUploaded", { - detail: { - file: files[0], - result: { - file_id: fileId, - file_path: filePath, - run: runJson, - deletion: deleteResult, - unified: true, - }, - }, - }) - ); + setFolderLoading(true); + setShowFolderDialog(false); - // Trigger search refresh after successful ingestion - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - } catch (error) { - window.dispatchEvent( - new CustomEvent("fileUploadError", { - detail: { - filename: files[0].name, - error: error instanceof Error ? error.message : "Upload failed", - }, - }) - ); - } finally { - window.dispatchEvent(new CustomEvent("fileUploadComplete")); - setFileUploading(false); - // Don't call refetchSearch() here - the knowledgeUpdated event will handle it - } - } + try { + const response = await fetch("/api/upload_path", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ path: folderPath }), + }); - // Reset file input - if (fileInputRef.current) { - fileInputRef.current.value = ""; - } - }; + const result = await response.json(); - const handleFolderUpload = async () => { - if (!folderPath.trim()) return; + if (response.status === 201) { + const taskId = result.task_id || result.id; - setFolderLoading(true); - setShowFolderDialog(false); + if (!taskId) { + throw new Error("No task ID received from server"); + } - try { - const response = await fetch("/api/upload_path", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ path: folderPath }), - }); + addTask(taskId); + setFolderPath(""); + // Refetch tasks to show the new task + refetchTasks(); + } else if (response.ok) { + setFolderPath(""); + // Refetch tasks even for direct uploads in case tasks were created + refetchTasks(); + } else { + console.error("Folder upload failed:", result.error); + if (response.status === 400) { + toast.error("Upload failed", { + description: result.error || "Bad request", + }); + } + } + } catch (error) { + console.error("Folder upload error:", error); + } finally { + setFolderLoading(false); + } + }; - const result = await response.json(); + const handleS3Upload = async () => { + if (!bucketUrl.trim()) return; - if (response.status === 201) { - const taskId = result.task_id || result.id; + setS3Loading(true); + setShowS3Dialog(false); - if (!taskId) { - throw new Error("No task ID received from server"); - } + try { + const response = await fetch("/api/upload_bucket", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ s3_url: bucketUrl }), + }); - addTask(taskId); - setFolderPath(""); - // Trigger search refresh after successful folder processing starts - console.log( - "Folder upload successful, dispatching knowledgeUpdated event" - ); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - } else if (response.ok) { - setFolderPath(""); - console.log( - "Folder upload successful (direct), dispatching knowledgeUpdated event" - ); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - } else { - console.error("Folder upload failed:", result.error); - if (response.status === 400) { - toast.error("Upload failed", { - description: result.error || "Bad request", - }); - } - } - } catch (error) { - console.error("Folder upload error:", error); - } finally { - setFolderLoading(false); - // Don't call refetchSearch() here - the knowledgeUpdated event will handle it - } - }; + const result = await response.json(); - const handleS3Upload = async () => { - if (!bucketUrl.trim()) return; + if (response.status === 201) { + const taskId = result.task_id || result.id; - setS3Loading(true); - setShowS3Dialog(false); + if (!taskId) { + throw new Error("No task ID received from server"); + } - try { - const response = await fetch("/api/upload_bucket", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ s3_url: bucketUrl }), - }); + addTask(taskId); + setBucketUrl("s3://"); + // Refetch tasks to show the new task + refetchTasks(); + } else { + console.error("S3 upload failed:", result.error); + if (response.status === 400) { + toast.error("Upload failed", { + description: result.error || "Bad request", + }); + } + } + } catch (error) { + console.error("S3 upload error:", error); + } finally { + setS3Loading(false); + } + }; - const result = await response.json(); + const cloudConnectorItems = Object.entries(cloudConnectors) + .filter(([, info]) => info.available) + .map(([type, info]) => ({ + label: info.name, + icon: PlugZap, + onClick: async () => { + setIsOpen(false); + if (info.connected && info.hasToken) { + setIsNavigatingToCloud(true); + try { + router.push(`/upload/${type}`); + // Keep loading state for a short time to show feedback + setTimeout(() => setIsNavigatingToCloud(false), 1000); + } catch { + setIsNavigatingToCloud(false); + } + } else { + router.push("/settings"); + } + }, + disabled: !info.connected || !info.hasToken, + tooltip: !info.connected + ? `Connect ${info.name} in Settings first` + : !info.hasToken + ? `Reconnect ${info.name} - access token required` + : undefined, + })); - if (response.status === 201) { - const taskId = result.task_id || result.id; + const menuItems = [ + { + label: "Add File", + icon: Upload, + onClick: handleFileUpload, + }, + { + label: "Process Folder", + icon: FolderOpen, + onClick: () => { + setIsOpen(false); + setShowFolderDialog(true); + }, + }, + ...(awsEnabled + ? [ + { + label: "Process S3 Bucket", + icon: Cloud, + onClick: () => { + setIsOpen(false); + setShowS3Dialog(true); + }, + }, + ] + : []), + ...cloudConnectorItems, + ]; - if (!taskId) { - throw new Error("No task ID received from server"); - } + // Comprehensive loading state + const isLoading = + fileUploading || folderLoading || s3Loading || isNavigatingToCloud; - addTask(taskId); - setBucketUrl("s3://"); - // Trigger search refresh after successful S3 processing starts - console.log("S3 upload successful, dispatching knowledgeUpdated event"); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - } else { - console.error("S3 upload failed:", result.error); - if (response.status === 400) { - toast.error("Upload failed", { - description: result.error || "Bad request", - }); - } - } - } catch (error) { - console.error("S3 upload error:", error); - } finally { - setS3Loading(false); - // Don't call refetchSearch() here - the knowledgeUpdated event will handle it - } - }; + return ( + <> +
+ - const cloudConnectorItems = Object.entries(cloudConnectors) - .filter(([, info]) => info.available) - .map(([type, info]) => ({ - label: info.name, - icon: PlugZap, - onClick: async () => { - setIsOpen(false); - if (info.connected && info.hasToken) { - setIsNavigatingToCloud(true); - try { - router.push(`/upload/${type}`); - // Keep loading state for a short time to show feedback - setTimeout(() => setIsNavigatingToCloud(false), 1000); - } catch { - setIsNavigatingToCloud(false); - } - } else { - router.push("/settings"); - } - }, - disabled: !info.connected || !info.hasToken, - tooltip: !info.connected - ? `Connect ${info.name} in Settings first` - : !info.hasToken - ? `Reconnect ${info.name} - access token required` - : undefined, - })); + {isOpen && !isLoading && ( +
+
+ {menuItems.map((item, index) => ( + + ))} +
+
+ )} - const menuItems = [ - { - label: "Add File", - icon: Upload, - onClick: handleFileUpload, - }, - { - label: "Process Folder", - icon: FolderOpen, - onClick: () => { - setIsOpen(false); - setShowFolderDialog(true); - }, - }, - ...(awsEnabled - ? [ - { - label: "Process S3 Bucket", - icon: Cloud, - onClick: () => { - setIsOpen(false); - setShowS3Dialog(true); - }, - }, - ] - : []), - ...cloudConnectorItems, - ]; + +
- // Comprehensive loading state - const isLoading = - fileUploading || folderLoading || s3Loading || isNavigatingToCloud; + {/* Process Folder Dialog */} + + + + + + Process Folder + + + Process all documents in a folder path + + +
+
+ + setFolderPath(e.target.value)} + /> +
+
+ + +
+
+ + - return ( - <> -
- - - {isOpen && !isLoading && ( -
-
- {menuItems.map((item, index) => ( - - ))} -
-
- )} - - -
- - {/* Process Folder Dialog */} - - - - - - Process Folder - - - Process all documents in a folder path - - -
-
- - setFolderPath(e.target.value)} - /> -
-
- - -
-
- - - - {/* Process S3 Bucket Dialog */} - - - - - - Process S3 Bucket - - - Process all documents from an S3 bucket. AWS credentials must be - configured. - - -
-
- - setBucketUrl(e.target.value)} - /> -
-
- - -
-
- - - - ); + {/* Process S3 Bucket Dialog */} + + + + + + Process S3 Bucket + + + Process all documents from an S3 bucket. AWS credentials must be + configured. + + +
+
+ + setBucketUrl(e.target.value)} + /> +
+
+ + +
+
+ + + + ); } diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index 5eb10ea9..b3275422 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -7,33 +7,18 @@ import { useCallback, useContext, useEffect, + useRef, useState, } from "react"; import { toast } from "sonner"; +import { useCancelTaskMutation } from "@/app/api/mutations/useCancelTaskMutation"; +import { + type Task, + useGetTasksQuery, +} from "@/app/api/queries/useGetTasksQuery"; import { useAuth } from "@/contexts/auth-context"; -export interface Task { - task_id: string; - status: - | "pending" - | "running" - | "processing" - | "completed" - | "failed" - | "error"; - total_files?: number; - processed_files?: number; - successful_files?: number; - failed_files?: number; - running_files?: number; - pending_files?: number; - created_at: string; - updated_at: string; - duration_seconds?: number; - result?: Record; - error?: string; - files?: Record>; -} +// Task interface is now imported from useGetTasksQuery export interface TaskFile { filename: string; @@ -58,20 +43,45 @@ interface TaskContextType { isFetching: boolean; isMenuOpen: boolean; toggleMenu: () => void; + // React Query states + isLoading: boolean; + error: Error | null; } const TaskContext = createContext(undefined); export function TaskProvider({ children }: { children: React.ReactNode }) { - const [tasks, setTasks] = useState([]); const [files, setFiles] = useState([]); - const [isPolling, setIsPolling] = useState(false); - const [isFetching, setIsFetching] = useState(false); const [isMenuOpen, setIsMenuOpen] = useState(false); + const previousTasksRef = useRef([]); const { isAuthenticated, isNoAuthMode } = useAuth(); const queryClient = useQueryClient(); + // Use React Query hooks + const { + data: tasks = [], + isLoading, + error, + refetch: refetchTasks, + isFetching, + } = useGetTasksQuery({ + enabled: isAuthenticated || isNoAuthMode, + }); + + const cancelTaskMutation = useCancelTaskMutation({ + onSuccess: () => { + toast.success("Task cancelled", { + description: "Task has been cancelled successfully", + }); + }, + onError: (error) => { + toast.error("Failed to cancel task", { + description: error.message, + }); + }, + }); + const refetchSearch = useCallback(() => { queryClient.invalidateQueries({ queryKey: ["search"], @@ -99,252 +109,171 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { [], ); - const fetchTasks = useCallback(async () => { - if (!isAuthenticated && !isNoAuthMode) return; - - setIsFetching(true); - try { - const response = await fetch("/api/tasks"); - if (response.ok) { - const data = await response.json(); - const newTasks = data.tasks || []; - - // Update tasks and check for status changes in the same state update - setTasks((prevTasks) => { - // Check for newly completed tasks to show toasts - if (prevTasks.length > 0) { - newTasks.forEach((newTask: Task) => { - const oldTask = prevTasks.find( - (t) => t.task_id === newTask.task_id, - ); - - // Update or add files from task.files if available - if (newTask.files && typeof newTask.files === "object") { - const taskFileEntries = Object.entries(newTask.files); - const now = new Date().toISOString(); - - taskFileEntries.forEach(([filePath, fileInfo]) => { - if (typeof fileInfo === "object" && fileInfo) { - const fileName = filePath.split("/").pop() || filePath; - const fileStatus = fileInfo.status as string; - - // Map backend file status to our TaskFile status - let mappedStatus: TaskFile["status"]; - switch (fileStatus) { - case "pending": - case "running": - mappedStatus = "processing"; - break; - case "completed": - mappedStatus = "active"; - break; - case "failed": - mappedStatus = "failed"; - break; - default: - mappedStatus = "processing"; - } - - setFiles((prevFiles) => { - const existingFileIndex = prevFiles.findIndex( - (f) => - f.source_url === filePath && - f.task_id === newTask.task_id, - ); - - // Detect connector type based on file path or other indicators - let connectorType = "local"; - if (filePath.includes("/") && !filePath.startsWith("/")) { - // Likely S3 key format (bucket/path/file.ext) - connectorType = "s3"; - } - - const fileEntry: TaskFile = { - filename: fileName, - mimetype: "", // We don't have this info from the task - source_url: filePath, - size: 0, // We don't have this info from the task - connector_type: connectorType, - status: mappedStatus, - task_id: newTask.task_id, - created_at: - typeof fileInfo.created_at === "string" - ? fileInfo.created_at - : now, - updated_at: - typeof fileInfo.updated_at === "string" - ? fileInfo.updated_at - : now, - }; - - if (existingFileIndex >= 0) { - // Update existing file - const updatedFiles = [...prevFiles]; - updatedFiles[existingFileIndex] = fileEntry; - return updatedFiles; - } else { - // Add new file - return [...prevFiles, fileEntry]; - } - }); - } - }); - } - - if ( - oldTask && - oldTask.status !== "completed" && - newTask.status === "completed" - ) { - // Task just completed - show success toast - toast.success("Task completed successfully", { - description: `Task ${newTask.task_id} has finished processing.`, - action: { - label: "View", - onClick: () => console.log("View task", newTask.task_id), - }, - }); - refetchSearch(); - // Dispatch knowledge updated event for all knowledge-related pages - console.log( - "Task completed successfully, dispatching knowledgeUpdated event", - ); - window.dispatchEvent(new CustomEvent("knowledgeUpdated")); - - // Remove files for this completed task from the files list - setFiles((prevFiles) => - prevFiles.filter((file) => file.task_id !== newTask.task_id), - ); - } else if ( - oldTask && - oldTask.status !== "failed" && - oldTask.status !== "error" && - (newTask.status === "failed" || newTask.status === "error") - ) { - // Task just failed - show error toast - toast.error("Task failed", { - description: `Task ${newTask.task_id} failed: ${ - newTask.error || "Unknown error" - }`, - }); - - // Files will be updated to failed status by the file parsing logic above - } - }); - } - - return newTasks; - }); - } - } catch (error) { - console.error("Failed to fetch tasks:", error); - } finally { - setIsFetching(false); + // Handle task status changes and file updates + useEffect(() => { + if (tasks.length === 0) { + // Store current tasks as previous for next comparison + previousTasksRef.current = tasks; + return; } - }, [isAuthenticated, isNoAuthMode, refetchSearch]); // Removed 'tasks' from dependencies to prevent infinite loop! + console.log(tasks, previousTasksRef.current); - const addTask = useCallback((taskId: string) => { - // Immediately start aggressive polling for the new task - let pollAttempts = 0; - const maxPollAttempts = 30; // Poll for up to 30 seconds + // Check for task status changes by comparing with previous tasks + tasks.forEach((currentTask) => { + const previousTask = previousTasksRef.current.find( + (prev) => prev.task_id === currentTask.task_id, + ); - const aggressivePoll = async () => { - try { - const response = await fetch("/api/tasks"); - if (response.ok) { - const data = await response.json(); - const newTasks = data.tasks || []; - const foundTask = newTasks.find( - (task: Task) => task.task_id === taskId, - ); + // Only show toasts if we have previous data and status has changed + if (((previousTask && previousTask.status !== currentTask.status) || (!previousTask && previousTasksRef.current.length !== 0))) { + console.log("task status changed", currentTask.status); + // Process files from failed task and add them to files list + if (currentTask.files && typeof currentTask.files === "object") { + console.log("processing files", currentTask.files); + const taskFileEntries = Object.entries(currentTask.files); + const now = new Date().toISOString(); - if (foundTask) { - // Task found! Update the tasks state - setTasks((prevTasks) => { - // Check if task is already in the list - const exists = prevTasks.some((t) => t.task_id === taskId); - if (!exists) { - return [...prevTasks, foundTask]; + taskFileEntries.forEach(([filePath, fileInfo]) => { + if (typeof fileInfo === "object" && fileInfo) { + const fileName = filePath.split("/").pop() || filePath; + const fileStatus = fileInfo.status as string; + + // Map backend file status to our TaskFile status + let mappedStatus: TaskFile["status"]; + switch (fileStatus) { + case "pending": + case "running": + mappedStatus = "processing"; + break; + case "completed": + mappedStatus = "active"; + break; + case "failed": + mappedStatus = "failed"; + break; + default: + mappedStatus = "processing"; } - // Update existing task - return prevTasks.map((t) => - t.task_id === taskId ? foundTask : t, - ); - }); - return; // Stop polling, we found it - } + + setFiles((prevFiles) => { + const existingFileIndex = prevFiles.findIndex( + (f) => + f.source_url === filePath && + f.task_id === currentTask.task_id, + ); + + // Detect connector type based on file path or other indicators + let connectorType = "local"; + if (filePath.includes("/") && !filePath.startsWith("/")) { + // Likely S3 key format (bucket/path/file.ext) + connectorType = "s3"; + } + + const fileEntry: TaskFile = { + filename: fileName, + mimetype: "", // We don't have this info from the task + source_url: filePath, + size: 0, // We don't have this info from the task + connector_type: connectorType, + status: mappedStatus, + task_id: currentTask.task_id, + created_at: + typeof fileInfo.created_at === "string" + ? fileInfo.created_at + : now, + updated_at: + typeof fileInfo.updated_at === "string" + ? fileInfo.updated_at + : now, + }; + + if (existingFileIndex >= 0) { + // Update existing file + const updatedFiles = [...prevFiles]; + updatedFiles[existingFileIndex] = fileEntry; + return updatedFiles; + } else { + // Add new file + return [...prevFiles, fileEntry]; + } + }); + } + }); + } + if ( + previousTask && previousTask.status !== "completed" && + currentTask.status === "completed" + ) { + // Task just completed - show success toast + toast.success("Task completed successfully", { + description: `Task ${currentTask.task_id} has finished processing.`, + action: { + label: "View", + onClick: () => console.log("View task", currentTask.task_id), + }, + }); + refetchSearch(); + // Remove files for this completed task from the files list + // setFiles((prevFiles) => + // prevFiles.filter((file) => file.task_id !== currentTask.task_id), + // ); + } else if ( + previousTask && previousTask.status !== "failed" && + previousTask.status !== "error" && + (currentTask.status === "failed" || currentTask.status === "error") + ) { + // Task just failed - show error toast + toast.error("Task failed", { + description: `Task ${currentTask.task_id} failed: ${ + currentTask.error || "Unknown error" + }`, + }); } - } catch (error) { - console.error("Aggressive polling failed:", error); } + }); - pollAttempts++; - if (pollAttempts < maxPollAttempts) { - // Continue polling every 1 second for new tasks - setTimeout(aggressivePoll, 1000); - } - }; + // Store current tasks as previous for next comparison + previousTasksRef.current = tasks; + }, [tasks, refetchSearch]); - // Start aggressive polling after a short delay to allow backend to process - setTimeout(aggressivePoll, 500); - }, []); + const addTask = useCallback( + (_taskId: string) => { + // React Query will automatically handle polling when tasks are active + // Just trigger a refetch to get the latest data + refetchTasks(); + }, + [refetchTasks], + ); const refreshTasks = useCallback(async () => { - await fetchTasks(); - }, [fetchTasks]); + await refetchTasks(); + }, [refetchTasks]); - const removeTask = useCallback((taskId: string) => { - setTasks((prev) => prev.filter((task) => task.task_id !== taskId)); + const removeTask = useCallback((_taskId: string) => { + // This is now handled by React Query automatically + // Tasks will be removed from the list when they're no longer returned by the API }, []); const cancelTask = useCallback( async (taskId: string) => { - try { - const response = await fetch(`/api/tasks/${taskId}/cancel`, { - method: "POST", - }); - - if (response.ok) { - // Immediately refresh tasks to show the updated status - await fetchTasks(); - toast.success("Task cancelled", { - description: `Task ${taskId.substring(0, 8)}... has been cancelled`, - }); - } else { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.error || "Failed to cancel task"); - } - } catch (error) { - console.error("Failed to cancel task:", error); - toast.error("Failed to cancel task", { - description: error instanceof Error ? error.message : "Unknown error", - }); - } + cancelTaskMutation.mutate({ taskId }); }, - [fetchTasks], + [cancelTaskMutation], ); const toggleMenu = useCallback(() => { setIsMenuOpen((prev) => !prev); }, []); - // Periodic polling for task updates - useEffect(() => { - if (!isAuthenticated && !isNoAuthMode) return; - - setIsPolling(true); - - // Initial fetch - fetchTasks(); - - // Set up polling interval - every 3 seconds (more responsive for active tasks) - const interval = setInterval(fetchTasks, 3000); - - return () => { - clearInterval(interval); - setIsPolling(false); - }; - }, [isAuthenticated, isNoAuthMode, fetchTasks]); + // Determine if we're polling based on React Query's refetch interval + const isPolling = + isFetching && + tasks.some( + (task) => + task.status === "pending" || + task.status === "running" || + task.status === "processing", + ); const value: TaskContextType = { tasks, @@ -358,6 +287,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { isFetching, isMenuOpen, toggleMenu, + isLoading, + error, }; return {children}; From db715c122bf42dfd9a049607eda247fd5f18ff25 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 18:00:09 -0300 Subject: [PATCH 08/55] made failed and processing files not accessible and deletable --- frontend/src/app/knowledge/page.tsx | 604 ++++++++++++++-------------- 1 file changed, 307 insertions(+), 297 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 1b8b60ef..41485289 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -17,270 +17,280 @@ import "@/components/AgGrid/registerAgGridModules"; import "@/components/AgGrid/agGridStyles.css"; import { toast } from "sonner"; import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdown"; +import { filterAccentClasses } from "@/components/knowledge-filter-panel"; import { StatusBadge } from "@/components/ui/status-badge"; import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog"; import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; -import { filterAccentClasses } from "@/components/knowledge-filter-panel"; // Function to get the appropriate icon for a connector type function getSourceIcon(connectorType?: string) { - switch (connectorType) { - case "google_drive": - return ( - - ); - case "onedrive": - return ( - - ); - case "sharepoint": - return ; - case "s3": - return ; - default: - return ( - - ); - } + switch (connectorType) { + case "google_drive": + return ( + + ); + case "onedrive": + return ( + + ); + case "sharepoint": + return ; + case "s3": + return ; + default: + return ( + + ); + } } function SearchPage() { - const router = useRouter(); - const { isMenuOpen, files: taskFiles } = useTask(); - const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = - useKnowledgeFilter(); - const [selectedRows, setSelectedRows] = useState([]); - const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); + const router = useRouter(); + const { isMenuOpen, files: taskFiles } = useTask(); + const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = + useKnowledgeFilter(); + const [selectedRows, setSelectedRows] = useState([]); + const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); - const deleteDocumentMutation = useDeleteDocument(); + const deleteDocumentMutation = useDeleteDocument(); - const { data = [], isFetching } = useGetSearchQuery( - parsedFilterData?.query || "*", - parsedFilterData - ); + const { data = [], isFetching } = useGetSearchQuery( + parsedFilterData?.query || "*", + parsedFilterData, + ); - const handleTableSearch = (e: ChangeEvent) => { - gridRef.current?.api.setGridOption("quickFilterText", e.target.value); - }; + const handleTableSearch = (e: ChangeEvent) => { + gridRef.current?.api.setGridOption("quickFilterText", e.target.value); + }; - // Convert TaskFiles to File format and merge with backend results - const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { - return { - filename: taskFile.filename, - mimetype: taskFile.mimetype, - source_url: taskFile.source_url, - size: taskFile.size, - connector_type: taskFile.connector_type, - status: taskFile.status, - }; - }); + // Convert TaskFiles to File format and merge with backend results + const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { + return { + filename: taskFile.filename, + mimetype: taskFile.mimetype, + source_url: taskFile.source_url, + size: taskFile.size, + connector_type: taskFile.connector_type, + status: taskFile.status, + }; + }); - const backendFiles = data as File[]; + const backendFiles = data as File[]; - const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { - return ( - taskFile.status !== "active" && - !backendFiles.some( - (backendFile) => backendFile.filename === taskFile.filename - ) - ); - }); + const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { + return ( + taskFile.status !== "active" && + !backendFiles.some( + (backendFile) => backendFile.filename === taskFile.filename, + ) + ); + }); - // Combine task files first, then backend files - const fileResults = [...backendFiles, ...filteredTaskFiles]; + // Combine task files first, then backend files + const fileResults = [...backendFiles, ...filteredTaskFiles]; - const gridRef = useRef(null); + const gridRef = useRef(null); - const [columnDefs] = useState[]>([ - { - field: "filename", - headerName: "Source", - checkboxSelection: true, - headerCheckboxSelection: true, - initialFlex: 2, - minWidth: 220, - cellRenderer: ({ data, value }: CustomCellRendererProps) => { - return ( - - ); - }, - }, - { - field: "size", - headerName: "Size", - valueFormatter: (params) => - params.value ? `${Math.round(params.value / 1024)} KB` : "-", - }, - { - field: "mimetype", - headerName: "Type", - }, - { - field: "owner", - headerName: "Owner", - valueFormatter: (params) => - params.data?.owner_name || params.data?.owner_email || "—", - }, - { - field: "chunkCount", - headerName: "Chunks", - valueFormatter: (params) => params.data?.chunkCount?.toString() || "-", - }, - { - field: "avgScore", - headerName: "Avg score", - initialFlex: 0.5, - cellRenderer: ({ value }: CustomCellRendererProps) => { - return ( - - {value?.toFixed(2) ?? "-"} - - ); - }, - }, - { - field: "status", - headerName: "Status", - cellRenderer: ({ data }: CustomCellRendererProps) => { - // Default to 'active' status if no status is provided - const status = data?.status || "active"; - return ; - }, - }, - { - cellRenderer: ({ data }: CustomCellRendererProps) => { - return ; - }, - cellStyle: { - alignItems: "center", - display: "flex", - justifyContent: "center", - padding: 0, - }, - colId: "actions", - filter: false, - minWidth: 0, - width: 40, - resizable: false, - sortable: false, - initialFlex: 0, - }, - ]); + const [columnDefs] = useState[]>([ + { + field: "filename", + headerName: "Source", + checkboxSelection: (data) => (data?.data?.status || "active") === "active", + headerCheckboxSelection: true, + initialFlex: 2, + minWidth: 220, + cellRenderer: ({ data, value }: CustomCellRendererProps) => { + return ( +
{((data?.status || "active") !== "active") && +
+ } +
+ ); + }, + }, + { + field: "size", + headerName: "Size", + valueFormatter: (params) => + params.value ? `${Math.round(params.value / 1024)} KB` : "-", + }, + { + field: "mimetype", + headerName: "Type", + }, + { + field: "owner", + headerName: "Owner", + valueFormatter: (params) => + params.data?.owner_name || params.data?.owner_email || "—", + }, + { + field: "chunkCount", + headerName: "Chunks", + valueFormatter: (params) => params.data?.chunkCount?.toString() || "-", + }, + { + field: "avgScore", + headerName: "Avg score", + initialFlex: 0.5, + cellRenderer: ({ value }: CustomCellRendererProps) => { + return ( + + {value?.toFixed(2) ?? "-"} + + ); + }, + }, + { + field: "status", + headerName: "Status", + cellRenderer: ({ data }: CustomCellRendererProps) => { + // Default to 'active' status if no status is provided + const status = data?.status || "active"; + return ; + }, + }, + { + cellRenderer: ({ data }: CustomCellRendererProps) => { + const status = data?.status || "active"; + if (status !== "active") { + return null; + } + return ; + }, + cellStyle: { + alignItems: "center", + display: "flex", + justifyContent: "center", + padding: 0, + }, + colId: "actions", + filter: false, + minWidth: 0, + width: 40, + resizable: false, + sortable: false, + initialFlex: 0, + }, + ]); - const defaultColDef: ColDef = { - resizable: false, - suppressMovable: true, - initialFlex: 1, - minWidth: 100, - }; + const defaultColDef: ColDef = { + resizable: false, + suppressMovable: true, + initialFlex: 1, + minWidth: 100, + }; - const onSelectionChanged = useCallback(() => { - if (gridRef.current) { - const selectedNodes = gridRef.current.api.getSelectedRows(); - setSelectedRows(selectedNodes); - } - }, []); + const onSelectionChanged = useCallback(() => { + if (gridRef.current) { + const selectedNodes = gridRef.current.api.getSelectedRows(); + setSelectedRows(selectedNodes); + } + }, []); - const handleBulkDelete = async () => { - if (selectedRows.length === 0) return; + const handleBulkDelete = async () => { + if (selectedRows.length === 0) return; - try { - // Delete each file individually since the API expects one filename at a time - const deletePromises = selectedRows.map((row) => - deleteDocumentMutation.mutateAsync({ filename: row.filename }) - ); + try { + // Delete each file individually since the API expects one filename at a time + const deletePromises = selectedRows.map((row) => + deleteDocumentMutation.mutateAsync({ filename: row.filename }), + ); - await Promise.all(deletePromises); + await Promise.all(deletePromises); - toast.success( - `Successfully deleted ${selectedRows.length} document${ - selectedRows.length > 1 ? "s" : "" - }` - ); - setSelectedRows([]); - setShowBulkDeleteDialog(false); + toast.success( + `Successfully deleted ${selectedRows.length} document${ + selectedRows.length > 1 ? "s" : "" + }`, + ); + setSelectedRows([]); + setShowBulkDeleteDialog(false); - // Clear selection in the grid - if (gridRef.current) { - gridRef.current.api.deselectAll(); - } - } catch (error) { - toast.error( - error instanceof Error - ? error.message - : "Failed to delete some documents" - ); - } - }; + // Clear selection in the grid + if (gridRef.current) { + gridRef.current.api.deselectAll(); + } + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Failed to delete some documents", + ); + } + }; - return ( -
-
-
-

Project Knowledge

- -
+ return ( +
+
+
+

Project Knowledge

+ +
- {/* Search Input Area */} -
-
-
- {selectedFilter?.name && ( -
- {selectedFilter?.name} - setSelectedFilter(null)} - /> -
- )} - - -
- {/* */} - {/* //TODO: Implement sync button */} - {/* */} - {selectedRows.length > 0 && ( - - )} -
-
- params.data.filename} - domLayout="normal" - onSelectionChanged={onSelectionChanged} - noRowsOverlayComponent={() => ( -
-
- No knowledge -
-
- Add files from local or your preferred cloud. -
-
- )} - /> -
+ {selectedRows.length > 0 && ( + + )} + +
+ params.data.filename} + domLayout="normal" + onSelectionChanged={onSelectionChanged} + noRowsOverlayComponent={() => ( +
+
+ No knowledge +
+
+ Add files from local or your preferred cloud. +
+
+ )} + /> +
- {/* Bulk Delete Confirmation Dialog */} - 1 ? "s" : "" - }? This will remove all chunks and data associated with these documents. This action cannot be undone. + {/* Bulk Delete Confirmation Dialog */} + 1 ? "s" : "" + }? This will remove all chunks and data associated with these documents. This action cannot be undone. Documents to be deleted: ${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`} - confirmText="Delete All" - onConfirm={handleBulkDelete} - isLoading={deleteDocumentMutation.isPending} - /> -
- ); + confirmText="Delete All" + onConfirm={handleBulkDelete} + isLoading={deleteDocumentMutation.isPending} + /> +
+ ); } export default function ProtectedSearchPage() { - return ( - - - - ); + return ( + + + + ); } From 581879f5f6ffc5807096f8d93c481e13c4787ee3 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 18:04:45 -0300 Subject: [PATCH 09/55] fixed animated processing icon --- .../ui/animated-processing-icon.tsx | 52 +++++++------------ frontend/src/components/ui/status-badge.tsx | 2 +- 2 files changed, 21 insertions(+), 33 deletions(-) diff --git a/frontend/src/components/ui/animated-processing-icon.tsx b/frontend/src/components/ui/animated-processing-icon.tsx index eb36b2ab..51815414 100644 --- a/frontend/src/components/ui/animated-processing-icon.tsx +++ b/frontend/src/components/ui/animated-processing-icon.tsx @@ -1,26 +1,16 @@ -interface AnimatedProcessingIconProps { - className?: string; - size?: number; -} +import type { SVGProps } from "react"; -export const AnimatedProcessingIcon = ({ - className = "", - size = 10, -}: AnimatedProcessingIconProps) => { - const width = Math.round((size * 6) / 10); - const height = size; - - return ( - - + + + + + + + ); }; diff --git a/frontend/src/components/ui/status-badge.tsx b/frontend/src/components/ui/status-badge.tsx index d3b1a323..07dce58b 100644 --- a/frontend/src/components/ui/status-badge.tsx +++ b/frontend/src/components/ui/status-badge.tsx @@ -50,7 +50,7 @@ export const StatusBadge = ({ status, className }: StatusBadgeProps) => { }`} > {status === "processing" && ( - + )} {config.label}
From 7d6feef9992b6bfeaf78db085a961dd5c4e72c4e Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 18:09:02 -0300 Subject: [PATCH 10/55] fixed status badge size --- frontend/src/components/ui/status-badge.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/ui/status-badge.tsx b/frontend/src/components/ui/status-badge.tsx index 07dce58b..e57ad3b5 100644 --- a/frontend/src/components/ui/status-badge.tsx +++ b/frontend/src/components/ui/status-badge.tsx @@ -50,7 +50,7 @@ export const StatusBadge = ({ status, className }: StatusBadgeProps) => { }`} > {status === "processing" && ( - + )} {config.label}
From 179a7403ccaec179e2d0cad3e88c29a3deca08f1 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Thu, 2 Oct 2025 18:09:18 -0300 Subject: [PATCH 11/55] fixed router, langflow files and processors to not use temp names --- src/api/langflow_files.py | 15 ++++++++------- src/api/router.py | 18 ++++++++++-------- src/models/processors.py | 8 ++------ 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/api/langflow_files.py b/src/api/langflow_files.py index 1c83724d..ae36bf04 100644 --- a/src/api/langflow_files.py +++ b/src/api/langflow_files.py @@ -189,19 +189,20 @@ async def upload_and_ingest_user_file( # Create temporary file for task processing import tempfile import os - + # Read file content content = await upload_file.read() - - # Create temporary file + + # Create temporary file with the actual filename (not a temp prefix) + # Store in temp directory but use the real filename + temp_dir = tempfile.gettempdir() safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_") - temp_fd, temp_path = tempfile.mkstemp( - suffix=f"_{safe_filename}" - ) + temp_path = os.path.join(temp_dir, safe_filename) + try: # Write content to temp file - with os.fdopen(temp_fd, 'wb') as temp_file: + with open(temp_path, 'wb') as temp_file: temp_file.write(content) logger.debug("Created temporary file for task processing", temp_path=temp_path) diff --git a/src/api/router.py b/src/api/router.py index 56789d41..857914c0 100644 --- a/src/api/router.py +++ b/src/api/router.py @@ -114,20 +114,22 @@ async def langflow_upload_ingest_task( temp_file_paths = [] try: + # Create temp directory reference once + temp_dir = tempfile.gettempdir() + for upload_file in upload_files: # Read file content content = await upload_file.read() - - # Create temporary file + + # Create temporary file with the actual filename (not a temp prefix) + # Store in temp directory but use the real filename safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_") - temp_fd, temp_path = tempfile.mkstemp( - suffix=f"_{safe_filename}" - ) - + temp_path = os.path.join(temp_dir, safe_filename) + # Write content to temp file - with os.fdopen(temp_fd, 'wb') as temp_file: + with open(temp_path, 'wb') as temp_file: temp_file.write(content) - + temp_file_paths.append(temp_path) logger.debug( diff --git a/src/models/processors.py b/src/models/processors.py index ecec9c49..0ce8e33d 100644 --- a/src/models/processors.py +++ b/src/models/processors.py @@ -574,12 +574,8 @@ class LangflowFileProcessor(TaskProcessor): content = f.read() # Create file tuple for upload - temp_filename = os.path.basename(item) - # Extract original filename from temp file suffix (remove tmp prefix) - if "_" in temp_filename: - filename = temp_filename.split("_", 1)[1] # Get everything after first _ - else: - filename = temp_filename + # The temp file now has the actual filename, no need to extract it + filename = os.path.basename(item) content_type, _ = mimetypes.guess_type(filename) if not content_type: content_type = 'application/octet-stream' From 12a72b2cd98ec56b800301cfe81c400464ea2c77 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Thu, 2 Oct 2025 16:30:22 -0600 Subject: [PATCH 12/55] remove unused fields --- frontend/src/app/knowledge/chunks/page.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index cb96eddc..267d0afc 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -298,12 +298,12 @@ function ChunksPageContent() {

Original document

-
+ {/*
Name
{fileData?.filename}
-
+
*/}
Type
@@ -318,23 +318,23 @@ function ChunksPageContent() { : "Unknown"}
-
+ {/*
Uploaded
N/A
-
+
*/} {/* TODO: Uncomment after data is available */} {/*
Source
*/} -
+ {/*
Updated
N/A
-
+
*/}
From 042ffb421fc55c9500530646182887f85834c073 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Thu, 2 Oct 2025 16:41:59 -0600 Subject: [PATCH 13/55] fix search --- frontend/src/app/knowledge/chunks/page.tsx | 47 +++++++++++++++------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index 267d0afc..bc00f886 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { ArrowLeft, Check, Copy, Loader2, Search } from "lucide-react"; +import { ArrowLeft, Check, Copy, Loader2, Search, X } from "lucide-react"; import { Suspense, useCallback, useEffect, useMemo, useState } from "react"; import { useRouter, useSearchParams } from "next/navigation"; import { ProtectedRoute } from "@/components/protected-route"; @@ -14,7 +14,7 @@ import { } from "../../api/queries/useGetSearchQuery"; import { Label } from "@/components/ui/label"; import { Checkbox } from "@/components/ui/checkbox"; -import { Input } from "@/components/ui/input"; +import { filterAccentClasses } from "@/components/knowledge-filter-panel"; const getFileTypeLabel = (mimetype: string) => { if (mimetype === "application/pdf") return "PDF"; @@ -26,8 +26,9 @@ const getFileTypeLabel = (mimetype: string) => { function ChunksPageContent() { const router = useRouter(); const searchParams = useSearchParams(); + const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = + useKnowledgeFilter(); const { isMenuOpen } = useTask(); - const { parsedFilterData, isPanelOpen } = useKnowledgeFilter(); const filename = searchParams.get("filename"); const [chunks, setChunks] = useState([]); @@ -158,17 +159,35 @@ function ChunksPageContent() {
-
- : null} - id="search-query" - type="text" - defaultValue={parsedFilterData?.query} - value={queryInputText} - onChange={(e) => setQueryInputText(e.target.value)} - placeholder="Search chunks..." - /> +
+
+ {selectedFilter?.name && ( +
+ {selectedFilter?.name} + setSelectedFilter(null)} + /> +
+ )} + + setQueryInputText(e.target.value)} + /> +
Date: Fri, 3 Oct 2025 10:11:39 -0300 Subject: [PATCH 14/55] fixed issues with already ingested files, fixed state issues --- frontend/src/app/knowledge/page.tsx | 11 +++++------ frontend/src/contexts/task-context.tsx | 9 +++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 41485289..c6d254c4 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -75,14 +75,13 @@ function SearchPage() { }; }); - const backendFiles = data as File[]; + const backendFiles = (data as File[]).filter((file) => !taskFilesAsFiles.some((taskFile) => taskFile.filename === file.filename && taskFile.status === "processing")); const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { return ( taskFile.status !== "active" && !backendFiles.some( - (backendFile) => backendFile.filename === taskFile.filename, - ) + (backendFile) => backendFile.filename === taskFile.filename,) ); }); @@ -91,7 +90,7 @@ function SearchPage() { const gridRef = useRef(null); - const [columnDefs] = useState[]>([ + const columnDefs = [ { field: "filename", headerName: "Source", @@ -189,8 +188,8 @@ function SearchPage() { resizable: false, sortable: false, initialFlex: 0, - }, - ]); + } + ]; const defaultColDef: ColDef = { resizable: false, diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index b3275422..8928ae8f 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -212,11 +212,12 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { onClick: () => console.log("View task", currentTask.task_id), }, }); + setTimeout(() => { refetchSearch(); - // Remove files for this completed task from the files list - // setFiles((prevFiles) => - // prevFiles.filter((file) => file.task_id !== currentTask.task_id), - // ); + setFiles((prevFiles) => + prevFiles.filter((file) => file.task_id !== currentTask.task_id && file.status !== "failed"), + ); + }, 500); } else if ( previousTask && previousTask.status !== "failed" && previousTask.status !== "error" && From 30f63a76b65af3b2011a4f83817e57528a37f3c7 Mon Sep 17 00:00:00 2001 From: Edwin Jose Date: Fri, 3 Oct 2025 11:02:45 -0400 Subject: [PATCH 15/55] add mcp Flows --- flows/openrag_agent.json | 302 ++++++++++++++- flows/openrag_url_mcp.json | 748 ++++++++++++++++++++++++++++++++++++- 2 files changed, 1031 insertions(+), 19 deletions(-) diff --git a/flows/openrag_agent.json b/flows/openrag_agent.json index 73dccee4..9aa875d1 100644 --- a/flows/openrag_agent.json +++ b/flows/openrag_agent.json @@ -142,6 +142,34 @@ "sourceHandle": "{œdataTypeœ:œLanguageModelComponentœ,œidœ:œLanguageModelComponent-0YME7œ,œnameœ:œmodel_outputœ,œoutput_typesœ:[œLanguageModelœ]}", "target": "Agent-crjWf", "targetHandle": "{œfieldNameœ:œagent_llmœ,œidœ:œAgent-crjWfœ,œinputTypesœ:[œLanguageModelœ],œtypeœ:œstrœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "MCP", + "id": "MCP-Uk5Vf", + "name": "component_as_tool", + "output_types": [ + "Tool" + ] + }, + "targetHandle": { + "fieldName": "tools", + "id": "Agent-crjWf", + "inputTypes": [ + "Tool" + ], + "type": "other" + } + }, + "id": "xy-edge__MCP-Uk5Vf{œdataTypeœ:œMCPœ,œidœ:œMCP-Uk5Vfœ,œnameœ:œcomponent_as_toolœ,œoutput_typesœ:[œToolœ]}-Agent-crjWf{œfieldNameœ:œtoolsœ,œidœ:œAgent-crjWfœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}", + "selected": false, + "source": "MCP-Uk5Vf", + "sourceHandle": "{œdataTypeœ:œMCPœ,œidœ:œMCP-Uk5Vfœ,œnameœ:œcomponent_as_toolœ,œoutput_typesœ:[œToolœ]}", + "target": "Agent-crjWf", + "targetHandle": "{œfieldNameœ:œtoolsœ,œidœ:œAgent-crjWfœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}" } ], "nodes": [ @@ -702,7 +730,7 @@ ], "frozen": false, "icon": "OpenSearch", - "last_updated": "2025-10-01T15:31:22.241Z", + "last_updated": "2025-10-03T14:42:51.280Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -1356,7 +1384,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-10-01T15:31:22.243Z", + "last_updated": "2025-10-03T14:42:51.281Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -1681,7 +1709,7 @@ ], "frozen": false, "icon": "bot", - "last_updated": "2025-10-01T15:31:45.879Z", + "last_updated": "2025-10-03T14:42:51.349Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -2220,7 +2248,7 @@ ], "frozen": false, "icon": "brain-circuit", - "last_updated": "2025-10-01T15:31:22.244Z", + "last_updated": "2025-10-03T14:42:51.281Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -2523,12 +2551,272 @@ }, "selected": false, "type": "genericNode" + }, + { + "data": { + "id": "MCP-Uk5Vf", + "node": { + "base_classes": [ + "DataFrame" + ], + "beta": false, + "category": "MCP", + "conditional_paths": [], + "custom_fields": {}, + "description": "Connect to an MCP server to use its tools.", + "display_name": "MCP Tools", + "documentation": "https://docs.langflow.org/mcp-client", + "edited": false, + "field_order": [ + "mcp_server", + "use_cache", + "tool", + "tool_placeholder" + ], + "frozen": false, + "icon": "Mcp", + "key": "mcp_lf-starter_project", + "last_updated": "2025-10-03T14:42:51.282Z", + "legacy": false, + "mcpServerName": "lf-starter_project", + "metadata": { + "code_hash": "756d1e10d0ca", + "dependencies": { + "dependencies": [ + { + "name": "langchain_core", + "version": "0.3.76" + }, + { + "name": "lfx", + "version": null + }, + { + "name": "langflow", + "version": null + } + ], + "total_dependencies": 3 + }, + "module": "lfx.components.agents.mcp_component.MCPToolsComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Toolset", + "group_outputs": false, + "hidden": null, + "method": "to_toolkit", + "name": "component_as_tool", + "options": null, + "required_inputs": null, + "selected": "Tool", + "tool_mode": true, + "types": [ + "Tool" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from __future__ import annotations\n\nimport asyncio\nimport uuid\nfrom typing import Any\n\nfrom langchain_core.tools import StructuredTool # noqa: TC002\n\nfrom lfx.base.agents.utils import maybe_unflatten_dict, safe_cache_get, safe_cache_set\nfrom lfx.base.mcp.util import MCPSseClient, MCPStdioClient, create_input_schema_from_json_schema, update_tools\nfrom lfx.custom.custom_component.component_with_cache import ComponentWithCache\nfrom lfx.inputs.inputs import InputTypes # noqa: TC001\nfrom lfx.io import BoolInput, DropdownInput, McpInput, MessageTextInput, Output\nfrom lfx.io.schema import flatten_schema, schema_to_langflow_inputs\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.services.deps import get_settings_service, get_storage_service, session_scope\n\n\nclass MCPToolsComponent(ComponentWithCache):\n schema_inputs: list = []\n tools: list[StructuredTool] = []\n _not_load_actions: bool = False\n _tool_cache: dict = {}\n _last_selected_server: str | None = None # Cache for the last selected server\n\n def __init__(self, **data) -> None:\n super().__init__(**data)\n # Initialize cache keys to avoid CacheMiss when accessing them\n self._ensure_cache_structure()\n\n # Initialize clients with access to the component cache\n self.stdio_client: MCPStdioClient = MCPStdioClient(component_cache=self._shared_component_cache)\n self.sse_client: MCPSseClient = MCPSseClient(component_cache=self._shared_component_cache)\n\n def _ensure_cache_structure(self):\n \"\"\"Ensure the cache has the required structure.\"\"\"\n # Check if servers key exists and is not CacheMiss\n servers_value = safe_cache_get(self._shared_component_cache, \"servers\")\n if servers_value is None:\n safe_cache_set(self._shared_component_cache, \"servers\", {})\n\n # Check if last_selected_server key exists and is not CacheMiss\n last_server_value = safe_cache_get(self._shared_component_cache, \"last_selected_server\")\n if last_server_value is None:\n safe_cache_set(self._shared_component_cache, \"last_selected_server\", \"\")\n\n default_keys: list[str] = [\n \"code\",\n \"_type\",\n \"tool_mode\",\n \"tool_placeholder\",\n \"mcp_server\",\n \"tool\",\n \"use_cache\",\n ]\n\n display_name = \"MCP Tools\"\n description = \"Connect to an MCP server to use its tools.\"\n documentation: str = \"https://docs.langflow.org/mcp-client\"\n icon = \"Mcp\"\n name = \"MCPTools\"\n\n inputs = [\n McpInput(\n name=\"mcp_server\",\n display_name=\"MCP Server\",\n info=\"Select the MCP Server that will be used by this component\",\n real_time_refresh=True,\n ),\n BoolInput(\n name=\"use_cache\",\n display_name=\"Use Cached Server\",\n info=(\n \"Enable caching of MCP Server and tools to improve performance. \"\n \"Disable to always fetch fresh tools and server updates.\"\n ),\n value=False,\n advanced=True,\n ),\n DropdownInput(\n name=\"tool\",\n display_name=\"Tool\",\n options=[],\n value=\"\",\n info=\"Select the tool to execute\",\n show=False,\n required=True,\n real_time_refresh=True,\n ),\n MessageTextInput(\n name=\"tool_placeholder\",\n display_name=\"Tool Placeholder\",\n info=\"Placeholder for the tool\",\n value=\"\",\n show=False,\n tool_mode=False,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Response\", name=\"response\", method=\"build_output\"),\n ]\n\n async def _validate_schema_inputs(self, tool_obj) -> list[InputTypes]:\n \"\"\"Validate and process schema inputs for a tool.\"\"\"\n try:\n if not tool_obj or not hasattr(tool_obj, \"args_schema\"):\n msg = \"Invalid tool object or missing input schema\"\n raise ValueError(msg)\n\n flat_schema = flatten_schema(tool_obj.args_schema.schema())\n input_schema = create_input_schema_from_json_schema(flat_schema)\n if not input_schema:\n msg = f\"Empty input schema for tool '{tool_obj.name}'\"\n raise ValueError(msg)\n\n schema_inputs = schema_to_langflow_inputs(input_schema)\n if not schema_inputs:\n msg = f\"No input parameters defined for tool '{tool_obj.name}'\"\n await logger.awarning(msg)\n return []\n\n except Exception as e:\n msg = f\"Error validating schema inputs: {e!s}\"\n await logger.aexception(msg)\n raise ValueError(msg) from e\n else:\n return schema_inputs\n\n async def update_tool_list(self, mcp_server_value=None):\n # Accepts mcp_server_value as dict {name, config} or uses self.mcp_server\n mcp_server = mcp_server_value if mcp_server_value is not None else getattr(self, \"mcp_server\", None)\n server_name = None\n server_config_from_value = None\n if isinstance(mcp_server, dict):\n server_name = mcp_server.get(\"name\")\n server_config_from_value = mcp_server.get(\"config\")\n else:\n server_name = mcp_server\n if not server_name:\n self.tools = []\n return [], {\"name\": server_name, \"config\": server_config_from_value}\n\n # Check if caching is enabled, default to False\n use_cache = getattr(self, \"use_cache\", False)\n\n # Use shared cache if available and caching is enabled\n cached = None\n if use_cache:\n servers_cache = safe_cache_get(self._shared_component_cache, \"servers\", {})\n cached = servers_cache.get(server_name) if isinstance(servers_cache, dict) else None\n\n if cached is not None:\n try:\n self.tools = cached[\"tools\"]\n self.tool_names = cached[\"tool_names\"]\n self._tool_cache = cached[\"tool_cache\"]\n server_config_from_value = cached[\"config\"]\n except (TypeError, KeyError, AttributeError) as e:\n # Handle corrupted cache data by clearing it and continuing to fetch fresh tools\n msg = f\"Unable to use cached data for MCP Server{server_name}: {e}\"\n await logger.awarning(msg)\n # Clear the corrupted cache entry\n current_servers_cache = safe_cache_get(self._shared_component_cache, \"servers\", {})\n if isinstance(current_servers_cache, dict) and server_name in current_servers_cache:\n current_servers_cache.pop(server_name)\n safe_cache_set(self._shared_component_cache, \"servers\", current_servers_cache)\n else:\n return self.tools, {\"name\": server_name, \"config\": server_config_from_value}\n\n try:\n try:\n from langflow.api.v2.mcp import get_server\n from langflow.services.database.models.user.crud import get_user_by_id\n except ImportError as e:\n msg = (\n \"Langflow MCP server functionality is not available. \"\n \"This feature requires the full Langflow installation.\"\n )\n raise ImportError(msg) from e\n async with session_scope() as db:\n if not self.user_id:\n msg = \"User ID is required for fetching MCP tools.\"\n raise ValueError(msg)\n current_user = await get_user_by_id(db, self.user_id)\n\n # Try to get server config from DB/API\n server_config = await get_server(\n server_name,\n current_user,\n db,\n storage_service=get_storage_service(),\n settings_service=get_settings_service(),\n )\n\n # If get_server returns empty but we have a config, use it\n if not server_config and server_config_from_value:\n server_config = server_config_from_value\n\n if not server_config:\n self.tools = []\n return [], {\"name\": server_name, \"config\": server_config}\n\n _, tool_list, tool_cache = await update_tools(\n server_name=server_name,\n server_config=server_config,\n mcp_stdio_client=self.stdio_client,\n mcp_sse_client=self.sse_client,\n )\n\n self.tool_names = [tool.name for tool in tool_list if hasattr(tool, \"name\")]\n self._tool_cache = tool_cache\n self.tools = tool_list\n\n # Cache the result only if caching is enabled\n if use_cache:\n cache_data = {\n \"tools\": tool_list,\n \"tool_names\": self.tool_names,\n \"tool_cache\": tool_cache,\n \"config\": server_config,\n }\n\n # Safely update the servers cache\n current_servers_cache = safe_cache_get(self._shared_component_cache, \"servers\", {})\n if isinstance(current_servers_cache, dict):\n current_servers_cache[server_name] = cache_data\n safe_cache_set(self._shared_component_cache, \"servers\", current_servers_cache)\n\n except (TimeoutError, asyncio.TimeoutError) as e:\n msg = f\"Timeout updating tool list: {e!s}\"\n await logger.aexception(msg)\n raise TimeoutError(msg) from e\n except Exception as e:\n msg = f\"Error updating tool list: {e!s}\"\n await logger.aexception(msg)\n raise ValueError(msg) from e\n else:\n return tool_list, {\"name\": server_name, \"config\": server_config}\n\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Toggle the visibility of connection-specific fields based on the selected mode.\"\"\"\n try:\n if field_name == \"tool\":\n try:\n if len(self.tools) == 0:\n try:\n self.tools, build_config[\"mcp_server\"][\"value\"] = await self.update_tool_list()\n build_config[\"tool\"][\"options\"] = [tool.name for tool in self.tools]\n build_config[\"tool\"][\"placeholder\"] = \"Select a tool\"\n except (TimeoutError, asyncio.TimeoutError) as e:\n msg = f\"Timeout updating tool list: {e!s}\"\n await logger.aexception(msg)\n if not build_config[\"tools_metadata\"][\"show\"]:\n build_config[\"tool\"][\"show\"] = True\n build_config[\"tool\"][\"options\"] = []\n build_config[\"tool\"][\"value\"] = \"\"\n build_config[\"tool\"][\"placeholder\"] = \"Timeout on MCP server\"\n else:\n build_config[\"tool\"][\"show\"] = False\n except ValueError:\n if not build_config[\"tools_metadata\"][\"show\"]:\n build_config[\"tool\"][\"show\"] = True\n build_config[\"tool\"][\"options\"] = []\n build_config[\"tool\"][\"value\"] = \"\"\n build_config[\"tool\"][\"placeholder\"] = \"Error on MCP Server\"\n else:\n build_config[\"tool\"][\"show\"] = False\n\n if field_value == \"\":\n return build_config\n tool_obj = None\n for tool in self.tools:\n if tool.name == field_value:\n tool_obj = tool\n break\n if tool_obj is None:\n msg = f\"Tool {field_value} not found in available tools: {self.tools}\"\n await logger.awarning(msg)\n return build_config\n await self._update_tool_config(build_config, field_value)\n except Exception as e:\n build_config[\"tool\"][\"options\"] = []\n msg = f\"Failed to update tools: {e!s}\"\n raise ValueError(msg) from e\n else:\n return build_config\n elif field_name == \"mcp_server\":\n if not field_value:\n build_config[\"tool\"][\"show\"] = False\n build_config[\"tool\"][\"options\"] = []\n build_config[\"tool\"][\"value\"] = \"\"\n build_config[\"tool\"][\"placeholder\"] = \"\"\n build_config[\"tool_placeholder\"][\"tool_mode\"] = False\n self.remove_non_default_keys(build_config)\n return build_config\n\n build_config[\"tool_placeholder\"][\"tool_mode\"] = True\n\n current_server_name = field_value.get(\"name\") if isinstance(field_value, dict) else field_value\n _last_selected_server = safe_cache_get(self._shared_component_cache, \"last_selected_server\", \"\")\n\n # To avoid unnecessary updates, only proceed if the server has actually changed\n if (_last_selected_server in (current_server_name, \"\")) and build_config[\"tool\"][\"show\"]:\n if current_server_name:\n servers_cache = safe_cache_get(self._shared_component_cache, \"servers\", {})\n if isinstance(servers_cache, dict):\n cached = servers_cache.get(current_server_name)\n if cached is not None and cached.get(\"tool_names\"):\n cached_tools = cached[\"tool_names\"]\n current_tools = build_config[\"tool\"][\"options\"]\n if current_tools == cached_tools:\n return build_config\n else:\n return build_config\n\n # Determine if \"Tool Mode\" is active by checking if the tool dropdown is hidden.\n is_in_tool_mode = build_config[\"tools_metadata\"][\"show\"]\n safe_cache_set(self._shared_component_cache, \"last_selected_server\", current_server_name)\n\n # Check if tools are already cached for this server before clearing\n cached_tools = None\n if current_server_name:\n use_cache = getattr(self, \"use_cache\", True)\n if use_cache:\n servers_cache = safe_cache_get(self._shared_component_cache, \"servers\", {})\n if isinstance(servers_cache, dict):\n cached = servers_cache.get(current_server_name)\n if cached is not None:\n try:\n cached_tools = cached[\"tools\"]\n self.tools = cached_tools\n self.tool_names = cached[\"tool_names\"]\n self._tool_cache = cached[\"tool_cache\"]\n except (TypeError, KeyError, AttributeError) as e:\n # Handle corrupted cache data by ignoring it\n msg = f\"Unable to use cached data for MCP Server,{current_server_name}: {e}\"\n await logger.awarning(msg)\n cached_tools = None\n\n # Only clear tools if we don't have cached tools for the current server\n if not cached_tools:\n self.tools = [] # Clear previous tools only if no cache\n\n self.remove_non_default_keys(build_config) # Clear previous tool inputs\n\n # Only show the tool dropdown if not in tool_mode\n if not is_in_tool_mode:\n build_config[\"tool\"][\"show\"] = True\n if cached_tools:\n # Use cached tools to populate options immediately\n build_config[\"tool\"][\"options\"] = [tool.name for tool in cached_tools]\n build_config[\"tool\"][\"placeholder\"] = \"Select a tool\"\n else:\n # Show loading state only when we need to fetch tools\n build_config[\"tool\"][\"placeholder\"] = \"Loading tools...\"\n build_config[\"tool\"][\"options\"] = []\n build_config[\"tool\"][\"value\"] = uuid.uuid4()\n else:\n # Keep the tool dropdown hidden if in tool_mode\n self._not_load_actions = True\n build_config[\"tool\"][\"show\"] = False\n\n elif field_name == \"tool_mode\":\n build_config[\"tool\"][\"placeholder\"] = \"\"\n build_config[\"tool\"][\"show\"] = not bool(field_value) and bool(build_config[\"mcp_server\"])\n self.remove_non_default_keys(build_config)\n self.tool = build_config[\"tool\"][\"value\"]\n if field_value:\n self._not_load_actions = True\n else:\n build_config[\"tool\"][\"value\"] = uuid.uuid4()\n build_config[\"tool\"][\"options\"] = []\n build_config[\"tool\"][\"show\"] = True\n build_config[\"tool\"][\"placeholder\"] = \"Loading tools...\"\n elif field_name == \"tools_metadata\":\n self._not_load_actions = False\n\n except Exception as e:\n msg = f\"Error in update_build_config: {e!s}\"\n await logger.aexception(msg)\n raise ValueError(msg) from e\n else:\n return build_config\n\n def get_inputs_for_all_tools(self, tools: list) -> dict:\n \"\"\"Get input schemas for all tools.\"\"\"\n inputs = {}\n for tool in tools:\n if not tool or not hasattr(tool, \"name\"):\n continue\n try:\n flat_schema = flatten_schema(tool.args_schema.schema())\n input_schema = create_input_schema_from_json_schema(flat_schema)\n langflow_inputs = schema_to_langflow_inputs(input_schema)\n inputs[tool.name] = langflow_inputs\n except (AttributeError, ValueError, TypeError, KeyError) as e:\n msg = f\"Error getting inputs for tool {getattr(tool, 'name', 'unknown')}: {e!s}\"\n logger.exception(msg)\n continue\n return inputs\n\n def remove_input_schema_from_build_config(\n self, build_config: dict, tool_name: str, input_schema: dict[list[InputTypes], Any]\n ):\n \"\"\"Remove the input schema for the tool from the build config.\"\"\"\n # Keep only schemas that don't belong to the current tool\n input_schema = {k: v for k, v in input_schema.items() if k != tool_name}\n # Remove all inputs from other tools\n for value in input_schema.values():\n for _input in value:\n if _input.name in build_config:\n build_config.pop(_input.name)\n\n def remove_non_default_keys(self, build_config: dict) -> None:\n \"\"\"Remove non-default keys from the build config.\"\"\"\n for key in list(build_config.keys()):\n if key not in self.default_keys:\n build_config.pop(key)\n\n async def _update_tool_config(self, build_config: dict, tool_name: str) -> None:\n \"\"\"Update tool configuration with proper error handling.\"\"\"\n if not self.tools:\n self.tools, build_config[\"mcp_server\"][\"value\"] = await self.update_tool_list()\n\n if not tool_name:\n return\n\n tool_obj = next((tool for tool in self.tools if tool.name == tool_name), None)\n if not tool_obj:\n msg = f\"Tool {tool_name} not found in available tools: {self.tools}\"\n self.remove_non_default_keys(build_config)\n build_config[\"tool\"][\"value\"] = \"\"\n await logger.awarning(msg)\n return\n\n try:\n # Store current values before removing inputs\n current_values = {}\n for key, value in build_config.items():\n if key not in self.default_keys and isinstance(value, dict) and \"value\" in value:\n current_values[key] = value[\"value\"]\n\n # Get all tool inputs and remove old ones\n input_schema_for_all_tools = self.get_inputs_for_all_tools(self.tools)\n self.remove_input_schema_from_build_config(build_config, tool_name, input_schema_for_all_tools)\n\n # Get and validate new inputs\n self.schema_inputs = await self._validate_schema_inputs(tool_obj)\n if not self.schema_inputs:\n msg = f\"No input parameters to configure for tool '{tool_name}'\"\n await logger.ainfo(msg)\n return\n\n # Add new inputs to build config\n for schema_input in self.schema_inputs:\n if not schema_input or not hasattr(schema_input, \"name\"):\n msg = \"Invalid schema input detected, skipping\"\n await logger.awarning(msg)\n continue\n\n try:\n name = schema_input.name\n input_dict = schema_input.to_dict()\n input_dict.setdefault(\"value\", None)\n input_dict.setdefault(\"required\", True)\n\n build_config[name] = input_dict\n\n # Preserve existing value if the parameter name exists in current_values\n if name in current_values:\n build_config[name][\"value\"] = current_values[name]\n\n except (AttributeError, KeyError, TypeError) as e:\n msg = f\"Error processing schema input {schema_input}: {e!s}\"\n await logger.aexception(msg)\n continue\n except ValueError as e:\n msg = f\"Schema validation error for tool {tool_name}: {e!s}\"\n await logger.aexception(msg)\n self.schema_inputs = []\n return\n except (AttributeError, KeyError, TypeError) as e:\n msg = f\"Error updating tool config: {e!s}\"\n await logger.aexception(msg)\n raise ValueError(msg) from e\n\n async def build_output(self) -> DataFrame:\n \"\"\"Build output with improved error handling and validation.\"\"\"\n try:\n self.tools, _ = await self.update_tool_list()\n if self.tool != \"\":\n # Set session context for persistent MCP sessions using Langflow session ID\n session_context = self._get_session_context()\n if session_context:\n self.stdio_client.set_session_context(session_context)\n self.sse_client.set_session_context(session_context)\n\n exec_tool = self._tool_cache[self.tool]\n tool_args = self.get_inputs_for_all_tools(self.tools)[self.tool]\n kwargs = {}\n for arg in tool_args:\n value = getattr(self, arg.name, None)\n if value is not None:\n if isinstance(value, Message):\n kwargs[arg.name] = value.text\n else:\n kwargs[arg.name] = value\n\n unflattened_kwargs = maybe_unflatten_dict(kwargs)\n\n output = await exec_tool.coroutine(**unflattened_kwargs)\n\n tool_content = []\n for item in output.content:\n item_dict = item.model_dump()\n tool_content.append(item_dict)\n return DataFrame(data=tool_content)\n return DataFrame(data=[{\"error\": \"You must select a tool\"}])\n except Exception as e:\n msg = f\"Error in build_output: {e!s}\"\n await logger.aexception(msg)\n raise ValueError(msg) from e\n\n def _get_session_context(self) -> str | None:\n \"\"\"Get the Langflow session ID for MCP session caching.\"\"\"\n # Try to get session ID from the component's execution context\n if hasattr(self, \"graph\") and hasattr(self.graph, \"session_id\"):\n session_id = self.graph.session_id\n # Include server name to ensure different servers get different sessions\n server_name = \"\"\n mcp_server = getattr(self, \"mcp_server\", None)\n if isinstance(mcp_server, dict):\n server_name = mcp_server.get(\"name\", \"\")\n elif mcp_server:\n server_name = str(mcp_server)\n return f\"{session_id}_{server_name}\" if session_id else None\n return None\n\n async def _get_tools(self):\n \"\"\"Get cached tools or update if necessary.\"\"\"\n mcp_server = getattr(self, \"mcp_server\", None)\n if not self._not_load_actions:\n tools, _ = await self.update_tool_list(mcp_server)\n return tools\n return []\n" + }, + "mcp_server": { + "_input_type": "McpInput", + "advanced": false, + "display_name": "MCP Server", + "dynamic": false, + "info": "Select the MCP Server that will be used by this component", + "name": "mcp_server", + "placeholder": "", + "real_time_refresh": true, + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "mcp", + "value": { + "config": { + "args": [ + "mcp-proxy", + "--headers", + "x-api-key", + "sk-yG6gCxpN87BUG3PYWlQWrePJxVNo__UHipbFtyxweCU", + "http://localhost:7860/api/v1/mcp/project/93eb5c48-0b07-4b83-93a5-1d5ba4a521e6/sse", + "--headers", + "X-Langflow-Global-Var-JWT", + "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vb3BlbnJhZy1iYWNrZW5kOjgwMDAiLCJzdWIiOiIxMDc1NDkxMDcwMzY3OTcwMTcwODIiLCJhdWQiOlsib3BlbnNlYXJjaCIsIm9wZW5yYWciXSwiZXhwIjoxNzYwMTA2NTg3LCJpYXQiOjE3NTk1MDE3ODcsImF1dGhfdGltZSI6MTc1OTUwMTc4NywidXNlcl9pZCI6IjEwNzU0OTEwNzAzNjc5NzAxNzA4MiIsImVtYWlsIjoibGFuZ2Zsb3dkZW1vQGdtYWlsLmNvbSIsIm5hbWUiOiJEZW1vIEVtYWlsIiwicHJlZmVycmVkX3VzZXJuYW1lIjoibGFuZ2Zsb3dkZW1vQGdtYWlsLmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJyb2xlcyI6WyJvcGVucmFnX3VzZXIiXX0.KXIVlw1MImvZdM5M5GDVFCbizdBTeMScRgPz2zGO6tZaO48-EIkRA-j2557XdPNgVTmHD4ARl8N66aok-CY1rysSoeDJZ0zK2kAXWORtt1Jrs_pdJOwfG9HwHHkaAbSFxmhvAUFr4akE8Ye83tRQQnBxw3f03Js_E37rYeKiMiNTuOSEHQQXkI-4rPDbnz1srDz6xW_JPv-PBD5y5004uJbqUdfSzYJQ9D7Tprq2DvzO0YIFHGgzJ7S6inZ4ky3yyBQ0nLgQt9ISjlNtLEkn2GvAjXDbVNZzwGFmnu8OgqFM9RL9TNtAdfUFDaip0nW-7Xc2KvVNZWzxG7asuf77BQ", + "--headers", + "X-Langflow-Global-Var-OWNER", + "107549107036797017082", + "--headers", + "X-Langflow-Global-Var-OWNER_NAME", + "Demo Email", + "--headers", + "X-Langflow-Global-Var-OWNER_EMAIL", + "langflowdemo@gmail.com" + ], + "command": "uvx" + }, + "name": "lf-starter_project" + } + }, + "tool": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Tool", + "dynamic": false, + "external_options": {}, + "info": "Select the tool to execute", + "name": "tool", + "options": [ + "opensearch_url_ingestion_flow" + ], + "options_metadata": [], + "placeholder": "", + "real_time_refresh": true, + "required": true, + "show": false, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "tool_placeholder": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Tool Placeholder", + "dynamic": false, + "info": "Placeholder for the tool", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "tool_placeholder", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": true, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "tools_metadata": { + "_input_type": "ToolsInput", + "advanced": false, + "display_name": "Actions", + "dynamic": false, + "info": "Modify tool names and descriptions to help agents understand when to use each tool.", + "is_list": true, + "list_add_label": "Add More", + "name": "tools_metadata", + "placeholder": "", + "real_time_refresh": true, + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "tools", + "value": [ + { + "args": { + "input_value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Message to be passed as input.", + "title": "Input Value" + } + }, + "description": "This flow is to ingest the URL to open search.", + "display_description": "This flow is to ingest the URL to open search.", + "display_name": "opensearch_url_ingestion_flow", + "name": "opensearch_url_ingestion_flow", + "readonly": false, + "status": true, + "tags": [ + "opensearch_url_ingestion_flow" + ] + } + ] + }, + "use_cache": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Use Cached Server", + "dynamic": false, + "info": "Enable caching of MCP Server and tools to improve performance. Disable to always fetch fresh tools and server updates.", + "list": false, + "list_add_label": "Add More", + "name": "use_cache", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": false + } + }, + "tool_mode": true + }, + "showNode": true, + "type": "MCP" + }, + "id": "MCP-Uk5Vf", + "measured": { + "height": 284, + "width": 320 + }, + "position": { + "x": 554.5386324758331, + "y": 845.4231682975634 + }, + "selected": false, + "type": "genericNode" } ], "viewport": { - "x": -325.1448543831343, - "y": 117.81831162150468, - "zoom": 0.6031333914833354 + "x": -184.98015964664273, + "y": 154.6885920024542, + "zoom": 0.602433700773958 } }, "description": "OpenRAG Open Search Agent", diff --git a/flows/openrag_url_mcp.json b/flows/openrag_url_mcp.json index 9bf255ff..c8f8cb70 100644 --- a/flows/openrag_url_mcp.json +++ b/flows/openrag_url_mcp.json @@ -199,6 +199,58 @@ "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-A98BLœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", "target": "SplitText-QIKhg", "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" + }, + { + "data": { + "sourceHandle": { + "dataType": "SplitText", + "id": "SplitText-QIKhg", + "name": "dataframe", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "df", + "id": "DataFrameOperations-RhKoe", + "inputTypes": [ + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__SplitText-QIKhg{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-RhKoe{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-RhKoeœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "source": "SplitText-QIKhg", + "sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "DataFrameOperations-RhKoe", + "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-RhKoeœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "data": { + "sourceHandle": { + "dataType": "DataFrameOperations", + "id": "DataFrameOperations-RhKoe", + "name": "output", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "input_value", + "id": "ChatOutput-Q1dhr", + "inputTypes": [ + "Data", + "DataFrame", + "Message" + ], + "type": "other" + } + }, + "id": "xy-edge__DataFrameOperations-RhKoe{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-RhKoeœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-ChatOutput-Q1dhr{œfieldNameœ:œinput_valueœ,œidœ:œChatOutput-Q1dhrœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}", + "source": "DataFrameOperations-RhKoe", + "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-RhKoeœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "ChatOutput-Q1dhr", + "targetHandle": "{œfieldNameœ:œinput_valueœ,œidœ:œChatOutput-Q1dhrœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" } ], "nodes": [ @@ -487,6 +539,7 @@ "frozen": false, "icon": "OpenAI", "legacy": false, + "lf_version": "1.6.0", "metadata": { "code_hash": "8a658ed6d4c9", "dependencies": { @@ -773,7 +826,7 @@ "dynamic": false, "info": "", "input_types": [], - "load_from_db": false, + "load_from_db": true, "name": "openai_api_key", "password": true, "placeholder": "", @@ -781,7 +834,7 @@ "show": true, "title_case": false, "type": "str", - "value": "" + "value": "OPENAI_API_KEY" }, "openai_api_type": { "_input_type": "MessageTextInput", @@ -1070,6 +1123,7 @@ "frozen": false, "icon": "OpenSearch", "legacy": false, + "lf_version": "1.6.0", "metadata": { "code_hash": "08d808984c3d", "dependencies": { @@ -1100,7 +1154,6 @@ "name": "search_results", "options": null, "required_inputs": null, - "selected": "Data", "tool_mode": true, "types": [ "Data" @@ -1134,7 +1187,6 @@ "name": "vectorstoreconnection", "options": null, "required_inputs": null, - "selected": "VectorStore", "tool_mode": true, "types": [ "VectorStore" @@ -1644,7 +1696,7 @@ }, "tool_mode": false }, - "selected_output": "search_results", + "selected_output": "dataframe", "showNode": true, "type": "OpenSearchVectorStoreComponent" }, @@ -1658,7 +1710,7 @@ "x": 2694.183983837566, "y": 1425.777807367294 }, - "selected": true, + "selected": false, "type": "genericNode" }, { @@ -2330,7 +2382,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-10-02T18:55:27.635Z", + "last_updated": "2025-10-03T14:31:53.355Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -2746,7 +2798,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-10-02T18:55:27.636Z", + "last_updated": "2025-10-03T14:31:53.355Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -3131,12 +3183,684 @@ }, "selected": false, "type": "genericNode" + }, + { + "data": { + "id": "ChatOutput-Q1dhr", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Display a chat message in the Playground.", + "display_name": "Chat Output", + "documentation": "https://docs.langflow.org/components-io#chat-output", + "edited": false, + "field_order": [ + "input_value", + "should_store_message", + "sender", + "sender_name", + "session_id", + "data_template", + "clean_data" + ], + "frozen": false, + "icon": "MessagesSquare", + "legacy": false, + "metadata": { + "code_hash": "9647f4d2f4b4", + "dependencies": { + "dependencies": [ + { + "name": "orjson", + "version": "3.10.15" + }, + { + "name": "fastapi", + "version": "0.117.1" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 3 + }, + "module": "lfx.components.input_output.chat_output.ChatOutput" + }, + "minimized": true, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Output Message", + "group_outputs": false, + "method": "message_response", + "name": "message", + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "clean_data": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Basic Clean Data", + "dynamic": false, + "info": "Whether to clean data before converting to string.", + "list": false, + "list_add_label": "Add More", + "name": "clean_data", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from collections.abc import Generator\nfrom typing import Any\n\nimport orjson\nfrom fastapi.encoders import jsonable_encoder\n\nfrom lfx.base.io.chat import ChatComponent\nfrom lfx.helpers.data import safe_convert\nfrom lfx.inputs.inputs import BoolInput, DropdownInput, HandleInput, MessageTextInput\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.schema.properties import Source\nfrom lfx.template.field.base import Output\nfrom lfx.utils.constants import (\n MESSAGE_SENDER_AI,\n MESSAGE_SENDER_NAME_AI,\n MESSAGE_SENDER_USER,\n)\n\n\nclass ChatOutput(ChatComponent):\n display_name = \"Chat Output\"\n description = \"Display a chat message in the Playground.\"\n documentation: str = \"https://docs.langflow.org/components-io#chat-output\"\n icon = \"MessagesSquare\"\n name = \"ChatOutput\"\n minimized = True\n\n inputs = [\n HandleInput(\n name=\"input_value\",\n display_name=\"Inputs\",\n info=\"Message to be passed as output.\",\n input_types=[\"Data\", \"DataFrame\", \"Message\"],\n required=True,\n ),\n BoolInput(\n name=\"should_store_message\",\n display_name=\"Store Messages\",\n info=\"Store the message in the history.\",\n value=True,\n advanced=True,\n ),\n DropdownInput(\n name=\"sender\",\n display_name=\"Sender Type\",\n options=[MESSAGE_SENDER_AI, MESSAGE_SENDER_USER],\n value=MESSAGE_SENDER_AI,\n advanced=True,\n info=\"Type of sender.\",\n ),\n MessageTextInput(\n name=\"sender_name\",\n display_name=\"Sender Name\",\n info=\"Name of the sender.\",\n value=MESSAGE_SENDER_NAME_AI,\n advanced=True,\n ),\n MessageTextInput(\n name=\"session_id\",\n display_name=\"Session ID\",\n info=\"The session ID of the chat. If empty, the current session ID parameter will be used.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"data_template\",\n display_name=\"Data Template\",\n value=\"{text}\",\n advanced=True,\n info=\"Template to convert Data to Text. If left empty, it will be dynamically set to the Data's text key.\",\n ),\n BoolInput(\n name=\"clean_data\",\n display_name=\"Basic Clean Data\",\n value=True,\n advanced=True,\n info=\"Whether to clean data before converting to string.\",\n ),\n ]\n outputs = [\n Output(\n display_name=\"Output Message\",\n name=\"message\",\n method=\"message_response\",\n ),\n ]\n\n def _build_source(self, id_: str | None, display_name: str | None, source: str | None) -> Source:\n source_dict = {}\n if id_:\n source_dict[\"id\"] = id_\n if display_name:\n source_dict[\"display_name\"] = display_name\n if source:\n # Handle case where source is a ChatOpenAI object\n if hasattr(source, \"model_name\"):\n source_dict[\"source\"] = source.model_name\n elif hasattr(source, \"model\"):\n source_dict[\"source\"] = str(source.model)\n else:\n source_dict[\"source\"] = str(source)\n return Source(**source_dict)\n\n async def message_response(self) -> Message:\n # First convert the input to string if needed\n text = self.convert_to_string()\n\n # Get source properties\n source, _, display_name, source_id = self.get_properties_from_source_component()\n\n # Create or use existing Message object\n if isinstance(self.input_value, Message):\n message = self.input_value\n # Update message properties\n message.text = text\n else:\n message = Message(text=text)\n\n # Set message properties\n message.sender = self.sender\n message.sender_name = self.sender_name\n message.session_id = self.session_id\n message.flow_id = self.graph.flow_id if hasattr(self, \"graph\") else None\n message.properties.source = self._build_source(source_id, display_name, source)\n\n # Store message if needed\n if self.session_id and self.should_store_message:\n stored_message = await self.send_message(message)\n self.message.value = stored_message\n message = stored_message\n\n self.status = message\n return message\n\n def _serialize_data(self, data: Data) -> str:\n \"\"\"Serialize Data object to JSON string.\"\"\"\n # Convert data.data to JSON-serializable format\n serializable_data = jsonable_encoder(data.data)\n # Serialize with orjson, enabling pretty printing with indentation\n json_bytes = orjson.dumps(serializable_data, option=orjson.OPT_INDENT_2)\n # Convert bytes to string and wrap in Markdown code blocks\n return \"```json\\n\" + json_bytes.decode(\"utf-8\") + \"\\n```\"\n\n def _validate_input(self) -> None:\n \"\"\"Validate the input data and raise ValueError if invalid.\"\"\"\n if self.input_value is None:\n msg = \"Input data cannot be None\"\n raise ValueError(msg)\n if isinstance(self.input_value, list) and not all(\n isinstance(item, Message | Data | DataFrame | str) for item in self.input_value\n ):\n invalid_types = [\n type(item).__name__\n for item in self.input_value\n if not isinstance(item, Message | Data | DataFrame | str)\n ]\n msg = f\"Expected Data or DataFrame or Message or str, got {invalid_types}\"\n raise TypeError(msg)\n if not isinstance(\n self.input_value,\n Message | Data | DataFrame | str | list | Generator | type(None),\n ):\n type_name = type(self.input_value).__name__\n msg = f\"Expected Data or DataFrame or Message or str, Generator or None, got {type_name}\"\n raise TypeError(msg)\n\n def convert_to_string(self) -> str | Generator[Any, None, None]:\n \"\"\"Convert input data to string with proper error handling.\"\"\"\n self._validate_input()\n if isinstance(self.input_value, list):\n clean_data: bool = getattr(self, \"clean_data\", False)\n return \"\\n\".join([safe_convert(item, clean_data=clean_data) for item in self.input_value])\n if isinstance(self.input_value, Generator):\n return self.input_value\n return safe_convert(self.input_value)\n" + }, + "data_template": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Data Template", + "dynamic": false, + "info": "Template to convert Data to Text. If left empty, it will be dynamically set to the Data's text key.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "data_template", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "{text}" + }, + "input_value": { + "_input_type": "HandleInput", + "advanced": false, + "display_name": "Inputs", + "dynamic": false, + "info": "Message to be passed as output.", + "input_types": [ + "Data", + "DataFrame", + "Message" + ], + "list": false, + "list_add_label": "Add More", + "name": "input_value", + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "type": "other", + "value": "" + }, + "sender": { + "_input_type": "DropdownInput", + "advanced": true, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Sender Type", + "dynamic": false, + "external_options": {}, + "info": "Type of sender.", + "name": "sender", + "options": [ + "Machine", + "User" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "Machine" + }, + "sender_name": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Sender Name", + "dynamic": false, + "info": "Name of the sender.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "sender_name", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "AI" + }, + "session_id": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Session ID", + "dynamic": false, + "info": "The session ID of the chat. If empty, the current session ID parameter will be used.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "session_id", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "should_store_message": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Store Messages", + "dynamic": false, + "info": "Store the message in the history.", + "list": false, + "list_add_label": "Add More", + "name": "should_store_message", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + } + }, + "tool_mode": false + }, + "showNode": false, + "type": "ChatOutput" + }, + "dragging": false, + "id": "ChatOutput-Q1dhr", + "measured": { + "height": 48, + "width": 192 + }, + "position": { + "x": 3135.060664388748, + "y": 2541.1604512818694 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "DataFrameOperations-RhKoe", + "node": { + "base_classes": [ + "DataFrame" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Perform various operations on a DataFrame.", + "display_name": "DataFrame Operations", + "documentation": "https://docs.langflow.org/components-processing#dataframe-operations", + "edited": false, + "field_order": [ + "df", + "operation", + "column_name", + "filter_value", + "filter_operator", + "ascending", + "new_column_name", + "new_column_value", + "columns_to_select", + "num_rows", + "replace_value", + "replacement_value" + ], + "frozen": false, + "icon": "table", + "last_updated": "2025-10-03T14:40:28.935Z", + "legacy": false, + "metadata": { + "code_hash": "b4d6b19b6eef", + "dependencies": { + "dependencies": [ + { + "name": "pandas", + "version": "2.2.3" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 2 + }, + "module": "lfx.components.processing.dataframe_operations.DataFrameOperationsComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "DataFrame", + "group_outputs": false, + "method": "perform_operation", + "name": "output", + "options": null, + "required_inputs": null, + "selected": "DataFrame", + "tool_mode": true, + "types": [ + "DataFrame" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "ascending": { + "_input_type": "BoolInput", + "advanced": false, + "display_name": "Sort Ascending", + "dynamic": true, + "info": "Whether to sort in ascending order.", + "list": false, + "list_add_label": "Add More", + "name": "ascending", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": true + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "import pandas as pd\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs import SortableListInput\nfrom lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\n\n\nclass DataFrameOperationsComponent(Component):\n display_name = \"DataFrame Operations\"\n description = \"Perform various operations on a DataFrame.\"\n documentation: str = \"https://docs.langflow.org/components-processing#dataframe-operations\"\n icon = \"table\"\n name = \"DataFrameOperations\"\n\n OPERATION_CHOICES = [\n \"Add Column\",\n \"Drop Column\",\n \"Filter\",\n \"Head\",\n \"Rename Column\",\n \"Replace Value\",\n \"Select Columns\",\n \"Sort\",\n \"Tail\",\n \"Drop Duplicates\",\n ]\n\n inputs = [\n DataFrameInput(\n name=\"df\",\n display_name=\"DataFrame\",\n info=\"The input DataFrame to operate on.\",\n required=True,\n ),\n SortableListInput(\n name=\"operation\",\n display_name=\"Operation\",\n placeholder=\"Select Operation\",\n info=\"Select the DataFrame operation to perform.\",\n options=[\n {\"name\": \"Add Column\", \"icon\": \"plus\"},\n {\"name\": \"Drop Column\", \"icon\": \"minus\"},\n {\"name\": \"Filter\", \"icon\": \"filter\"},\n {\"name\": \"Head\", \"icon\": \"arrow-up\"},\n {\"name\": \"Rename Column\", \"icon\": \"pencil\"},\n {\"name\": \"Replace Value\", \"icon\": \"replace\"},\n {\"name\": \"Select Columns\", \"icon\": \"columns\"},\n {\"name\": \"Sort\", \"icon\": \"arrow-up-down\"},\n {\"name\": \"Tail\", \"icon\": \"arrow-down\"},\n {\"name\": \"Drop Duplicates\", \"icon\": \"copy-x\"},\n ],\n real_time_refresh=True,\n limit=1,\n ),\n StrInput(\n name=\"column_name\",\n display_name=\"Column Name\",\n info=\"The column name to use for the operation.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"filter_value\",\n display_name=\"Filter Value\",\n info=\"The value to filter rows by.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"filter_operator\",\n display_name=\"Filter Operator\",\n options=[\n \"equals\",\n \"not equals\",\n \"contains\",\n \"not contains\",\n \"starts with\",\n \"ends with\",\n \"greater than\",\n \"less than\",\n ],\n value=\"equals\",\n info=\"The operator to apply for filtering rows.\",\n advanced=False,\n dynamic=True,\n show=False,\n ),\n BoolInput(\n name=\"ascending\",\n display_name=\"Sort Ascending\",\n info=\"Whether to sort in ascending order.\",\n dynamic=True,\n show=False,\n value=True,\n ),\n StrInput(\n name=\"new_column_name\",\n display_name=\"New Column Name\",\n info=\"The new column name when renaming or adding a column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"new_column_value\",\n display_name=\"New Column Value\",\n info=\"The value to populate the new column with.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"columns_to_select\",\n display_name=\"Columns to Select\",\n dynamic=True,\n is_list=True,\n show=False,\n ),\n IntInput(\n name=\"num_rows\",\n display_name=\"Number of Rows\",\n info=\"Number of rows to return (for head/tail).\",\n dynamic=True,\n show=False,\n value=5,\n ),\n MessageTextInput(\n name=\"replace_value\",\n display_name=\"Value to Replace\",\n info=\"The value to replace in the column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"replacement_value\",\n display_name=\"Replacement Value\",\n info=\"The value to replace with.\",\n dynamic=True,\n show=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"DataFrame\",\n name=\"output\",\n method=\"perform_operation\",\n info=\"The resulting DataFrame after the operation.\",\n )\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n dynamic_fields = [\n \"column_name\",\n \"filter_value\",\n \"filter_operator\",\n \"ascending\",\n \"new_column_name\",\n \"new_column_value\",\n \"columns_to_select\",\n \"num_rows\",\n \"replace_value\",\n \"replacement_value\",\n ]\n for field in dynamic_fields:\n build_config[field][\"show\"] = False\n\n if field_name == \"operation\":\n # Handle SortableListInput format\n if isinstance(field_value, list):\n operation_name = field_value[0].get(\"name\", \"\") if field_value else \"\"\n else:\n operation_name = field_value or \"\"\n\n # If no operation selected, all dynamic fields stay hidden (already set to False above)\n if not operation_name:\n return build_config\n\n if operation_name == \"Filter\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"filter_value\"][\"show\"] = True\n build_config[\"filter_operator\"][\"show\"] = True\n elif operation_name == \"Sort\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"ascending\"][\"show\"] = True\n elif operation_name == \"Drop Column\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Rename Column\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"new_column_name\"][\"show\"] = True\n elif operation_name == \"Add Column\":\n build_config[\"new_column_name\"][\"show\"] = True\n build_config[\"new_column_value\"][\"show\"] = True\n elif operation_name == \"Select Columns\":\n build_config[\"columns_to_select\"][\"show\"] = True\n elif operation_name in {\"Head\", \"Tail\"}:\n build_config[\"num_rows\"][\"show\"] = True\n elif operation_name == \"Replace Value\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"replace_value\"][\"show\"] = True\n build_config[\"replacement_value\"][\"show\"] = True\n elif operation_name == \"Drop Duplicates\":\n build_config[\"column_name\"][\"show\"] = True\n\n return build_config\n\n def perform_operation(self) -> DataFrame:\n df_copy = self.df.copy()\n\n # Handle SortableListInput format for operation\n operation_input = getattr(self, \"operation\", [])\n if isinstance(operation_input, list) and len(operation_input) > 0:\n op = operation_input[0].get(\"name\", \"\")\n else:\n op = \"\"\n\n # If no operation selected, return original DataFrame\n if not op:\n return df_copy\n\n if op == \"Filter\":\n return self.filter_rows_by_value(df_copy)\n if op == \"Sort\":\n return self.sort_by_column(df_copy)\n if op == \"Drop Column\":\n return self.drop_column(df_copy)\n if op == \"Rename Column\":\n return self.rename_column(df_copy)\n if op == \"Add Column\":\n return self.add_column(df_copy)\n if op == \"Select Columns\":\n return self.select_columns(df_copy)\n if op == \"Head\":\n return self.head(df_copy)\n if op == \"Tail\":\n return self.tail(df_copy)\n if op == \"Replace Value\":\n return self.replace_values(df_copy)\n if op == \"Drop Duplicates\":\n return self.drop_duplicates(df_copy)\n msg = f\"Unsupported operation: {op}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def filter_rows_by_value(self, df: DataFrame) -> DataFrame:\n column = df[self.column_name]\n filter_value = self.filter_value\n\n # Handle regular DropdownInput format (just a string value)\n operator = getattr(self, \"filter_operator\", \"equals\") # Default to equals for backward compatibility\n\n if operator == \"equals\":\n mask = column == filter_value\n elif operator == \"not equals\":\n mask = column != filter_value\n elif operator == \"contains\":\n mask = column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"not contains\":\n mask = ~column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"starts with\":\n mask = column.astype(str).str.startswith(str(filter_value), na=False)\n elif operator == \"ends with\":\n mask = column.astype(str).str.endswith(str(filter_value), na=False)\n elif operator == \"greater than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column > numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) > str(filter_value)\n elif operator == \"less than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column < numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) < str(filter_value)\n else:\n mask = column == filter_value # Fallback to equals\n\n return DataFrame(df[mask])\n\n def sort_by_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))\n\n def drop_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop(columns=[self.column_name]))\n\n def rename_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))\n\n def add_column(self, df: DataFrame) -> DataFrame:\n df[self.new_column_name] = [self.new_column_value] * len(df)\n return DataFrame(df)\n\n def select_columns(self, df: DataFrame) -> DataFrame:\n columns = [col.strip() for col in self.columns_to_select]\n return DataFrame(df[columns])\n\n def head(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.head(self.num_rows))\n\n def tail(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.tail(self.num_rows))\n\n def replace_values(self, df: DataFrame) -> DataFrame:\n df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)\n return DataFrame(df)\n\n def drop_duplicates(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop_duplicates(subset=self.column_name))\n" + }, + "column_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Column Name", + "dynamic": true, + "info": "The column name to use for the operation.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "column_name", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "columns_to_select": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Columns to Select", + "dynamic": true, + "info": "", + "list": true, + "list_add_label": "Add More", + "load_from_db": false, + "name": "columns_to_select", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "df": { + "_input_type": "DataFrameInput", + "advanced": false, + "display_name": "DataFrame", + "dynamic": false, + "info": "The input DataFrame to operate on.", + "input_types": [ + "DataFrame" + ], + "list": false, + "list_add_label": "Add More", + "name": "df", + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "other", + "value": "" + }, + "filter_operator": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Filter Operator", + "dynamic": true, + "external_options": {}, + "info": "The operator to apply for filtering rows.", + "name": "filter_operator", + "options": [ + "equals", + "not equals", + "contains", + "not contains", + "starts with", + "ends with", + "greater than", + "less than" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "equals" + }, + "filter_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Filter Value", + "dynamic": true, + "info": "The value to filter rows by.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "filter_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "new_column_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "New Column Name", + "dynamic": true, + "info": "The new column name when renaming or adding a column.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "new_column_name", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "new_column_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "New Column Value", + "dynamic": true, + "info": "The value to populate the new column with.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "new_column_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "num_rows": { + "_input_type": "IntInput", + "advanced": false, + "display_name": "Number of Rows", + "dynamic": true, + "info": "Number of rows to return (for head/tail).", + "list": false, + "list_add_label": "Add More", + "name": "num_rows", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 5 + }, + "operation": { + "_input_type": "SortableListInput", + "advanced": false, + "display_name": "Operation", + "dynamic": false, + "info": "Select the DataFrame operation to perform.", + "limit": 1, + "name": "operation", + "options": [ + { + "icon": "plus", + "name": "Add Column" + }, + { + "icon": "minus", + "name": "Drop Column" + }, + { + "icon": "filter", + "name": "Filter" + }, + { + "icon": "arrow-up", + "name": "Head" + }, + { + "icon": "pencil", + "name": "Rename Column" + }, + { + "icon": "replace", + "name": "Replace Value" + }, + { + "icon": "columns", + "name": "Select Columns" + }, + { + "icon": "arrow-up-down", + "name": "Sort" + }, + { + "icon": "arrow-down", + "name": "Tail" + }, + { + "icon": "copy-x", + "name": "Drop Duplicates" + } + ], + "placeholder": "Select Operation", + "real_time_refresh": true, + "required": false, + "search_category": [], + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "sortableList", + "value": [ + { + "chosen": false, + "icon": "arrow-up", + "name": "Head", + "selected": false + } + ] + }, + "replace_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Value to Replace", + "dynamic": true, + "info": "The value to replace in the column.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "replace_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "replacement_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Replacement Value", + "dynamic": true, + "info": "The value to replace with.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "replacement_value", + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + } + }, + "tool_mode": false + }, + "showNode": true, + "type": "DataFrameOperations" + }, + "dragging": false, + "id": "DataFrameOperations-RhKoe", + "measured": { + "height": 317, + "width": 320 + }, + "position": { + "x": 2773.2060092972047, + "y": 2337.54590413581 + }, + "selected": true, + "type": "genericNode" } ], "viewport": { - "x": -455.58644249058534, - "y": -635.4279993708312, - "zoom": 0.5860911118774581 + "x": -1399.3594540015256, + "y": -1163.4584036787048, + "zoom": 0.7595394813583742 } }, "description": "This flow is to ingest the URL to open search.", @@ -3144,8 +3868,8 @@ "id": "72c3d17c-2dac-4a73-b48a-6518473d7830", "is_component": false, "last_tested_version": "1.6.0", - "mcp_enabled": true, "name": "OpenSearch URL Ingestion Flow", + "mcp_enabled": true, "tags": [ "openai", "astradb", From 7b860b3f6f55c7c3a2d09b392f37a4c39a0d51ed Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 09:08:33 -0600 Subject: [PATCH 16/55] fix frontend ID --- frontend/src/app/api/queries/useGetSearchQuery.ts | 5 +++-- frontend/src/app/knowledge/chunks/page.tsx | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/api/queries/useGetSearchQuery.ts b/frontend/src/app/api/queries/useGetSearchQuery.ts index 5383178d..d0c1a3a9 100644 --- a/frontend/src/app/api/queries/useGetSearchQuery.ts +++ b/frontend/src/app/api/queries/useGetSearchQuery.ts @@ -29,6 +29,7 @@ export interface ChunkResult { owner_email?: string; file_size?: number; connector_type?: string; + index?: number; } export interface File { @@ -55,7 +56,7 @@ export interface File { export const useGetSearchQuery = ( query: string, queryData?: ParsedQueryData | null, - options?: Omit, + options?: Omit ) => { const queryClient = useQueryClient(); @@ -184,7 +185,7 @@ export const useGetSearchQuery = ( queryFn: getFiles, ...options, }, - queryClient, + queryClient ); return queryResult; diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index bc00f886..9b02506b 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -86,7 +86,9 @@ function ChunksPageContent() { return; } - setChunks(fileData?.chunks || []); + setChunks( + fileData?.chunks?.map((chunk, i) => ({ ...chunk, index: i + 1 })) || [] + ); }, [data, filename]); // Set selected state for all checkboxes when selectAll changes @@ -246,7 +248,7 @@ function ChunksPageContent() { />
- Chunk {chunk.page} + Chunk {chunk.index} {chunk.text.length} chars From aa1ad5c50f213c930fc4a5816f59310337f803b3 Mon Sep 17 00:00:00 2001 From: Edwin Jose Date: Fri, 3 Oct 2025 11:09:17 -0400 Subject: [PATCH 17/55] Update openrag_agent.json --- flows/openrag_agent.json | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/flows/openrag_agent.json b/flows/openrag_agent.json index 9aa875d1..3b83f107 100644 --- a/flows/openrag_agent.json +++ b/flows/openrag_agent.json @@ -170,6 +170,31 @@ "sourceHandle": "{œdataTypeœ:œMCPœ,œidœ:œMCP-Uk5Vfœ,œnameœ:œcomponent_as_toolœ,œoutput_typesœ:[œToolœ]}", "target": "Agent-crjWf", "targetHandle": "{œfieldNameœ:œtoolsœ,œidœ:œAgent-crjWfœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}" + }, + { + "data": { + "sourceHandle": { + "dataType": "TextInput", + "id": "TextInput-aHsQb", + "name": "text", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "filter_expression", + "id": "OpenSearch-iYfjf", + "inputTypes": [ + "Message" + ], + "type": "str" + } + }, + "id": "xy-edge__TextInput-aHsQb{œdataTypeœ:œTextInputœ,œidœ:œTextInput-aHsQbœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearch-iYfjf{œfieldNameœ:œfilter_expressionœ,œidœ:œOpenSearch-iYfjfœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}", + "source": "TextInput-aHsQb", + "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-aHsQbœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", + "target": "OpenSearch-iYfjf", + "targetHandle": "{œfieldNameœ:œfilter_expressionœ,œidœ:œOpenSearch-iYfjfœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}" } ], "nodes": [ @@ -730,7 +755,7 @@ ], "frozen": false, "icon": "OpenSearch", - "last_updated": "2025-10-03T14:42:51.280Z", + "last_updated": "2025-10-03T15:07:58.068Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -1384,7 +1409,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-10-03T14:42:51.281Z", + "last_updated": "2025-10-03T15:07:58.070Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -1663,9 +1688,9 @@ }, "position": { "x": 727.4791597769406, - "y": 518.0820551650631 + "y": 416.82609966052854 }, - "selected": false, + "selected": true, "type": "genericNode" }, { @@ -1709,7 +1734,7 @@ ], "frozen": false, "icon": "bot", - "last_updated": "2025-10-03T14:42:51.349Z", + "last_updated": "2025-10-03T15:07:58.132Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -2248,7 +2273,7 @@ ], "frozen": false, "icon": "brain-circuit", - "last_updated": "2025-10-03T14:42:51.281Z", + "last_updated": "2025-10-03T15:07:58.071Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -2576,7 +2601,7 @@ "frozen": false, "icon": "Mcp", "key": "mcp_lf-starter_project", - "last_updated": "2025-10-03T14:42:51.282Z", + "last_updated": "2025-10-03T15:07:58.072Z", "legacy": false, "mcpServerName": "lf-starter_project", "metadata": { From ac333b99d7317b8b3400e280d836454ffae64676 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 09:12:35 -0600 Subject: [PATCH 18/55] added value --- frontend/src/app/knowledge/chunks/page.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index 9b02506b..bbb7f019 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -188,6 +188,7 @@ function ChunksPageContent() { type="text" placeholder="Search your documents..." onChange={(e) => setQueryInputText(e.target.value)} + value={queryInputText} />
From 11a6f0e8d415ce989929c49e566da75d474b11bf Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 10:47:31 -0600 Subject: [PATCH 19/55] updated header and cloud picker header --- frontend/src/app/upload/[provider]/page.tsx | 659 +++++++++--------- .../components/cloud-picker/picker-header.tsx | 8 +- 2 files changed, 332 insertions(+), 335 deletions(-) diff --git a/frontend/src/app/upload/[provider]/page.tsx b/frontend/src/app/upload/[provider]/page.tsx index 7c72ec3d..8e0a306c 100644 --- a/frontend/src/app/upload/[provider]/page.tsx +++ b/frontend/src/app/upload/[provider]/page.tsx @@ -12,367 +12,370 @@ import { useTask } from "@/contexts/task-context"; // CloudFile interface is now imported from the unified cloud picker interface CloudConnector { - id: string; - name: string; - description: string; - status: "not_connected" | "connecting" | "connected" | "error"; - type: string; - connectionId?: string; - clientId: string; - hasAccessToken: boolean; - accessTokenError?: string; + id: string; + name: string; + description: string; + status: "not_connected" | "connecting" | "connected" | "error"; + type: string; + connectionId?: string; + clientId: string; + hasAccessToken: boolean; + accessTokenError?: string; } export default function UploadProviderPage() { - const params = useParams(); - const router = useRouter(); - const provider = params.provider as string; - const { addTask, tasks } = useTask(); + const params = useParams(); + const router = useRouter(); + const provider = params.provider as string; + const { addTask, tasks } = useTask(); - const [connector, setConnector] = useState(null); - const [isLoading, setIsLoading] = useState(true); - const [error, setError] = useState(null); - const [accessToken, setAccessToken] = useState(null); - const [selectedFiles, setSelectedFiles] = useState([]); - const [isIngesting, setIsIngesting] = useState(false); - const [currentSyncTaskId, setCurrentSyncTaskId] = useState( - null, - ); - const [ingestSettings, setIngestSettings] = useState({ - chunkSize: 1000, - chunkOverlap: 200, - ocr: false, - pictureDescriptions: false, - embeddingModel: "text-embedding-3-small", - }); + const [connector, setConnector] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const [accessToken, setAccessToken] = useState(null); + const [selectedFiles, setSelectedFiles] = useState([]); + const [isIngesting, setIsIngesting] = useState(false); + const [currentSyncTaskId, setCurrentSyncTaskId] = useState( + null + ); + const [ingestSettings, setIngestSettings] = useState({ + chunkSize: 1000, + chunkOverlap: 200, + ocr: false, + pictureDescriptions: false, + embeddingModel: "text-embedding-3-small", + }); - useEffect(() => { - const fetchConnectorInfo = async () => { - setIsLoading(true); - setError(null); + useEffect(() => { + const fetchConnectorInfo = async () => { + setIsLoading(true); + setError(null); - try { - // Fetch available connectors to validate the provider - const connectorsResponse = await fetch("/api/connectors"); - if (!connectorsResponse.ok) { - throw new Error("Failed to load connectors"); - } + try { + // Fetch available connectors to validate the provider + const connectorsResponse = await fetch("/api/connectors"); + if (!connectorsResponse.ok) { + throw new Error("Failed to load connectors"); + } - const connectorsResult = await connectorsResponse.json(); - const providerInfo = connectorsResult.connectors[provider]; + const connectorsResult = await connectorsResponse.json(); + const providerInfo = connectorsResult.connectors[provider]; - if (!providerInfo || !providerInfo.available) { - setError( - `Cloud provider "${provider}" is not available or configured.`, - ); - return; - } + if (!providerInfo || !providerInfo.available) { + setError( + `Cloud provider "${provider}" is not available or configured.` + ); + return; + } - // Check connector status - const statusResponse = await fetch( - `/api/connectors/${provider}/status`, - ); - if (!statusResponse.ok) { - throw new Error(`Failed to check ${provider} status`); - } + // Check connector status + const statusResponse = await fetch( + `/api/connectors/${provider}/status` + ); + if (!statusResponse.ok) { + throw new Error(`Failed to check ${provider} status`); + } - const statusData = await statusResponse.json(); - const connections = statusData.connections || []; - const activeConnection = connections.find( - (conn: { is_active: boolean; connection_id: string }) => - conn.is_active, - ); - const isConnected = activeConnection !== undefined; + const statusData = await statusResponse.json(); + const connections = statusData.connections || []; + const activeConnection = connections.find( + (conn: { is_active: boolean; connection_id: string }) => + conn.is_active + ); + const isConnected = activeConnection !== undefined; - let hasAccessToken = false; - let accessTokenError: string | undefined; + let hasAccessToken = false; + let accessTokenError: string | undefined; - // Try to get access token for connected connectors - if (isConnected && activeConnection) { - try { - const tokenResponse = await fetch( - `/api/connectors/${provider}/token?connection_id=${activeConnection.connection_id}`, - ); - if (tokenResponse.ok) { - const tokenData = await tokenResponse.json(); - if (tokenData.access_token) { - hasAccessToken = true; - setAccessToken(tokenData.access_token); - } - } else { - const errorData = await tokenResponse - .json() - .catch(() => ({ error: "Token unavailable" })); - accessTokenError = errorData.error || "Access token unavailable"; - } - } catch { - accessTokenError = "Failed to fetch access token"; - } - } + // Try to get access token for connected connectors + if (isConnected && activeConnection) { + try { + const tokenResponse = await fetch( + `/api/connectors/${provider}/token?connection_id=${activeConnection.connection_id}` + ); + if (tokenResponse.ok) { + const tokenData = await tokenResponse.json(); + if (tokenData.access_token) { + hasAccessToken = true; + setAccessToken(tokenData.access_token); + } + } else { + const errorData = await tokenResponse + .json() + .catch(() => ({ error: "Token unavailable" })); + accessTokenError = errorData.error || "Access token unavailable"; + } + } catch { + accessTokenError = "Failed to fetch access token"; + } + } - setConnector({ - id: provider, - name: providerInfo.name, - description: providerInfo.description, - status: isConnected ? "connected" : "not_connected", - type: provider, - connectionId: activeConnection?.connection_id, - clientId: activeConnection?.client_id, - hasAccessToken, - accessTokenError, - }); - } catch (error) { - console.error("Failed to load connector info:", error); - setError( - error instanceof Error - ? error.message - : "Failed to load connector information", - ); - } finally { - setIsLoading(false); - } - }; + setConnector({ + id: provider, + name: providerInfo.name, + description: providerInfo.description, + status: isConnected ? "connected" : "not_connected", + type: provider, + connectionId: activeConnection?.connection_id, + clientId: activeConnection?.client_id, + hasAccessToken, + accessTokenError, + }); + } catch (error) { + console.error("Failed to load connector info:", error); + setError( + error instanceof Error + ? error.message + : "Failed to load connector information" + ); + } finally { + setIsLoading(false); + } + }; - if (provider) { - fetchConnectorInfo(); - } - }, [provider]); + if (provider) { + fetchConnectorInfo(); + } + }, [provider]); - // Watch for sync task completion and redirect - useEffect(() => { - if (!currentSyncTaskId) return; + // Watch for sync task completion and redirect + useEffect(() => { + if (!currentSyncTaskId) return; - const currentTask = tasks.find( - (task) => task.task_id === currentSyncTaskId, - ); + const currentTask = tasks.find( + (task) => task.task_id === currentSyncTaskId + ); - if (currentTask && currentTask.status === "completed") { - // Task completed successfully, show toast and redirect - setIsIngesting(false); - setTimeout(() => { - router.push("/knowledge"); - }, 2000); // 2 second delay to let user see toast - } else if (currentTask && currentTask.status === "failed") { - // Task failed, clear the tracking but don't redirect - setIsIngesting(false); - setCurrentSyncTaskId(null); - } - }, [tasks, currentSyncTaskId, router]); + if (currentTask && currentTask.status === "completed") { + // Task completed successfully, show toast and redirect + setIsIngesting(false); + setTimeout(() => { + router.push("/knowledge"); + }, 2000); // 2 second delay to let user see toast + } else if (currentTask && currentTask.status === "failed") { + // Task failed, clear the tracking but don't redirect + setIsIngesting(false); + setCurrentSyncTaskId(null); + } + }, [tasks, currentSyncTaskId, router]); - const handleFileSelected = (files: CloudFile[]) => { - setSelectedFiles(files); - console.log(`Selected ${files.length} files from ${provider}:`, files); - // You can add additional handling here like triggering sync, etc. - }; + const handleFileSelected = (files: CloudFile[]) => { + setSelectedFiles(files); + console.log(`Selected ${files.length} files from ${provider}:`, files); + // You can add additional handling here like triggering sync, etc. + }; - const handleSync = async (connector: CloudConnector) => { - if (!connector.connectionId || selectedFiles.length === 0) return; + const handleSync = async (connector: CloudConnector) => { + if (!connector.connectionId || selectedFiles.length === 0) return; - setIsIngesting(true); + setIsIngesting(true); - try { - const syncBody: { - connection_id: string; - max_files?: number; - selected_files?: string[]; - settings?: IngestSettings; - } = { - connection_id: connector.connectionId, - selected_files: selectedFiles.map((file) => file.id), - settings: ingestSettings, - }; + try { + const syncBody: { + connection_id: string; + max_files?: number; + selected_files?: string[]; + settings?: IngestSettings; + } = { + connection_id: connector.connectionId, + selected_files: selectedFiles.map((file) => file.id), + settings: ingestSettings, + }; - const response = await fetch(`/api/connectors/${connector.type}/sync`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(syncBody), - }); + const response = await fetch(`/api/connectors/${connector.type}/sync`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(syncBody), + }); - const result = await response.json(); + const result = await response.json(); - if (response.status === 201) { - const taskIds = result.task_ids; - if (taskIds && taskIds.length > 0) { - const taskId = taskIds[0]; // Use the first task ID - addTask(taskId); - setCurrentSyncTaskId(taskId); - } - } else { - console.error("Sync failed:", result.error); - } - } catch (error) { - console.error("Sync error:", error); - setIsIngesting(false); - } - }; + if (response.status === 201) { + const taskIds = result.task_ids; + if (taskIds && taskIds.length > 0) { + const taskId = taskIds[0]; // Use the first task ID + addTask(taskId); + setCurrentSyncTaskId(taskId); + } + } else { + console.error("Sync failed:", result.error); + } + } catch (error) { + console.error("Sync error:", error); + setIsIngesting(false); + } + }; - const getProviderDisplayName = () => { - const nameMap: { [key: string]: string } = { - google_drive: "Google Drive", - onedrive: "OneDrive", - sharepoint: "SharePoint", - }; - return nameMap[provider] || provider; - }; + const getProviderDisplayName = () => { + const nameMap: { [key: string]: string } = { + google_drive: "Google Drive", + onedrive: "OneDrive", + sharepoint: "SharePoint", + }; + return nameMap[provider] || provider; + }; - if (isLoading) { - return ( -
-
-
-
-

Loading {getProviderDisplayName()} connector...

-
-
-
- ); - } + if (isLoading) { + return ( +
+
+
+
+

Loading {getProviderDisplayName()} connector...

+
+
+
+ ); + } - if (error || !connector) { - return ( -
-
- -
+ if (error || !connector) { + return ( +
+
+ +
-
-
- -

- Provider Not Available -

-

{error}

- -
-
-
- ); - } +
+
+ +

+ Provider Not Available +

+

{error}

+ +
+
+
+ ); + } - if (connector.status !== "connected") { - return ( -
-
- -
+ if (connector.status !== "connected") { + return ( +
+
+ +
-
-
- -

- {connector.name} Not Connected -

-

- You need to connect your {connector.name} account before you can - select files. -

- -
-
-
- ); - } +
+
+ +

+ {connector.name} Not Connected +

+

+ You need to connect your {connector.name} account before you can + select files. +

+ +
+
+
+ ); + } - if (!connector.hasAccessToken) { - return ( -
-
- -
+ if (!connector.hasAccessToken) { + return ( +
+
+ +
-
-
- -

- Access Token Required -

-

- {connector.accessTokenError || - `Unable to get access token for ${connector.name}. Try reconnecting your account.`} -

- -
-
-
- ); - } +
+
+ +

+ Access Token Required +

+

+ {connector.accessTokenError || + `Unable to get access token for ${connector.name}. Try reconnecting your account.`} +

+ +
+
+
+ ); + } - return ( -
-
- -

- Add from {getProviderDisplayName()} -

-
+ return ( +
+
+ +

+ Add from {getProviderDisplayName()} +

+
-
- -
+
+ +
-
-
- - -
-
-
- ); +
+
+ + +
+
+
+ ); } diff --git a/frontend/src/components/cloud-picker/picker-header.tsx b/frontend/src/components/cloud-picker/picker-header.tsx index 05dcaebd..e0d9cfa4 100644 --- a/frontend/src/components/cloud-picker/picker-header.tsx +++ b/frontend/src/components/cloud-picker/picker-header.tsx @@ -51,19 +51,13 @@ export const PickerHeader = ({ Select files from {getProviderName(provider)} to ingest.

-
- csv, json, pdf,{" "} - +16 more{" "} - 150 MB max -
); From 088ed2697b5eadf5802cf8b39e2e8c71c93c03e1 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 12:00:13 -0600 Subject: [PATCH 20/55] updates from sync --- frontend/src/app/knowledge/chunks/page.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index bbb7f019..85a4e3f2 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -161,7 +161,7 @@ function ChunksPageContent() {
-
+
{selectedFilter?.name && (
setQueryInputText(e.target.value)} value={queryInputText} />
-
+ {/*
Select all -
+
*/}
@@ -269,6 +269,10 @@ function ChunksPageContent() {
+ + {chunk.score.toFixed(2)} score + + {/* TODO: Update to use active toggle */} {/* Date: Fri, 3 Oct 2025 12:12:47 -0600 Subject: [PATCH 21/55] remove checkboxes --- frontend/src/app/knowledge/chunks/page.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index 85a4e3f2..cde2c3e8 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -240,14 +240,14 @@ function ChunksPageContent() { >
-
+ {/*
handleChunkCardCheckboxChange(index) } /> -
+
*/} Chunk {chunk.index} @@ -282,7 +282,7 @@ function ChunksPageContent() { Active */}
-
+
{chunk.text}
From 589839fedc1199e6060e0b57e9bd9ee08d8081b1 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 15:34:54 -0300 Subject: [PATCH 22/55] implement duplicate handling on backend and frontend --- .../components/duplicate-handling-dialog.tsx | 71 ++++++ frontend/components/knowledge-dropdown.tsx | 233 ++++++++++++------ frontend/src/contexts/task-context.tsx | 3 +- src/api/documents.py | 92 +++++-- src/api/router.py | 13 +- src/main.py | 11 + src/models/processors.py | 139 +++++++++-- src/models/tasks.py | 3 +- src/services/task_service.py | 23 +- 9 files changed, 470 insertions(+), 118 deletions(-) create mode 100644 frontend/components/duplicate-handling-dialog.tsx diff --git a/frontend/components/duplicate-handling-dialog.tsx b/frontend/components/duplicate-handling-dialog.tsx new file mode 100644 index 00000000..2f92ea50 --- /dev/null +++ b/frontend/components/duplicate-handling-dialog.tsx @@ -0,0 +1,71 @@ +"use client"; + +import { RotateCcw } from "lucide-react"; +import type React from "react"; +import { Button } from "./ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "./ui/dialog"; + +interface DuplicateHandlingDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + filename: string; + onOverwrite: () => void | Promise; + isLoading?: boolean; +} + +export const DuplicateHandlingDialog: React.FC< + DuplicateHandlingDialogProps +> = ({ open, onOpenChange, filename, onOverwrite, isLoading = false }) => { + const handleOverwrite = async () => { + try { + await onOverwrite(); + onOpenChange(false); + } catch (error) { + // Error handling is done by the parent component + } + }; + + return ( + + + + Overwrite document + + Overwriting will replace the existing document with another version. + This can't be undone. + + + + + + + + + + ); +}; diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index 9b71ee81..0b106360 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -13,6 +13,7 @@ import { useRouter } from "next/navigation"; import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery"; +import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -41,6 +42,7 @@ export function KnowledgeDropdown({ const [isOpen, setIsOpen] = useState(false); const [showFolderDialog, setShowFolderDialog] = useState(false); const [showS3Dialog, setShowS3Dialog] = useState(false); + const [showDuplicateDialog, setShowDuplicateDialog] = useState(false); const [awsEnabled, setAwsEnabled] = useState(false); const [folderPath, setFolderPath] = useState("/app/documents/"); const [bucketUrl, setBucketUrl] = useState("s3://"); @@ -48,6 +50,8 @@ export function KnowledgeDropdown({ const [s3Loading, setS3Loading] = useState(false); const [fileUploading, setFileUploading] = useState(false); const [isNavigatingToCloud, setIsNavigatingToCloud] = useState(false); + const [pendingFile, setPendingFile] = useState(null); + const [duplicateFilename, setDuplicateFilename] = useState(""); const [cloudConnectors, setCloudConnectors] = useState<{ [key: string]: { name: string; @@ -168,101 +172,52 @@ export function KnowledgeDropdown({ const handleFileChange = async (e: React.ChangeEvent) => { const files = e.target.files; if (files && files.length > 0) { - // Close dropdown and disable button immediately after file selection - setIsOpen(false); - setFileUploading(true); + const file = files[0]; - // Trigger the same file upload event as the chat page - window.dispatchEvent( - new CustomEvent("fileUploadStart", { - detail: { filename: files[0].name }, - }), - ); + // Close dropdown immediately after file selection + setIsOpen(false); try { - const formData = new FormData(); - formData.append("file", files[0]); + // Check if filename already exists (using ORIGINAL filename) + console.log("[Duplicate Check] Checking file:", file.name); + const checkResponse = await fetch( + `/api/documents/check-filename?filename=${encodeURIComponent(file.name)}`, + ); - // Use router upload and ingest endpoint (automatically routes based on configuration) - const uploadIngestRes = await fetch("/api/router/upload_ingest", { - method: "POST", - body: formData, - }); + console.log("[Duplicate Check] Response status:", checkResponse.status); - const uploadIngestJson = await uploadIngestRes.json(); - - if (!uploadIngestRes.ok) { + if (!checkResponse.ok) { + const errorText = await checkResponse.text(); + console.error("[Duplicate Check] Error response:", errorText); throw new Error( - uploadIngestJson?.error || "Upload and ingest failed", + `Failed to check duplicates: ${checkResponse.statusText}`, ); } - // Extract results from the response - handle both unified and simple formats - const fileId = uploadIngestJson?.upload?.id || uploadIngestJson?.id || uploadIngestJson?.task_id; - const filePath = - uploadIngestJson?.upload?.path || - uploadIngestJson?.path || - "uploaded"; - const runJson = uploadIngestJson?.ingestion; - const deleteResult = uploadIngestJson?.deletion; - console.log("c", uploadIngestJson ) - if (!fileId) { - throw new Error("Upload successful but no file id returned"); - } - // Check if ingestion actually succeeded - if ( - runJson && - runJson.status !== "COMPLETED" && - runJson.status !== "SUCCESS" - ) { - const errorMsg = runJson.error || "Ingestion pipeline failed"; - throw new Error( - `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.`, - ); - } - // Log deletion status if provided - if (deleteResult) { - if (deleteResult.status === "deleted") { - console.log( - "File successfully cleaned up from Langflow:", - deleteResult.file_id, - ); - } else if (deleteResult.status === "delete_failed") { - console.warn( - "Failed to cleanup file from Langflow:", - deleteResult.error, - ); + const checkData = await checkResponse.json(); + console.log("[Duplicate Check] Result:", checkData); + + if (checkData.exists) { + // Show duplicate handling dialog + console.log("[Duplicate Check] Duplicate detected, showing dialog"); + setPendingFile(file); + setDuplicateFilename(file.name); + setShowDuplicateDialog(true); + // Reset file input + if (fileInputRef.current) { + fileInputRef.current.value = ""; } + return; } - // Notify UI - window.dispatchEvent( - new CustomEvent("fileUploaded", { - detail: { - file: files[0], - result: { - file_id: fileId, - file_path: filePath, - run: runJson, - deletion: deleteResult, - unified: true, - }, - }, - }), - ); - refetchTasks(); + // No duplicate, proceed with upload + console.log("[Duplicate Check] No duplicate, proceeding with upload"); + await uploadFile(file, false); } catch (error) { - window.dispatchEvent( - new CustomEvent("fileUploadError", { - detail: { - filename: files[0].name, - error: error instanceof Error ? error.message : "Upload failed", - }, - }), - ); - } finally { - window.dispatchEvent(new CustomEvent("fileUploadComplete")); - setFileUploading(false); + console.error("[Duplicate Check] Exception:", error); + toast.error("Failed to check for duplicates", { + description: error instanceof Error ? error.message : "Unknown error", + }); } } @@ -272,6 +227,111 @@ export function KnowledgeDropdown({ } }; + const uploadFile = async (file: File, replace: boolean) => { + setFileUploading(true); + + // Trigger the same file upload event as the chat page + window.dispatchEvent( + new CustomEvent("fileUploadStart", { + detail: { filename: file.name }, + }), + ); + + try { + const formData = new FormData(); + formData.append("file", file); + formData.append("replace_duplicates", replace.toString()); + + // Use router upload and ingest endpoint (automatically routes based on configuration) + const uploadIngestRes = await fetch("/api/router/upload_ingest", { + method: "POST", + body: formData, + }); + + const uploadIngestJson = await uploadIngestRes.json(); + + if (!uploadIngestRes.ok) { + throw new Error(uploadIngestJson?.error || "Upload and ingest failed"); + } + + // Extract results from the response - handle both unified and simple formats + const fileId = + uploadIngestJson?.upload?.id || + uploadIngestJson?.id || + uploadIngestJson?.task_id; + const filePath = + uploadIngestJson?.upload?.path || uploadIngestJson?.path || "uploaded"; + const runJson = uploadIngestJson?.ingestion; + const deleteResult = uploadIngestJson?.deletion; + console.log("c", uploadIngestJson); + if (!fileId) { + throw new Error("Upload successful but no file id returned"); + } + // Check if ingestion actually succeeded + if ( + runJson && + runJson.status !== "COMPLETED" && + runJson.status !== "SUCCESS" + ) { + const errorMsg = runJson.error || "Ingestion pipeline failed"; + throw new Error( + `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.`, + ); + } + // Log deletion status if provided + if (deleteResult) { + if (deleteResult.status === "deleted") { + console.log( + "File successfully cleaned up from Langflow:", + deleteResult.file_id, + ); + } else if (deleteResult.status === "delete_failed") { + console.warn( + "Failed to cleanup file from Langflow:", + deleteResult.error, + ); + } + } + // Notify UI + window.dispatchEvent( + new CustomEvent("fileUploaded", { + detail: { + file: file, + result: { + file_id: fileId, + file_path: filePath, + run: runJson, + deletion: deleteResult, + unified: true, + }, + }, + }), + ); + + refetchTasks(); + } catch (error) { + window.dispatchEvent( + new CustomEvent("fileUploadError", { + detail: { + filename: file.name, + error: error instanceof Error ? error.message : "Upload failed", + }, + }), + ); + } finally { + window.dispatchEvent(new CustomEvent("fileUploadComplete")); + setFileUploading(false); + } + }; + + const handleOverwriteFile = async () => { + if (pendingFile) { + await uploadFile(pendingFile, true); + setPendingFile(null); + setDuplicateFilename(""); + } + }; + const handleFolderUpload = async () => { if (!folderPath.trim()) return; @@ -611,6 +671,15 @@ export function KnowledgeDropdown({ + + {/* Duplicate Handling Dialog */} + ); } diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index 8928ae8f..26e8ca00 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -135,7 +135,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { taskFileEntries.forEach(([filePath, fileInfo]) => { if (typeof fileInfo === "object" && fileInfo) { - const fileName = filePath.split("/").pop() || filePath; + // Use the filename from backend if available, otherwise extract from path + const fileName = (fileInfo as any).filename || filePath.split("/").pop() || filePath; const fileStatus = fileInfo.status as string; // Map backend file status to our TaskFile status diff --git a/src/api/documents.py b/src/api/documents.py index 82afb349..a367c9fe 100644 --- a/src/api/documents.py +++ b/src/api/documents.py @@ -6,14 +6,13 @@ from config.settings import INDEX_NAME logger = get_logger(__name__) -async def delete_documents_by_filename(request: Request, document_service, session_manager): - """Delete all documents with a specific filename""" - data = await request.json() - filename = data.get("filename") - +async def check_filename_exists(request: Request, document_service, session_manager): + """Check if a document with a specific filename already exists""" + filename = request.query_params.get("filename") + if not filename: - return JSONResponse({"error": "filename is required"}, status_code=400) - + return JSONResponse({"error": "filename parameter is required"}, status_code=400) + user = request.state.user jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token) @@ -22,34 +21,99 @@ async def delete_documents_by_filename(request: Request, document_service, sessi opensearch_client = session_manager.get_user_opensearch_client( user.user_id, jwt_token ) - + + # Search for any document with this exact filename + # Try both .keyword (exact match) and regular field (analyzed match) + search_body = { + "query": { + "bool": { + "should": [ + {"term": {"filename.keyword": filename}}, + {"term": {"filename": filename}} + ], + "minimum_should_match": 1 + } + }, + "size": 1, + "_source": ["filename"] + } + + logger.debug(f"Checking filename existence: {filename}") + + response = await opensearch_client.search( + index=INDEX_NAME, + body=search_body + ) + + # Check if any hits were found + hits = response.get("hits", {}).get("hits", []) + exists = len(hits) > 0 + + logger.debug(f"Filename check result - exists: {exists}, hits: {len(hits)}") + + return JSONResponse({ + "exists": exists, + "filename": filename + }, status_code=200) + + except Exception as e: + logger.error("Error checking filename existence", filename=filename, error=str(e)) + error_str = str(e) + if "AuthenticationException" in error_str: + return JSONResponse({"error": "Access denied: insufficient permissions"}, status_code=403) + else: + return JSONResponse({"error": str(e)}, status_code=500) + + +async def delete_documents_by_filename(request: Request, document_service, session_manager): + """Delete all documents with a specific filename""" + data = await request.json() + filename = data.get("filename") + + if not filename: + return JSONResponse({"error": "filename is required"}, status_code=400) + + user = request.state.user + jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token) + + try: + # Get user's OpenSearch client + opensearch_client = session_manager.get_user_opensearch_client( + user.user_id, jwt_token + ) + # Delete by query to remove all chunks of this document + # Use both .keyword and regular field to ensure we catch all variations delete_query = { "query": { "bool": { - "must": [ + "should": [ + {"term": {"filename.keyword": filename}}, {"term": {"filename": filename}} - ] + ], + "minimum_should_match": 1 } } } - + + logger.debug(f"Deleting documents with filename: {filename}") + result = await opensearch_client.delete_by_query( index=INDEX_NAME, body=delete_query, conflicts="proceed" ) - + deleted_count = result.get("deleted", 0) logger.info(f"Deleted {deleted_count} chunks for filename {filename}", user_id=user.user_id) - + return JSONResponse({ "success": True, "deleted_chunks": deleted_count, "filename": filename, "message": f"All documents with filename '{filename}' deleted successfully" }, status_code=200) - + except Exception as e: logger.error("Error deleting documents by filename", filename=filename, error=str(e)) error_str = str(e) diff --git a/src/api/router.py b/src/api/router.py index 23ce5bdf..327757be 100644 --- a/src/api/router.py +++ b/src/api/router.py @@ -77,6 +77,7 @@ async def langflow_upload_ingest_task( settings_json = form.get("settings") tweaks_json = form.get("tweaks") delete_after_ingest = form.get("delete_after_ingest", "true").lower() == "true" + replace_duplicates = form.get("replace_duplicates", "false").lower() == "true" # Parse JSON fields if provided settings = None @@ -112,7 +113,8 @@ async def langflow_upload_ingest_task( import tempfile import os temp_file_paths = [] - + original_filenames = [] + try: # Create temp directory reference once temp_dir = tempfile.gettempdir() @@ -121,8 +123,11 @@ async def langflow_upload_ingest_task( # Read file content content = await upload_file.read() - # Create temporary file with the actual filename (not a temp prefix) - # Store in temp directory but use the real filename + # Store ORIGINAL filename (not transformed) + original_filenames.append(upload_file.filename) + + # Create temporary file with TRANSFORMED filename for filesystem safety + # Transform: spaces and / to underscore safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_") temp_path = os.path.join(temp_dir, safe_filename) @@ -153,6 +158,7 @@ async def langflow_upload_ingest_task( task_id = await task_service.create_langflow_upload_task( user_id=user_id, file_paths=temp_file_paths, + original_filenames=original_filenames, langflow_file_service=langflow_file_service, session_manager=session_manager, jwt_token=jwt_token, @@ -162,6 +168,7 @@ async def langflow_upload_ingest_task( tweaks=tweaks, settings=settings, delete_after_ingest=delete_after_ingest, + replace_duplicates=replace_duplicates, ) logger.debug("Langflow upload task created successfully", task_id=task_id) diff --git a/src/main.py b/src/main.py index 230ded79..bf6da342 100644 --- a/src/main.py +++ b/src/main.py @@ -953,6 +953,17 @@ async def create_app(): methods=["POST", "GET"], ), # Document endpoints + Route( + "/documents/check-filename", + require_auth(services["session_manager"])( + partial( + documents.check_filename_exists, + document_service=services["document_service"], + session_manager=services["session_manager"], + ) + ), + methods=["GET"], + ), Route( "/documents/delete-by-filename", require_auth(services["session_manager"])( diff --git a/src/models/processors.py b/src/models/processors.py index a1d72777..6d7b74b4 100644 --- a/src/models/processors.py +++ b/src/models/processors.py @@ -55,6 +55,108 @@ class TaskProcessor: await asyncio.sleep(retry_delay) retry_delay *= 2 # Exponential backoff + async def check_filename_exists( + self, + filename: str, + opensearch_client, + ) -> bool: + """ + Check if a document with the given filename already exists in OpenSearch. + Returns True if any chunks with this filename exist. + """ + from config.settings import INDEX_NAME + import asyncio + + max_retries = 3 + retry_delay = 1.0 + + for attempt in range(max_retries): + try: + # Search for any document with this exact filename + search_body = { + "query": { + "term": { + "filename.keyword": filename + } + }, + "size": 1, + "_source": False + } + + response = await opensearch_client.search( + index=INDEX_NAME, + body=search_body + ) + + # Check if any hits were found + hits = response.get("hits", {}).get("hits", []) + return len(hits) > 0 + + except (asyncio.TimeoutError, Exception) as e: + if attempt == max_retries - 1: + logger.error( + "OpenSearch filename check failed after retries", + filename=filename, + error=str(e), + attempt=attempt + 1 + ) + # On final failure, assume document doesn't exist (safer to reprocess than skip) + logger.warning( + "Assuming filename doesn't exist due to connection issues", + filename=filename + ) + return False + else: + logger.warning( + "OpenSearch filename check failed, retrying", + filename=filename, + error=str(e), + attempt=attempt + 1, + retry_in=retry_delay + ) + await asyncio.sleep(retry_delay) + retry_delay *= 2 # Exponential backoff + + async def delete_document_by_filename( + self, + filename: str, + opensearch_client, + ) -> None: + """ + Delete all chunks of a document with the given filename from OpenSearch. + """ + from config.settings import INDEX_NAME + + try: + # Delete all documents with this filename + delete_body = { + "query": { + "term": { + "filename.keyword": filename + } + } + } + + response = await opensearch_client.delete_by_query( + index=INDEX_NAME, + body=delete_body + ) + + deleted_count = response.get("deleted", 0) + logger.info( + "Deleted existing document chunks", + filename=filename, + deleted_count=deleted_count + ) + + except Exception as e: + logger.error( + "Failed to delete existing document", + filename=filename, + error=str(e) + ) + raise + async def process_document_standard( self, file_path: str, @@ -527,6 +629,7 @@ class LangflowFileProcessor(TaskProcessor): tweaks: dict = None, settings: dict = None, delete_after_ingest: bool = True, + replace_duplicates: bool = False, ): super().__init__() self.langflow_file_service = langflow_file_service @@ -539,6 +642,7 @@ class LangflowFileProcessor(TaskProcessor): self.tweaks = tweaks or {} self.settings = settings self.delete_after_ingest = delete_after_ingest + self.replace_duplicates = replace_duplicates async def process_item( self, upload_task: UploadTask, item: str, file_task: FileTask @@ -554,33 +658,40 @@ class LangflowFileProcessor(TaskProcessor): file_task.updated_at = time.time() try: - # Compute hash and check if already exists - from utils.hash_utils import hash_id - file_hash = hash_id(item) + # Use the ORIGINAL filename stored in file_task (not the transformed temp path) + # This ensures we check/store the original filename with spaces, etc. + original_filename = file_task.filename or os.path.basename(item) - # Check if document already exists + # Check if document with same filename already exists opensearch_client = self.session_manager.get_user_opensearch_client( self.owner_user_id, self.jwt_token ) - if await self.check_document_exists(file_hash, opensearch_client): - file_task.status = TaskStatus.COMPLETED - file_task.result = {"status": "unchanged", "id": file_hash} + + filename_exists = await self.check_filename_exists(original_filename, opensearch_client) + + if filename_exists and not self.replace_duplicates: + # Duplicate exists and user hasn't confirmed replacement + file_task.status = TaskStatus.FAILED + file_task.error = f"File with name '{original_filename}' already exists" file_task.updated_at = time.time() - upload_task.successful_files += 1 + upload_task.failed_files += 1 return + elif filename_exists and self.replace_duplicates: + # Delete existing document before uploading new one + logger.info(f"Replacing existing document: {original_filename}") + await self.delete_document_by_filename(original_filename, opensearch_client) # Read file content for processing with open(item, 'rb') as f: content = f.read() - # Create file tuple for upload - # The temp file now has the actual filename, no need to extract it - filename = os.path.basename(item) - content_type, _ = mimetypes.guess_type(filename) + # Create file tuple for upload using ORIGINAL filename + # This ensures the document is indexed with the original name + content_type, _ = mimetypes.guess_type(original_filename) if not content_type: content_type = 'application/octet-stream' - - file_tuple = (filename, content, content_type) + + file_tuple = (original_filename, content, content_type) # Get JWT token using same logic as DocumentFileProcessor # This will handle anonymous JWT creation if needed diff --git a/src/models/tasks.py b/src/models/tasks.py index 236927ab..253cabb5 100644 --- a/src/models/tasks.py +++ b/src/models/tasks.py @@ -20,7 +20,8 @@ class FileTask: retry_count: int = 0 created_at: float = field(default_factory=time.time) updated_at: float = field(default_factory=time.time) - + filename: Optional[str] = None # Original filename for display + @property def duration_seconds(self) -> float: """Duration in seconds from creation to last update""" diff --git a/src/services/task_service.py b/src/services/task_service.py index be5312a0..eb5825c0 100644 --- a/src/services/task_service.py +++ b/src/services/task_service.py @@ -59,6 +59,7 @@ class TaskService: file_paths: list, langflow_file_service, session_manager, + original_filenames: list = None, jwt_token: str = None, owner_name: str = None, owner_email: str = None, @@ -66,6 +67,7 @@ class TaskService: tweaks: dict = None, settings: dict = None, delete_after_ingest: bool = True, + replace_duplicates: bool = False, ) -> str: """Create a new upload task for Langflow file processing with upload and ingest""" # Use LangflowFileProcessor with user context @@ -82,18 +84,31 @@ class TaskService: tweaks=tweaks, settings=settings, delete_after_ingest=delete_after_ingest, + replace_duplicates=replace_duplicates, ) - return await self.create_custom_task(user_id, file_paths, processor) + return await self.create_custom_task(user_id, file_paths, processor, original_filenames) - async def create_custom_task(self, user_id: str, items: list, processor) -> str: + async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: list = None) -> str: """Create a new task with custom processor for any type of items""" + import os # Store anonymous tasks under a stable key so they can be retrieved later store_user_id = user_id or AnonymousUser().user_id task_id = str(uuid.uuid4()) + + # Create file tasks with original filenames if provided + file_tasks = {} + for i, item in enumerate(items): + if original_filenames and i < len(original_filenames): + filename = original_filenames[i] + else: + filename = os.path.basename(str(item)) + + file_tasks[str(item)] = FileTask(file_path=str(item), filename=filename) + upload_task = UploadTask( task_id=task_id, total_files=len(items), - file_tasks={str(item): FileTask(file_path=str(item)) for item in items}, + file_tasks=file_tasks, ) # Attach the custom processor to the task @@ -268,6 +283,7 @@ class TaskService: "created_at": file_task.created_at, "updated_at": file_task.updated_at, "duration_seconds": file_task.duration_seconds, + "filename": file_task.filename, } # Count running and pending files @@ -322,6 +338,7 @@ class TaskService: "created_at": file_task.created_at, "updated_at": file_task.updated_at, "duration_seconds": file_task.duration_seconds, + "filename": file_task.filename, } if file_task.status.value == "running": From fb29f72598aad139e4c0623d861094a18e7b8fba Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 15:39:51 -0300 Subject: [PATCH 23/55] update task toast --- frontend/src/contexts/task-context.tsx | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index 26e8ca00..d25130f7 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -205,9 +205,19 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { previousTask && previousTask.status !== "completed" && currentTask.status === "completed" ) { - // Task just completed - show success toast - toast.success("Task completed successfully", { - description: `Task ${currentTask.task_id} has finished processing.`, + // Task just completed - show success toast with file counts + const successfulFiles = currentTask.successful_files || 0; + const failedFiles = currentTask.failed_files || 0; + + let description = ""; + if (failedFiles > 0) { + description = `${successfulFiles} file${successfulFiles !== 1 ? 's' : ''} uploaded successfully, ${failedFiles} file${failedFiles !== 1 ? 's' : ''} failed`; + } else { + description = `${successfulFiles} file${successfulFiles !== 1 ? 's' : ''} uploaded successfully`; + } + + toast.success("Task completed", { + description, action: { label: "View", onClick: () => console.log("View task", currentTask.task_id), From 9336aa287e87032bd7385e3824e548df5de26f0e Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:12:09 -0300 Subject: [PATCH 24/55] fixed lint --- frontend/src/app/knowledge/page.tsx | 110 ++++++++++++++++++---------- 1 file changed, 71 insertions(+), 39 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index c6d254c4..9e933903 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -1,10 +1,15 @@ "use client"; -import type { ColDef } from "ag-grid-community"; +import type { ColDef, GetRowIdParams } from "ag-grid-community"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; import { Building2, Cloud, HardDrive, Search, Trash2, X } from "lucide-react"; import { useRouter } from "next/navigation"; -import { type ChangeEvent, useCallback, useRef, useState } from "react"; +import { + type ChangeEvent, + useCallback, + useRef, + useState, +} from "react"; import { SiGoogledrive } from "react-icons/si"; import { TbBrandOnedrive } from "react-icons/tb"; import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; @@ -54,15 +59,10 @@ function SearchPage() { const deleteDocumentMutation = useDeleteDocument(); - const { data = [], isFetching } = useGetSearchQuery( + const { data: searchData = [], isFetching } = useGetSearchQuery( parsedFilterData?.query || "*", parsedFilterData, ); - - const handleTableSearch = (e: ChangeEvent) => { - gridRef.current?.api.setGridOption("quickFilterText", e.target.value); - }; - // Convert TaskFiles to File format and merge with backend results const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { return { @@ -75,60 +75,91 @@ function SearchPage() { }; }); - const backendFiles = (data as File[]).filter((file) => !taskFilesAsFiles.some((taskFile) => taskFile.filename === file.filename && taskFile.status === "processing")); + // Create a map of task files by filename for quick lookup + const taskFileMap = new Map( + taskFilesAsFiles.map((file) => [file.filename, file]), + ); + + // Override backend files with task file status if they exist + const backendFiles = (searchData as File[]) + .map((file) => { + const taskFile = taskFileMap.get(file.filename); + if (taskFile) { + // Override backend file with task file data (includes status) + return { ...file, ...taskFile }; + } + return file; + }) + .filter((file) => { + // Only filter out files that are currently processing AND in taskFiles + const taskFile = taskFileMap.get(file.filename); + return !taskFile || taskFile.status !== "processing"; + }); const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { return ( taskFile.status !== "active" && !backendFiles.some( - (backendFile) => backendFile.filename === taskFile.filename,) + (backendFile) => backendFile.filename === taskFile.filename, + ) ); }); // Combine task files first, then backend files const fileResults = [...backendFiles, ...filteredTaskFiles]; + const handleTableSearch = (e: ChangeEvent) => { + gridRef.current?.api.setGridOption("quickFilterText", e.target.value); + }; + const gridRef = useRef(null); const columnDefs = [ { field: "filename", headerName: "Source", - checkboxSelection: (data) => (data?.data?.status || "active") === "active", + checkboxSelection: (params: CustomCellRendererProps) => + (params?.data?.status || "active") === "active", headerCheckboxSelection: true, initialFlex: 2, minWidth: 220, cellRenderer: ({ data, value }: CustomCellRendererProps) => { + // Read status directly from data on each render + const status = data?.status || "active"; + const isActive = status === "active"; + console.log(data?.filename, status, "a"); return ( -
{((data?.status || "active") !== "active") && -
- } -
+
+
+ +
); }, }, { field: "size", headerName: "Size", - valueFormatter: (params) => + valueFormatter: (params: CustomCellRendererProps) => params.value ? `${Math.round(params.value / 1024)} KB` : "-", }, { @@ -138,13 +169,13 @@ function SearchPage() { { field: "owner", headerName: "Owner", - valueFormatter: (params) => + valueFormatter: (params: CustomCellRendererProps) => params.data?.owner_name || params.data?.owner_email || "—", }, { field: "chunkCount", headerName: "Chunks", - valueFormatter: (params) => params.data?.chunkCount?.toString() || "-", + valueFormatter: (params: CustomCellRendererProps) => params.data?.chunkCount?.toString() || "-", }, { field: "avgScore", @@ -162,6 +193,7 @@ function SearchPage() { field: "status", headerName: "Status", cellRenderer: ({ data }: CustomCellRendererProps) => { + console.log(data?.filename, data?.status, "b"); // Default to 'active' status if no status is provided const status = data?.status || "active"; return ; @@ -188,7 +220,7 @@ function SearchPage() { resizable: false, sortable: false, initialFlex: 0, - } + }, ]; const defaultColDef: ColDef = { @@ -323,7 +355,7 @@ function SearchPage() { []} defaultColDef={defaultColDef} loading={isFetching} ref={gridRef} @@ -331,7 +363,7 @@ function SearchPage() { rowSelection="multiple" rowMultiSelectWithClick={false} suppressRowClickSelection={true} - getRowId={(params) => params.data.filename} + getRowId={(params: GetRowIdParams) => params.data?.filename} domLayout="normal" onSelectionChanged={onSelectionChanged} noRowsOverlayComponent={() => ( From ca9bee8222bd615501c90e8e7f01913243c3d08d Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:12:18 -0300 Subject: [PATCH 25/55] removed file from query when overwriting --- frontend/components/knowledge-dropdown.tsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index 0b106360..7fe84259 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -1,5 +1,6 @@ "use client"; +import { useQueryClient } from "@tanstack/react-query"; import { ChevronDown, Cloud, @@ -26,6 +27,7 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { useTask } from "@/contexts/task-context"; import { cn } from "@/lib/utils"; +import type { File as SearchFile } from "@/src/app/api/queries/useGetSearchQuery"; interface KnowledgeDropdownProps { active?: boolean; @@ -38,6 +40,7 @@ export function KnowledgeDropdown({ }: KnowledgeDropdownProps) { const { addTask } = useTask(); const { refetch: refetchTasks } = useGetTasksQuery(); + const queryClient = useQueryClient(); const router = useRouter(); const [isOpen, setIsOpen] = useState(false); const [showFolderDialog, setShowFolderDialog] = useState(false); @@ -326,6 +329,15 @@ export function KnowledgeDropdown({ const handleOverwriteFile = async () => { if (pendingFile) { + // Remove the old file from all search query caches before overwriting + queryClient.setQueriesData({ queryKey: ["search"] }, (oldData: []) => { + if (!oldData) return oldData; + // Filter out the file that's being overwritten + return oldData.filter( + (file: SearchFile) => file.filename !== pendingFile.name, + ); + }); + await uploadFile(pendingFile, true); setPendingFile(null); setDuplicateFilename(""); @@ -676,7 +688,6 @@ export function KnowledgeDropdown({ From 4de9a0d085848abbab76db05b9f06c9d3d81198f Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:12:24 -0300 Subject: [PATCH 26/55] removed unused prop --- frontend/components/duplicate-handling-dialog.tsx | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/frontend/components/duplicate-handling-dialog.tsx b/frontend/components/duplicate-handling-dialog.tsx index 2f92ea50..d5cb2edf 100644 --- a/frontend/components/duplicate-handling-dialog.tsx +++ b/frontend/components/duplicate-handling-dialog.tsx @@ -15,21 +15,16 @@ import { interface DuplicateHandlingDialogProps { open: boolean; onOpenChange: (open: boolean) => void; - filename: string; onOverwrite: () => void | Promise; isLoading?: boolean; } export const DuplicateHandlingDialog: React.FC< DuplicateHandlingDialogProps -> = ({ open, onOpenChange, filename, onOverwrite, isLoading = false }) => { +> = ({ open, onOpenChange, onOverwrite, isLoading = false }) => { const handleOverwrite = async () => { - try { - await onOverwrite(); - onOpenChange(false); - } catch (error) { - // Error handling is done by the parent component - } + await onOverwrite(); + onOpenChange(false); }; return ( From 1fdb251a47a7168f9431e00c8f714a9e088db09a Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:15:32 -0300 Subject: [PATCH 27/55] UI minor tweaks --- frontend/components/ui/input.tsx | 2 +- frontend/src/app/knowledge/chunks/page.tsx | 626 +++++++++++---------- frontend/src/app/knowledge/page.tsx | 3 +- 3 files changed, 316 insertions(+), 315 deletions(-) diff --git a/frontend/components/ui/input.tsx b/frontend/components/ui/input.tsx index ffcda454..86e638a1 100644 --- a/frontend/components/ui/input.tsx +++ b/frontend/components/ui/input.tsx @@ -44,7 +44,7 @@ const Input = React.forwardRef( placeholder={placeholder} className={cn( "primary-input", - icon && "pl-9", + icon && "!pl-9", type === "password" && "!pr-8", icon ? inputClassName : className )} diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index cb96eddc..7d49a56e 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -1,289 +1,291 @@ "use client"; import { ArrowLeft, Check, Copy, Loader2, Search } from "lucide-react"; -import { Suspense, useCallback, useEffect, useMemo, useState } from "react"; import { useRouter, useSearchParams } from "next/navigation"; +import { Suspense, useCallback, useEffect, useMemo, useState } from "react"; import { ProtectedRoute } from "@/components/protected-route"; import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; import { useTask } from "@/contexts/task-context"; import { - type ChunkResult, - type File, - useGetSearchQuery, + type ChunkResult, + type File, + useGetSearchQuery, } from "../../api/queries/useGetSearchQuery"; -import { Label } from "@/components/ui/label"; -import { Checkbox } from "@/components/ui/checkbox"; -import { Input } from "@/components/ui/input"; const getFileTypeLabel = (mimetype: string) => { - if (mimetype === "application/pdf") return "PDF"; - if (mimetype === "text/plain") return "Text"; - if (mimetype === "application/msword") return "Word Document"; - return "Unknown"; + if (mimetype === "application/pdf") return "PDF"; + if (mimetype === "text/plain") return "Text"; + if (mimetype === "application/msword") return "Word Document"; + return "Unknown"; }; function ChunksPageContent() { - const router = useRouter(); - const searchParams = useSearchParams(); - const { isMenuOpen } = useTask(); - const { parsedFilterData, isPanelOpen } = useKnowledgeFilter(); + const router = useRouter(); + const searchParams = useSearchParams(); + const { isMenuOpen } = useTask(); + const { parsedFilterData, isPanelOpen } = useKnowledgeFilter(); - const filename = searchParams.get("filename"); - const [chunks, setChunks] = useState([]); - const [chunksFilteredByQuery, setChunksFilteredByQuery] = useState< - ChunkResult[] - >([]); - const [selectedChunks, setSelectedChunks] = useState>(new Set()); - const [activeCopiedChunkIndex, setActiveCopiedChunkIndex] = useState< - number | null - >(null); + const filename = searchParams.get("filename"); + const [chunks, setChunks] = useState([]); + const [chunksFilteredByQuery, setChunksFilteredByQuery] = useState< + ChunkResult[] + >([]); + const [selectedChunks, setSelectedChunks] = useState>(new Set()); + const [activeCopiedChunkIndex, setActiveCopiedChunkIndex] = useState< + number | null + >(null); - // Calculate average chunk length - const averageChunkLength = useMemo( - () => - chunks.reduce((acc, chunk) => acc + chunk.text.length, 0) / - chunks.length || 0, - [chunks] - ); + // Calculate average chunk length + const averageChunkLength = useMemo( + () => + chunks.reduce((acc, chunk) => acc + chunk.text.length, 0) / + chunks.length || 0, + [chunks], + ); - const [selectAll, setSelectAll] = useState(false); - const [queryInputText, setQueryInputText] = useState( - parsedFilterData?.query ?? "" - ); + const [selectAll, setSelectAll] = useState(false); + const [queryInputText, setQueryInputText] = useState( + parsedFilterData?.query ?? "", + ); - // Use the same search query as the knowledge page, but we'll filter for the specific file - const { data = [], isFetching } = useGetSearchQuery("*", parsedFilterData); + // Use the same search query as the knowledge page, but we'll filter for the specific file + const { data = [], isFetching } = useGetSearchQuery("*", parsedFilterData); - useEffect(() => { - if (queryInputText === "") { - setChunksFilteredByQuery(chunks); - } else { - setChunksFilteredByQuery( - chunks.filter((chunk) => - chunk.text.toLowerCase().includes(queryInputText.toLowerCase()) - ) - ); - } - }, [queryInputText, chunks]); + useEffect(() => { + if (queryInputText === "") { + setChunksFilteredByQuery(chunks); + } else { + setChunksFilteredByQuery( + chunks.filter((chunk) => + chunk.text.toLowerCase().includes(queryInputText.toLowerCase()), + ), + ); + } + }, [queryInputText, chunks]); - const handleCopy = useCallback((text: string, index: number) => { - // Trim whitespace and remove new lines/tabs for cleaner copy - navigator.clipboard.writeText(text.trim().replace(/[\n\r\t]/gm, "")); - setActiveCopiedChunkIndex(index); - setTimeout(() => setActiveCopiedChunkIndex(null), 10 * 1000); // 10 seconds - }, []); + const handleCopy = useCallback((text: string, index: number) => { + // Trim whitespace and remove new lines/tabs for cleaner copy + navigator.clipboard.writeText(text.trim().replace(/[\n\r\t]/gm, "")); + setActiveCopiedChunkIndex(index); + setTimeout(() => setActiveCopiedChunkIndex(null), 10 * 1000); // 10 seconds + }, []); - const fileData = (data as File[]).find( - (file: File) => file.filename === filename - ); + const fileData = (data as File[]).find( + (file: File) => file.filename === filename, + ); - // Extract chunks for the specific file - useEffect(() => { - if (!filename || !(data as File[]).length) { - setChunks([]); - return; - } + // Extract chunks for the specific file + useEffect(() => { + if (!filename || !(data as File[]).length) { + setChunks([]); + return; + } - setChunks(fileData?.chunks || []); - }, [data, filename]); + setChunks(fileData?.chunks || []); + }, [data, filename]); - // Set selected state for all checkboxes when selectAll changes - useEffect(() => { - if (selectAll) { - setSelectedChunks(new Set(chunks.map((_, index) => index))); - } else { - setSelectedChunks(new Set()); - } - }, [selectAll, setSelectedChunks, chunks]); + // Set selected state for all checkboxes when selectAll changes + useEffect(() => { + if (selectAll) { + setSelectedChunks(new Set(chunks.map((_, index) => index))); + } else { + setSelectedChunks(new Set()); + } + }, [selectAll, setSelectedChunks, chunks]); - const handleBack = useCallback(() => { - router.push("/knowledge"); - }, [router]); + const handleBack = useCallback(() => { + router.push("/knowledge"); + }, [router]); - const handleChunkCardCheckboxChange = useCallback( - (index: number) => { - setSelectedChunks((prevSelected) => { - const newSelected = new Set(prevSelected); - if (newSelected.has(index)) { - newSelected.delete(index); - } else { - newSelected.add(index); - } - return newSelected; - }); - }, - [setSelectedChunks] - ); + const handleChunkCardCheckboxChange = useCallback( + (index: number) => { + setSelectedChunks((prevSelected) => { + const newSelected = new Set(prevSelected); + if (newSelected.has(index)) { + newSelected.delete(index); + } else { + newSelected.add(index); + } + return newSelected; + }); + }, + [setSelectedChunks], + ); - if (!filename) { - return ( -
-
- -

No file specified

-

- Please select a file from the knowledge page -

-
-
- ); - } + if (!filename) { + return ( +
+
+ +

No file specified

+

+ Please select a file from the knowledge page +

+
+
+ ); + } - return ( -
-
- {/* Header */} -
-
- -

- {/* Removes file extension from filename */} - {filename.replace(/\.[^/.]+$/, "")} -

-
-
-
- : null} - id="search-query" - type="text" - defaultValue={parsedFilterData?.query} - value={queryInputText} - onChange={(e) => setQueryInputText(e.target.value)} - placeholder="Search chunks..." - /> -
-
- - setSelectAll(!!handleSelectAll) - } - /> - -
-
-
+ return ( +
+
+ {/* Header */} +
+
+ +

+ {/* Removes file extension from filename */} + {filename.replace(/\.[^/.]+$/, "")} +

+
+
+
+ : null + } + id="search-query" + type="text" + defaultValue={parsedFilterData?.query} + value={queryInputText} + onChange={(e) => setQueryInputText(e.target.value)} + placeholder="Search chunks..." + /> +
+
+ + setSelectAll(!!handleSelectAll) + } + /> + +
+
+
- {/* Content Area - matches knowledge page structure */} -
- {isFetching ? ( -
-
- -

- Loading chunks... -

-
-
- ) : chunks.length === 0 ? ( -
-
- -

No chunks found

-

- This file may not have been indexed yet -

-
-
- ) : ( -
- {chunksFilteredByQuery.map((chunk, index) => ( -
-
-
-
- - handleChunkCardCheckboxChange(index) - } - /> -
- - Chunk {chunk.page} - - - {chunk.text.length} chars - -
- -
-
+ {/* Content Area - matches knowledge page structure */} +
+ {isFetching ? ( +
+
+ +

+ Loading chunks... +

+
+
+ ) : chunks.length === 0 ? ( +
+
+ +

No chunks found

+

+ This file may not have been indexed yet +

+
+
+ ) : ( +
+ {chunksFilteredByQuery.map((chunk, index) => ( +
+
+
+
+ + handleChunkCardCheckboxChange(index) + } + /> +
+ + Chunk {chunk.page} + + + {chunk.text.length} chars + +
+ +
+
- {/* TODO: Update to use active toggle */} - {/* + {/* TODO: Update to use active toggle */} + {/* Active */} -
-
- {chunk.text} -
-
- ))} -
- )} -
-
- {/* Right panel - Summary (TODO), Technical details, */} -
-
-

Technical details

-
-
-
Total chunks
-
- {chunks.length} -
-
-
-
Avg length
-
- {averageChunkLength.toFixed(0)} chars -
-
- {/* TODO: Uncomment after data is available */} - {/*
+
+
+ {chunk.text} +
+
+ ))} +
+ )} +
+
+ {/* Right panel - Summary (TODO), Technical details, */} +
+
+

Technical details

+
+
+
Total chunks
+
+ {chunks.length} +
+
+
+
Avg length
+
+ {averageChunkLength.toFixed(0)} chars +
+
+ {/* TODO: Uncomment after data is available */} + {/*
Process time
@@ -293,76 +295,76 @@ function ChunksPageContent() {
*/} -
-
-
-

Original document

-
-
-
Name
-
- {fileData?.filename} -
-
-
-
Type
-
- {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} -
-
-
-
Size
-
- {fileData?.size - ? `${Math.round(fileData.size / 1024)} KB` - : "Unknown"} -
-
-
-
Uploaded
-
- N/A -
-
- {/* TODO: Uncomment after data is available */} - {/*
+
+
+
+

Original document

+
+
+
Name
+
+ {fileData?.filename} +
+
+
+
Type
+
+ {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} +
+
+
+
Size
+
+ {fileData?.size + ? `${Math.round(fileData.size / 1024)} KB` + : "Unknown"} +
+
+
+
Uploaded
+
+ N/A +
+
+ {/* TODO: Uncomment after data is available */} + {/*
Source
*/} -
-
Updated
-
- N/A -
-
-
-
-
-
- ); +
+
Updated
+
+ N/A +
+
+ +
+
+
+ ); } function ChunksPage() { - return ( - -
- -

Loading...

-
-
- } - > - - - ); + return ( + +
+ +

Loading...

+
+ + } + > + + + ); } export default function ProtectedChunksPage() { - return ( - - - - ); + return ( + + + + ); } diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 9e933903..5395d85d 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -310,14 +310,13 @@ function SearchPage() { )} From 99cc806a559f526e9e4a837c41096eef19fd3116 Mon Sep 17 00:00:00 2001 From: Brent O'Neill Date: Fri, 3 Oct 2025 13:53:39 -0600 Subject: [PATCH 28/55] fix empty state --- frontend/src/app/knowledge/chunks/page.tsx | 103 +++++++++++---------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/frontend/src/app/knowledge/chunks/page.tsx b/frontend/src/app/knowledge/chunks/page.tsx index cde2c3e8..31b741a4 100644 --- a/frontend/src/app/knowledge/chunks/page.tsx +++ b/frontend/src/app/knowledge/chunks/page.tsx @@ -224,10 +224,9 @@ function ChunksPageContent() { ) : chunks.length === 0 ? (
- -

No chunks found

-

- This file may not have been indexed yet +

No knowledge

+

+ Clear the knowledge filter or return to the knowledge page

@@ -292,24 +291,29 @@ function ChunksPageContent() { {/* Right panel - Summary (TODO), Technical details, */} -
-
-

Technical details

-
-
-
Total chunks
-
- {chunks.length} -
-
-
-
Avg length
-
- {averageChunkLength.toFixed(0)} chars -
-
- {/* TODO: Uncomment after data is available */} - {/*
+ {chunks.length > 0 && ( +
+
+

+ Technical details +

+
+
+
+ Total chunks +
+
+ {chunks.length} +
+
+
+
Avg length
+
+ {averageChunkLength.toFixed(0)} chars +
+
+ {/* TODO: Uncomment after data is available */} + {/*
Process time
@@ -319,51 +323,54 @@ function ChunksPageContent() {
*/} -
-
-
-

Original document

-
- {/*
+
+
+
+

+ Original document +

+
+ {/*
Name
{fileData?.filename}
*/} -
-
Type
-
- {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} -
-
-
-
Size
-
- {fileData?.size - ? `${Math.round(fileData.size / 1024)} KB` - : "Unknown"} -
-
- {/*
+
+
Type
+
+ {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} +
+
+
+
Size
+
+ {fileData?.size + ? `${Math.round(fileData.size / 1024)} KB` + : "Unknown"} +
+
+ {/*
Uploaded
N/A
*/} - {/* TODO: Uncomment after data is available */} - {/*
+ {/* TODO: Uncomment after data is available */} + {/*
Source
*/} - {/*
+ {/*
Updated
N/A
*/} -
+
+
- + )} ); } From 946d3edc89d09af8fabd5e6b1981466d76fae7ee Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 16:54:40 -0300 Subject: [PATCH 29/55] refresh tasks on entering page, make failed files persist --- frontend/src/app/knowledge/page.tsx | 7 ++++++- frontend/src/contexts/task-context.tsx | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/knowledge/page.tsx b/frontend/src/app/knowledge/page.tsx index 5395d85d..73cc2c7d 100644 --- a/frontend/src/app/knowledge/page.tsx +++ b/frontend/src/app/knowledge/page.tsx @@ -7,6 +7,7 @@ import { useRouter } from "next/navigation"; import { type ChangeEvent, useCallback, + useEffect, useRef, useState, } from "react"; @@ -51,7 +52,7 @@ function getSourceIcon(connectorType?: string) { function SearchPage() { const router = useRouter(); - const { isMenuOpen, files: taskFiles } = useTask(); + const { isMenuOpen, files: taskFiles, refreshTasks } = useTask(); const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = useKnowledgeFilter(); const [selectedRows, setSelectedRows] = useState([]); @@ -59,6 +60,10 @@ function SearchPage() { const deleteDocumentMutation = useDeleteDocument(); + useEffect(() => { + refreshTasks(); + }, [refreshTasks]); + const { data: searchData = [], isFetching } = useGetSearchQuery( parsedFilterData?.query || "*", parsedFilterData, diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index d25130f7..69a1214f 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -226,7 +226,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { setTimeout(() => { refetchSearch(); setFiles((prevFiles) => - prevFiles.filter((file) => file.task_id !== currentTask.task_id && file.status !== "failed"), + prevFiles.filter((file) => file.task_id !== currentTask.task_id || file.status === "failed"), ); }, 500); } else if ( @@ -258,6 +258,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { ); const refreshTasks = useCallback(async () => { + setFiles([]); await refetchTasks(); }, [refetchTasks]); From e5f8c152efdb4f9a429c192f317a8523ecefb130 Mon Sep 17 00:00:00 2001 From: Edwin Jose Date: Fri, 3 Oct 2025 16:03:07 -0400 Subject: [PATCH 30/55] Switch base image to langflow-nightly in Dockerfile Replaces the custom Python build steps with the official langflow-nightly:1.6.3.dev0 image, simplifying the Dockerfile and reducing build complexity. --- Dockerfile.langflow | 48 ++------------------------------------------- 1 file changed, 2 insertions(+), 46 deletions(-) diff --git a/Dockerfile.langflow b/Dockerfile.langflow index 86ee0ea5..71baf447 100644 --- a/Dockerfile.langflow +++ b/Dockerfile.langflow @@ -1,49 +1,5 @@ -FROM python:3.12-slim +FROM langflowai/langflow-nightly:1.6.3.dev0 -# Set environment variables -ENV DEBIAN_FRONTEND=noninteractive -ENV PYTHONUNBUFFERED=1 -ENV RUSTFLAGS="--cfg reqwest_unstable" - -# Accept build arguments for git repository and branch -ARG GIT_REPO=https://github.com/langflow-ai/langflow.git -ARG GIT_BRANCH=test-openai-responses - -WORKDIR /app - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - build-essential \ - curl \ - git \ - ca-certificates \ - gnupg \ - npm \ - rustc cargo pkg-config libssl-dev \ - && rm -rf /var/lib/apt/lists/* - -# Install uv for faster Python package management -RUN pip install uv - -# Clone the repository and checkout the specified branch -RUN git clone --depth 1 --branch ${GIT_BRANCH} ${GIT_REPO} /app - -# Install backend dependencies -RUN uv sync --frozen --no-install-project --no-editable --extra postgresql - -# Build frontend -WORKDIR /app/src/frontend -RUN NODE_OPTIONS=--max_old_space_size=4096 npm ci && \ - NODE_OPTIONS=--max_old_space_size=4096 npm run build && \ - mkdir -p /app/src/backend/base/langflow/frontend && \ - cp -r build/* /app/src/backend/base/langflow/frontend/ - -# Return to app directory and install the project -WORKDIR /app -RUN uv sync --frozen --no-dev --no-editable --extra postgresql - -# Expose ports EXPOSE 7860 -# Start the backend server -CMD ["uv", "run", "langflow", "run", "--host", "0.0.0.0", "--port", "7860"] +CMD ["langflow", "run", "--host", "0.0.0.0", "--port", "7860"] \ No newline at end of file From 509b6c613228fa5ec592533d6a5e69b8bfa76aa2 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 17:06:07 -0300 Subject: [PATCH 31/55] make view button open menu --- frontend/src/contexts/task-context.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index 69a1214f..a8693526 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -220,7 +220,7 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { description, action: { label: "View", - onClick: () => console.log("View task", currentTask.task_id), + onClick: () => setIsMenuOpen(true), }, }); setTimeout(() => { @@ -252,7 +252,9 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { (_taskId: string) => { // React Query will automatically handle polling when tasks are active // Just trigger a refetch to get the latest data - refetchTasks(); + setTimeout(() => { + refetchTasks(); + }, 500); }, [refetchTasks], ); From 7201a914be856775a0bafc7ef445a8882766d3ae Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 17:12:36 -0300 Subject: [PATCH 32/55] open tasks menu when clicking view --- frontend/src/components/task-notification-menu.tsx | 11 +++++++++-- frontend/src/contexts/task-context.tsx | 10 +++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/task-notification-menu.tsx b/frontend/src/components/task-notification-menu.tsx index e17f9579..fed7e6f1 100644 --- a/frontend/src/components/task-notification-menu.tsx +++ b/frontend/src/components/task-notification-menu.tsx @@ -1,6 +1,6 @@ "use client" -import { useState } from 'react' +import { useEffect, useState } from 'react' import { Bell, CheckCircle, XCircle, Clock, Loader2, ChevronDown, ChevronUp, X } from 'lucide-react' import { Button } from '@/components/ui/button' import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card' @@ -8,9 +8,16 @@ import { Badge } from '@/components/ui/badge' import { useTask, Task } from '@/contexts/task-context' export function TaskNotificationMenu() { - const { tasks, isFetching, isMenuOpen, cancelTask } = useTask() + const { tasks, isFetching, isMenuOpen, isRecentTasksExpanded, cancelTask } = useTask() const [isExpanded, setIsExpanded] = useState(false) + // Sync local state with context state + useEffect(() => { + if (isRecentTasksExpanded) { + setIsExpanded(true) + } + }, [isRecentTasksExpanded]) + // Don't render if menu is closed if (!isMenuOpen) return null diff --git a/frontend/src/contexts/task-context.tsx b/frontend/src/contexts/task-context.tsx index a8693526..f5d0c0c4 100644 --- a/frontend/src/contexts/task-context.tsx +++ b/frontend/src/contexts/task-context.tsx @@ -43,6 +43,8 @@ interface TaskContextType { isFetching: boolean; isMenuOpen: boolean; toggleMenu: () => void; + isRecentTasksExpanded: boolean; + setRecentTasksExpanded: (expanded: boolean) => void; // React Query states isLoading: boolean; error: Error | null; @@ -53,6 +55,7 @@ const TaskContext = createContext(undefined); export function TaskProvider({ children }: { children: React.ReactNode }) { const [files, setFiles] = useState([]); const [isMenuOpen, setIsMenuOpen] = useState(false); + const [isRecentTasksExpanded, setIsRecentTasksExpanded] = useState(false); const previousTasksRef = useRef([]); const { isAuthenticated, isNoAuthMode } = useAuth(); @@ -220,7 +223,10 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { description, action: { label: "View", - onClick: () => setIsMenuOpen(true), + onClick: () => { + setIsMenuOpen(true); + setIsRecentTasksExpanded(true); + }, }, }); setTimeout(() => { @@ -302,6 +308,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) { isFetching, isMenuOpen, toggleMenu, + isRecentTasksExpanded, + setRecentTasksExpanded: setIsRecentTasksExpanded, isLoading, error, }; From 5ea70cc7797b652eb829617d466d18e706a8adae Mon Sep 17 00:00:00 2001 From: Edwin Jose Date: Fri, 3 Oct 2025 16:17:05 -0400 Subject: [PATCH 33/55] Refactor file upload task to use filename mapping (#195) Changed the handling of original filenames in Langflow upload tasks to use a mapping from file paths to original filenames instead of a list. Updated both the API router and TaskService to support this change, improving reliability when associating uploaded files with their original names. --- src/api/router.py | 83 ++++++++++++++++++++---------------- src/services/task_service.py | 25 ++++++----- 2 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/api/router.py b/src/api/router.py index 327757be..15a9b116 100644 --- a/src/api/router.py +++ b/src/api/router.py @@ -13,27 +13,27 @@ logger = get_logger(__name__) async def upload_ingest_router( - request: Request, - document_service=None, - langflow_file_service=None, + request: Request, + document_service=None, + langflow_file_service=None, session_manager=None, - task_service=None + task_service=None, ): """ Router endpoint that automatically routes upload requests based on configuration. - + - If DISABLE_INGEST_WITH_LANGFLOW is True: uses traditional OpenRAG upload (/upload) - If DISABLE_INGEST_WITH_LANGFLOW is False (default): uses Langflow upload-ingest via task service - + This provides a single endpoint that users can call regardless of backend configuration. All langflow uploads are processed as background tasks for better scalability. """ try: logger.debug( - "Router upload_ingest endpoint called", - disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW + "Router upload_ingest endpoint called", + disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW, ) - + # Route based on configuration if DISABLE_INGEST_WITH_LANGFLOW: # Route to traditional OpenRAG upload @@ -42,8 +42,10 @@ async def upload_ingest_router( else: # Route to Langflow upload and ingest using task service logger.debug("Routing to Langflow upload-ingest pipeline via task service") - return await langflow_upload_ingest_task(request, langflow_file_service, session_manager, task_service) - + return await langflow_upload_ingest_task( + request, langflow_file_service, session_manager, task_service + ) + except Exception as e: logger.error("Error in upload_ingest_router", error=str(e)) error_msg = str(e) @@ -57,17 +59,14 @@ async def upload_ingest_router( async def langflow_upload_ingest_task( - request: Request, - langflow_file_service, - session_manager, - task_service + request: Request, langflow_file_service, session_manager, task_service ): """Task-based langflow upload and ingest for single/multiple files""" try: logger.debug("Task-based langflow upload_ingest endpoint called") form = await request.form() upload_files = form.getlist("file") - + if not upload_files or len(upload_files) == 0: logger.error("No files provided in task-based upload request") return JSONResponse({"error": "Missing files"}, status_code=400) @@ -82,10 +81,11 @@ async def langflow_upload_ingest_task( # Parse JSON fields if provided settings = None tweaks = None - + if settings_json: try: import json + settings = json.loads(settings_json) except json.JSONDecodeError as e: logger.error("Invalid settings JSON", error=str(e)) @@ -94,6 +94,7 @@ async def langflow_upload_ingest_task( if tweaks_json: try: import json + tweaks = json.loads(tweaks_json) except json.JSONDecodeError as e: logger.error("Invalid tweaks JSON", error=str(e)) @@ -107,11 +108,14 @@ async def langflow_upload_ingest_task( jwt_token = getattr(request.state, "jwt_token", None) if not user_id: - return JSONResponse({"error": "User authentication required"}, status_code=401) + return JSONResponse( + {"error": "User authentication required"}, status_code=401 + ) # Create temporary files for task processing import tempfile import os + temp_file_paths = [] original_filenames = [] @@ -132,7 +136,7 @@ async def langflow_upload_ingest_task( temp_path = os.path.join(temp_dir, safe_filename) # Write content to temp file - with open(temp_path, 'wb') as temp_file: + with open(temp_path, "wb") as temp_file: temp_file.write(content) temp_file_paths.append(temp_path) @@ -143,22 +147,22 @@ async def langflow_upload_ingest_task( user_id=user_id, has_settings=bool(settings), has_tweaks=bool(tweaks), - delete_after_ingest=delete_after_ingest + delete_after_ingest=delete_after_ingest, ) # Create langflow upload task - print(f"tweaks: {tweaks}") - print(f"settings: {settings}") - print(f"jwt_token: {jwt_token}") - print(f"user_name: {user_name}") - print(f"user_email: {user_email}") - print(f"session_id: {session_id}") - print(f"delete_after_ingest: {delete_after_ingest}") - print(f"temp_file_paths: {temp_file_paths}") + logger.debug( + f"Preparing to create langflow upload task: tweaks={tweaks}, settings={settings}, jwt_token={jwt_token}, user_name={user_name}, user_email={user_email}, session_id={session_id}, delete_after_ingest={delete_after_ingest}, temp_file_paths={temp_file_paths}", + ) + # Create a map between temp_file_paths and original_filenames + file_path_to_original_filename = dict(zip(temp_file_paths, original_filenames)) + logger.debug( + f"File path to original filename map: {file_path_to_original_filename}", + ) task_id = await task_service.create_langflow_upload_task( user_id=user_id, file_paths=temp_file_paths, - original_filenames=original_filenames, + original_filenames=file_path_to_original_filename, langflow_file_service=langflow_file_service, session_manager=session_manager, jwt_token=jwt_token, @@ -172,20 +176,24 @@ async def langflow_upload_ingest_task( ) logger.debug("Langflow upload task created successfully", task_id=task_id) - - return JSONResponse({ - "task_id": task_id, - "message": f"Langflow upload task created for {len(upload_files)} file(s)", - "file_count": len(upload_files) - }, status_code=202) # 202 Accepted for async processing - + + return JSONResponse( + { + "task_id": task_id, + "message": f"Langflow upload task created for {len(upload_files)} file(s)", + "file_count": len(upload_files), + }, + status_code=202, + ) # 202 Accepted for async processing + except Exception: # Clean up temp files on error from utils.file_utils import safe_unlink + for temp_path in temp_file_paths: safe_unlink(temp_path) raise - + except Exception as e: logger.error( "Task-based langflow upload_ingest endpoint failed", @@ -193,5 +201,6 @@ async def langflow_upload_ingest_task( error=str(e), ) import traceback + logger.error("Full traceback", traceback=traceback.format_exc()) return JSONResponse({"error": str(e)}, status_code=500) diff --git a/src/services/task_service.py b/src/services/task_service.py index eb5825c0..735ad483 100644 --- a/src/services/task_service.py +++ b/src/services/task_service.py @@ -1,6 +1,5 @@ import asyncio import random -from typing import Dict, Optional import time import uuid @@ -59,7 +58,7 @@ class TaskService: file_paths: list, langflow_file_service, session_manager, - original_filenames: list = None, + original_filenames: dict | None = None, jwt_token: str = None, owner_name: str = None, owner_email: str = None, @@ -88,7 +87,7 @@ class TaskService: ) return await self.create_custom_task(user_id, file_paths, processor, original_filenames) - async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: list = None) -> str: + async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: dict | None = None) -> str: """Create a new task with custom processor for any type of items""" import os # Store anonymous tasks under a stable key so they can be retrieved later @@ -96,14 +95,18 @@ class TaskService: task_id = str(uuid.uuid4()) # Create file tasks with original filenames if provided - file_tasks = {} - for i, item in enumerate(items): - if original_filenames and i < len(original_filenames): - filename = original_filenames[i] - else: - filename = os.path.basename(str(item)) - - file_tasks[str(item)] = FileTask(file_path=str(item), filename=filename) + normalized_originals = ( + {str(k): v for k, v in original_filenames.items()} if original_filenames else {} + ) + file_tasks = { + str(item): FileTask( + file_path=str(item), + filename=normalized_originals.get( + str(item), os.path.basename(str(item)) + ), + ) + for item in items + } upload_task = UploadTask( task_id=task_id, From 3c97a06256d5e9e9ad8016072da3dabddc930dd1 Mon Sep 17 00:00:00 2001 From: Mike Fortman Date: Fri, 3 Oct 2025 15:29:13 -0500 Subject: [PATCH 34/55] Fix layout aligment and folder name --- docker-compose-cpu.yml | 2 +- docker-compose.yml | 2 +- frontend/src/components/layout-wrapper.tsx | 5 +++-- src/tui/_assets/docker-compose-cpu.yml | 2 +- src/tui/_assets/docker-compose.yml | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml index 9c121f89..968f3ba7 100644 --- a/docker-compose-cpu.yml +++ b/docker-compose-cpu.yml @@ -109,5 +109,5 @@ services: - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE} - LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI} - - DEFAULT_FOLDER_NAME="OpenRAG" + - DEFAULT_FOLDER_NAME=OpenRAG - HIDE_GETTING_STARTED_PROGRESS=true diff --git a/docker-compose.yml b/docker-compose.yml index 64226fd5..264efe3a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,5 +117,5 @@ services: - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE} - LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI} - - DEFAULT_FOLDER_NAME="OpenRAG" + - DEFAULT_FOLDER_NAME=OpenRAG - HIDE_GETTING_STARTED_PROGRESS=true diff --git a/frontend/src/components/layout-wrapper.tsx b/frontend/src/components/layout-wrapper.tsx index d657d876..79417654 100644 --- a/frontend/src/components/layout-wrapper.tsx +++ b/frontend/src/components/layout-wrapper.tsx @@ -12,6 +12,7 @@ import { KnowledgeFilterPanel } from "@/components/knowledge-filter-panel"; import Logo from "@/components/logo/logo"; import { Navigation } from "@/components/navigation"; import { TaskNotificationMenu } from "@/components/task-notification-menu"; +import { Button } from "@/components/ui/button"; import { UserNav } from "@/components/user-nav"; import { useAuth } from "@/contexts/auth-context"; import { useChat } from "@/contexts/chat-context"; @@ -34,7 +35,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { refreshConversations, startNewConversation, } = useChat(); - const { isLoading: isSettingsLoading } = useGetSettingsQuery({ + const { isLoading: isSettingsLoading, data: settings } = useGetSettingsQuery({ enabled: isAuthenticated || isNoAuthMode, }); const { @@ -102,7 +103,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) { // For all other pages, render with Langflow-styled navigation and task menu return (
- +
{/* Logo/Title */} diff --git a/src/tui/_assets/docker-compose-cpu.yml b/src/tui/_assets/docker-compose-cpu.yml index 9c121f89..968f3ba7 100644 --- a/src/tui/_assets/docker-compose-cpu.yml +++ b/src/tui/_assets/docker-compose-cpu.yml @@ -109,5 +109,5 @@ services: - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE} - LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI} - - DEFAULT_FOLDER_NAME="OpenRAG" + - DEFAULT_FOLDER_NAME=OpenRAG - HIDE_GETTING_STARTED_PROGRESS=true diff --git a/src/tui/_assets/docker-compose.yml b/src/tui/_assets/docker-compose.yml index 3f1bf1a6..e8cb25d4 100644 --- a/src/tui/_assets/docker-compose.yml +++ b/src/tui/_assets/docker-compose.yml @@ -109,5 +109,5 @@ services: - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE} - LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI} - - DEFAULT_FOLDER_NAME="OpenRAG" + - DEFAULT_FOLDER_NAME=OpenRAG - HIDE_GETTING_STARTED_PROGRESS=true From ea61aa7136ad84974e108479c8f0436103bb9ecd Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 17:29:28 -0300 Subject: [PATCH 35/55] added url ingest id to everything --- .env.example | 1 + docker-compose-cpu.yml | 1 + docker-compose.yml | 1 + src/config/settings.py | 1 + src/services/flows_service.py | 16 +++++++++++++++- src/tui/_assets/docker-compose-cpu.yml | 1 + src/tui/_assets/docker-compose.yml | 1 + src/tui/managers/env_manager.py | 3 +++ 8 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index ee2a838c..8d412670 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,7 @@ LANGFLOW_SECRET_KEY= # flow ids for chat and ingestion flows LANGFLOW_CHAT_FLOW_ID=1098eea1-6649-4e1d-aed1-b77249fb8dd0 LANGFLOW_INGEST_FLOW_ID=5488df7c-b93f-4f87-a446-b67028bc0813 +LANGFLOW_URL_INGEST_FLOW_ID=72c3d17c-2dac-4a73-b48a-6518473d7830 # Ingest flow using docling # LANGFLOW_INGEST_FLOW_ID=1402618b-e6d1-4ff2-9a11-d6ce71186915 NUDGES_FLOW_ID=ebc01d31-1976-46ce-a385-b0240327226c diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml index d0de6ce9..242caa89 100644 --- a/docker-compose-cpu.yml +++ b/docker-compose-cpu.yml @@ -55,6 +55,7 @@ services: - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} + - LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID} - DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - OPENSEARCH_PORT=9200 diff --git a/docker-compose.yml b/docker-compose.yml index be31fb71..246959a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,6 +54,7 @@ services: - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} + - LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID} - DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - OPENSEARCH_PORT=9200 diff --git a/src/config/settings.py b/src/config/settings.py index 27ea9502..6f55520d 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -34,6 +34,7 @@ _legacy_flow_id = os.getenv("FLOW_ID") LANGFLOW_CHAT_FLOW_ID = os.getenv("LANGFLOW_CHAT_FLOW_ID") or _legacy_flow_id LANGFLOW_INGEST_FLOW_ID = os.getenv("LANGFLOW_INGEST_FLOW_ID") +LANGFLOW_URL_INGEST_FLOW_ID = os.getenv("LANGFLOW_URL_INGEST_FLOW_ID") NUDGES_FLOW_ID = os.getenv("NUDGES_FLOW_ID") if _legacy_flow_id and not os.getenv("LANGFLOW_CHAT_FLOW_ID"): diff --git a/src/services/flows_service.py b/src/services/flows_service.py index 999d9930..429eabe7 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -1,5 +1,6 @@ from config.settings import ( DISABLE_INGEST_WITH_LANGFLOW, + LANGFLOW_URL_INGEST_FLOW_ID, NUDGES_FLOW_ID, LANGFLOW_URL, LANGFLOW_CHAT_FLOW_ID, @@ -116,9 +117,11 @@ class FlowsService: flow_id = LANGFLOW_CHAT_FLOW_ID elif flow_type == "ingest": flow_id = LANGFLOW_INGEST_FLOW_ID + elif flow_type == "url_ingest": + flow_id = LANGFLOW_URL_INGEST_FLOW_ID else: raise ValueError( - "flow_type must be either 'nudges', 'retrieval', or 'ingest'" + "flow_type must be either 'nudges', 'retrieval', 'ingest', or 'url_ingest'" ) if not flow_id: @@ -291,6 +294,13 @@ class FlowsService: "llm_name": None, # Ingestion flow might not have LLM "llm_text_name": None, }, + { + "name": "url_ingest", + "flow_id": LANGFLOW_URL_INGEST_FLOW_ID, + "embedding_name": OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME, + "llm_name": None, + "llm_text_name": None, + }, ] results = [] @@ -716,6 +726,10 @@ class FlowsService: "name": "ingest", "flow_id": LANGFLOW_INGEST_FLOW_ID, }, + { + "name": "url_ingest", + "flow_id": LANGFLOW_URL_INGEST_FLOW_ID, + }, ] # Determine target component IDs based on provider diff --git a/src/tui/_assets/docker-compose-cpu.yml b/src/tui/_assets/docker-compose-cpu.yml index d0de6ce9..242caa89 100644 --- a/src/tui/_assets/docker-compose-cpu.yml +++ b/src/tui/_assets/docker-compose-cpu.yml @@ -55,6 +55,7 @@ services: - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} + - LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID} - DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - OPENSEARCH_PORT=9200 diff --git a/src/tui/_assets/docker-compose.yml b/src/tui/_assets/docker-compose.yml index bd81e0ab..8b5ddb74 100644 --- a/src/tui/_assets/docker-compose.yml +++ b/src/tui/_assets/docker-compose.yml @@ -54,6 +54,7 @@ services: - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} + - LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID} - DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - OPENSEARCH_PORT=9200 diff --git a/src/tui/managers/env_manager.py b/src/tui/managers/env_manager.py index 9954b463..9510fb70 100644 --- a/src/tui/managers/env_manager.py +++ b/src/tui/managers/env_manager.py @@ -33,6 +33,7 @@ class EnvConfig: langflow_superuser_password: str = "" langflow_chat_flow_id: str = "1098eea1-6649-4e1d-aed1-b77249fb8dd0" langflow_ingest_flow_id: str = "5488df7c-b93f-4f87-a446-b67028bc0813" + langflow_url_ingest_flow_id: str = "72c3d17c-2dac-4a73-b48a-6518473d7830" # OAuth settings google_oauth_client_id: str = "" @@ -114,6 +115,7 @@ class EnvManager: "LANGFLOW_SUPERUSER_PASSWORD": "langflow_superuser_password", "LANGFLOW_CHAT_FLOW_ID": "langflow_chat_flow_id", "LANGFLOW_INGEST_FLOW_ID": "langflow_ingest_flow_id", + "LANGFLOW_URL_INGEST_FLOW_ID": "langflow_url_ingest_flow_id", "NUDGES_FLOW_ID": "nudges_flow_id", "GOOGLE_OAUTH_CLIENT_ID": "google_oauth_client_id", "GOOGLE_OAUTH_CLIENT_SECRET": "google_oauth_client_secret", @@ -255,6 +257,7 @@ class EnvManager: f.write( f"LANGFLOW_INGEST_FLOW_ID={self._quote_env_value(self.config.langflow_ingest_flow_id)}\n" ) + f.write(f"LANGFLOW_URL_INGEST_FLOW_ID={self._quote_env_value(self.config.langflow_url_ingest_flow_id)}\n") f.write(f"NUDGES_FLOW_ID={self._quote_env_value(self.config.nudges_flow_id)}\n") f.write(f"OPENSEARCH_PASSWORD={self._quote_env_value(self.config.opensearch_password)}\n") f.write(f"OPENAI_API_KEY={self._quote_env_value(self.config.openai_api_key)}\n") From b93faa26b0799fb6fd2cc9a3d1b95dafb6edb86a Mon Sep 17 00:00:00 2001 From: Mike Fortman Date: Fri, 3 Oct 2025 15:38:18 -0500 Subject: [PATCH 36/55] fix flow names and descriptions from "Open Search" to "OpenSearch" --- flows/openrag_agent.json | 4 ++-- flows/openrag_nudges.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flows/openrag_agent.json b/flows/openrag_agent.json index 9e1f33cc..a840610d 100644 --- a/flows/openrag_agent.json +++ b/flows/openrag_agent.json @@ -2556,12 +2556,12 @@ "zoom": 0.602433700773958 } }, - "description": "OpenRAG Open Search Agent", + "description": "OpenRAG OpenSearch Agent", "endpoint_name": null, "id": "1098eea1-6649-4e1d-aed1-b77249fb8dd0", "is_component": false, "last_tested_version": "1.6.0", - "name": "OpenRAG Open Search Agent", + "name": "OpenRAG OpenSearch Agent", "tags": [ "assistants", "agents" diff --git a/flows/openrag_nudges.json b/flows/openrag_nudges.json index 95ff9172..7ed390d7 100644 --- a/flows/openrag_nudges.json +++ b/flows/openrag_nudges.json @@ -2337,12 +2337,12 @@ "zoom": 0.5380793988167256 } }, - "description": "OpenRAG Open Search Nudges generator, based on the Open Search documents and the chat history.", + "description": "OpenRAG OpenSearch Nudges generator, based on the OpenSearch documents and the chat history.", "endpoint_name": null, "id": "ebc01d31-1976-46ce-a385-b0240327226c", "is_component": false, "last_tested_version": "1.6.0", - "name": "OpenRAG Open Search Nudges", + "name": "OpenRAG OpenSearch Nudges", "tags": [ "assistants", "agents" From 9d6827f9e37cea3f2ce5d7a27e55d8834a8f0721 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 17:40:53 -0300 Subject: [PATCH 37/55] updated openrag json --- flows/openrag_url_mcp.json | 976 ++++++++++++++----------------------- 1 file changed, 356 insertions(+), 620 deletions(-) diff --git a/flows/openrag_url_mcp.json b/flows/openrag_url_mcp.json index c8f8cb70..729e2145 100644 --- a/flows/openrag_url_mcp.json +++ b/flows/openrag_url_mcp.json @@ -30,34 +30,6 @@ "target": "OpenSearchHybrid-Ve6bS", "targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" }, - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "OpenAIEmbeddings", - "id": "OpenAIEmbeddings-joRJ6", - "name": "embeddings", - "output_types": [ - "Embeddings" - ] - }, - "targetHandle": { - "fieldName": "embedding", - "id": "OpenSearchHybrid-Ve6bS", - "inputTypes": [ - "Embeddings" - ], - "type": "other" - } - }, - "id": "xy-edge__OpenAIEmbeddings-joRJ6{œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-joRJ6œ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}-OpenSearchHybrid-Ve6bS{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}", - "selected": false, - "source": "OpenAIEmbeddings-joRJ6", - "sourceHandle": "{œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-joRJ6œ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}", - "target": "OpenSearchHybrid-Ve6bS", - "targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}" - }, { "animated": false, "className": "", @@ -201,6 +173,8 @@ "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" }, { + "animated": false, + "className": "", "data": { "sourceHandle": { "dataType": "SplitText", @@ -220,12 +194,15 @@ } }, "id": "xy-edge__SplitText-QIKhg{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-RhKoe{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-RhKoeœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, "source": "SplitText-QIKhg", "sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", "target": "DataFrameOperations-RhKoe", "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-RhKoeœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" }, { + "animated": false, + "className": "", "data": { "sourceHandle": { "dataType": "DataFrameOperations", @@ -247,10 +224,38 @@ } }, "id": "xy-edge__DataFrameOperations-RhKoe{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-RhKoeœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-ChatOutput-Q1dhr{œfieldNameœ:œinput_valueœ,œidœ:œChatOutput-Q1dhrœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}", + "selected": false, "source": "DataFrameOperations-RhKoe", "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-RhKoeœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", "target": "ChatOutput-Q1dhr", "targetHandle": "{œfieldNameœ:œinput_valueœ,œidœ:œChatOutput-Q1dhrœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "data": { + "sourceHandle": { + "dataType": "EmbeddingModel", + "id": "EmbeddingModel-eC65s", + "name": "embeddings", + "output_types": [ + "Embeddings" + ] + }, + "targetHandle": { + "fieldName": "embedding", + "id": "OpenSearchHybrid-Ve6bS", + "inputTypes": [ + "Embeddings" + ], + "type": "other" + } + }, + "id": "xy-edge__EmbeddingModel-eC65s{œdataTypeœ:œEmbeddingModelœ,œidœ:œEmbeddingModel-eC65sœ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}-OpenSearchHybrid-Ve6bS{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}", + "selected": false, + "source": "EmbeddingModel-eC65s", + "sourceHandle": "{œdataTypeœ:œEmbeddingModelœ,œidœ:œEmbeddingModel-eC65sœ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}", + "target": "OpenSearchHybrid-Ve6bS", + "targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}" } ], "nodes": [ @@ -497,589 +502,6 @@ "type": "genericNode", "width": 320 }, - { - "data": { - "description": "Generate embeddings using OpenAI models.", - "display_name": "OpenAI Embeddings", - "id": "OpenAIEmbeddings-joRJ6", - "node": { - "base_classes": [ - "Embeddings" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Generate embeddings using OpenAI models.", - "display_name": "OpenAI Embeddings", - "documentation": "", - "edited": false, - "field_order": [ - "default_headers", - "default_query", - "chunk_size", - "client", - "deployment", - "embedding_ctx_length", - "max_retries", - "model", - "model_kwargs", - "openai_api_key", - "openai_api_base", - "openai_api_type", - "openai_api_version", - "openai_organization", - "openai_proxy", - "request_timeout", - "show_progress_bar", - "skip_empty", - "tiktoken_model_name", - "tiktoken_enable", - "dimensions" - ], - "frozen": false, - "icon": "OpenAI", - "legacy": false, - "lf_version": "1.6.0", - "metadata": { - "code_hash": "8a658ed6d4c9", - "dependencies": { - "dependencies": [ - { - "name": "langchain_openai", - "version": "0.3.23" - }, - { - "name": "lfx", - "version": null - } - ], - "total_dependencies": 2 - }, - "module": "custom_components.openai_embeddings" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "Embedding Model", - "group_outputs": false, - "method": "build_embeddings", - "name": "embeddings", - "options": null, - "required_inputs": null, - "selected": "Embeddings", - "tool_mode": true, - "types": [ - "Embeddings" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "chunk_size": { - "_input_type": "IntInput", - "advanced": true, - "display_name": "Chunk Size", - "dynamic": false, - "info": "", - "list": false, - "list_add_label": "Add More", - "name": "chunk_size", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "int", - "value": 1000 - }, - "client": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "Client", - "dynamic": false, - "info": "", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "client", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from langchain_openai import OpenAIEmbeddings\n\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import BoolInput, DictInput, DropdownInput, FloatInput, IntInput, MessageTextInput, SecretStrInput\n\n\nclass OpenAIEmbeddingsComponent(LCEmbeddingsModel):\n display_name = \"OpenAI Embeddings\"\n description = \"Generate embeddings using OpenAI models.\"\n icon = \"OpenAI\"\n name = \"OpenAIEmbeddings\"\n\n inputs = [\n DictInput(\n name=\"default_headers\",\n display_name=\"Default Headers\",\n advanced=True,\n info=\"Default headers to use for the API request.\",\n ),\n DictInput(\n name=\"default_query\",\n display_name=\"Default Query\",\n advanced=True,\n info=\"Default query parameters to use for the API request.\",\n ),\n IntInput(name=\"chunk_size\", display_name=\"Chunk Size\", advanced=True, value=1000),\n MessageTextInput(name=\"client\", display_name=\"Client\", advanced=True),\n MessageTextInput(name=\"deployment\", display_name=\"Deployment\", advanced=True),\n IntInput(name=\"embedding_ctx_length\", display_name=\"Embedding Context Length\", advanced=True, value=1536),\n IntInput(name=\"max_retries\", display_name=\"Max Retries\", value=3, advanced=True),\n DropdownInput(\n name=\"model\",\n display_name=\"Model\",\n advanced=False,\n options=OPENAI_EMBEDDING_MODEL_NAMES,\n value=\"text-embedding-3-small\",\n ),\n DictInput(name=\"model_kwargs\", display_name=\"Model Kwargs\", advanced=True),\n SecretStrInput(name=\"openai_api_key\", display_name=\"OpenAI API Key\", value=\"OPENAI_API_KEY\", required=True),\n MessageTextInput(name=\"openai_api_base\", display_name=\"OpenAI API Base\", advanced=True),\n MessageTextInput(name=\"openai_api_type\", display_name=\"OpenAI API Type\", advanced=True),\n MessageTextInput(name=\"openai_api_version\", display_name=\"OpenAI API Version\", advanced=True),\n MessageTextInput(\n name=\"openai_organization\",\n display_name=\"OpenAI Organization\",\n advanced=True,\n ),\n MessageTextInput(name=\"openai_proxy\", display_name=\"OpenAI Proxy\", advanced=True),\n FloatInput(name=\"request_timeout\", display_name=\"Request Timeout\", advanced=True),\n BoolInput(name=\"show_progress_bar\", display_name=\"Show Progress Bar\", advanced=True),\n BoolInput(name=\"skip_empty\", display_name=\"Skip Empty\", advanced=True),\n MessageTextInput(\n name=\"tiktoken_model_name\",\n display_name=\"TikToken Model Name\",\n advanced=True,\n ),\n BoolInput(\n name=\"tiktoken_enable\",\n display_name=\"TikToken Enable\",\n advanced=True,\n value=True,\n info=\"If False, you must have transformers installed.\",\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n ]\n\n def build_embeddings(self) -> Embeddings:\n return OpenAIEmbeddings(\n client=self.client or None,\n model=self.model,\n dimensions=self.dimensions or None,\n deployment=self.deployment or None,\n api_version=self.openai_api_version or None,\n base_url=self.openai_api_base or None,\n openai_api_type=self.openai_api_type or None,\n openai_proxy=self.openai_proxy or None,\n embedding_ctx_length=self.embedding_ctx_length,\n api_key=self.openai_api_key or None,\n organization=self.openai_organization or None,\n allowed_special=\"all\",\n disallowed_special=\"all\",\n chunk_size=self.chunk_size,\n max_retries=self.max_retries,\n timeout=self.request_timeout or None,\n tiktoken_enabled=self.tiktoken_enable,\n tiktoken_model_name=self.tiktoken_model_name or None,\n show_progress_bar=self.show_progress_bar,\n model_kwargs=self.model_kwargs,\n skip_empty=self.skip_empty,\n default_headers=self.default_headers or None,\n default_query=self.default_query or None,\n )\n" - }, - "default_headers": { - "_input_type": "DictInput", - "advanced": true, - "display_name": "Default Headers", - "dynamic": false, - "info": "Default headers to use for the API request.", - "list": false, - "list_add_label": "Add More", - "name": "default_headers", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "type": "dict", - "value": {} - }, - "default_query": { - "_input_type": "DictInput", - "advanced": true, - "display_name": "Default Query", - "dynamic": false, - "info": "Default query parameters to use for the API request.", - "list": false, - "list_add_label": "Add More", - "name": "default_query", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "type": "dict", - "value": {} - }, - "deployment": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "Deployment", - "dynamic": false, - "info": "", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "deployment", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "" - }, - "dimensions": { - "_input_type": "IntInput", - "advanced": true, - "display_name": "Dimensions", - "dynamic": false, - "info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.", - "list": false, - "list_add_label": "Add More", - "name": "dimensions", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "int", - "value": "" - }, - "embedding_ctx_length": { - "_input_type": "IntInput", - "advanced": true, - "display_name": "Embedding Context Length", - "dynamic": false, - "info": "", - "list": false, - "list_add_label": "Add More", - "name": "embedding_ctx_length", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "int", - "value": 1536 - }, - "max_retries": { - "_input_type": "IntInput", - "advanced": true, - "display_name": "Max Retries", - "dynamic": false, - "info": "", - "list": false, - "list_add_label": "Add More", - "name": "max_retries", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "int", - "value": 3 - }, - "model": { - "_input_type": "DropdownInput", - "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Model", - "dynamic": false, - "external_options": {}, - "info": "", - "name": "model", - "options": [ - "text-embedding-3-small", - "text-embedding-3-large", - "text-embedding-ada-002" - ], - "options_metadata": [], - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "toggle": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "str", - "value": "text-embedding-3-small" - }, - "model_kwargs": { - "_input_type": "DictInput", - "advanced": true, - "display_name": "Model Kwargs", - "dynamic": false, - "info": "", - "list": false, - "list_add_label": "Add More", - "name": "model_kwargs", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "type": "dict", - "value": {} - }, - "openai_api_base": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "OpenAI API Base", - "dynamic": false, - "info": "", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "openai_api_base", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "" - }, - "openai_api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "OpenAI API Key", - "dynamic": false, - "info": "", - "input_types": [], - "load_from_db": true, - "name": "openai_api_key", - "password": true, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "str", - "value": "OPENAI_API_KEY" - }, - "openai_api_type": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "OpenAI API Type", - "dynamic": false, - "info": "", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "openai_api_type", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "" - }, - "openai_api_version": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "OpenAI API Version", - "dynamic": false, - "info": "", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "openai_api_version", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "" - }, - "openai_organization": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "OpenAI Organization", - "dynamic": false, - "info": "", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "openai_organization", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "" - }, - "openai_proxy": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "OpenAI Proxy", - "dynamic": false, - "info": "", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "openai_proxy", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "" - }, - "request_timeout": { - "_input_type": "FloatInput", - "advanced": true, - "display_name": "Request Timeout", - "dynamic": false, - "info": "", - "list": false, - "list_add_label": "Add More", - "name": "request_timeout", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "float", - "value": "" - }, - "show_progress_bar": { - "_input_type": "BoolInput", - "advanced": true, - "display_name": "Show Progress Bar", - "dynamic": false, - "info": "", - "list": false, - "list_add_label": "Add More", - "name": "show_progress_bar", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "bool", - "value": false - }, - "skip_empty": { - "_input_type": "BoolInput", - "advanced": true, - "display_name": "Skip Empty", - "dynamic": false, - "info": "", - "list": false, - "list_add_label": "Add More", - "name": "skip_empty", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "bool", - "value": false - }, - "tiktoken_enable": { - "_input_type": "BoolInput", - "advanced": true, - "display_name": "TikToken Enable", - "dynamic": false, - "info": "If False, you must have transformers installed.", - "list": false, - "list_add_label": "Add More", - "name": "tiktoken_enable", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "bool", - "value": true - }, - "tiktoken_model_name": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "TikToken Model Name", - "dynamic": false, - "info": "", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "tiktoken_model_name", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "" - } - }, - "tool_mode": false - }, - "selected_output": "embeddings", - "type": "OpenAIEmbeddings" - }, - "dragging": false, - "height": 320, - "id": "OpenAIEmbeddings-joRJ6", - "measured": { - "height": 320, - "width": 320 - }, - "position": { - "x": 1870.4219509914485, - "y": 2193.4259215896764 - }, - "positionAbsolute": { - "x": 1690.9220896443658, - "y": 1866.483269483266 - }, - "selected": false, - "type": "genericNode", - "width": 320 - }, - { - "data": { - "id": "note-Bm5Xw", - "node": { - "description": "### 💡 Add your OpenAI API key here 👇", - "display_name": "", - "documentation": "", - "template": { - "backgroundColor": "transparent" - } - }, - "type": "note" - }, - "dragging": false, - "height": 324, - "id": "note-Bm5Xw", - "measured": { - "height": 324, - "width": 324 - }, - "position": { - "x": 1868.5365759938197, - "y": 2117.7924922977318 - }, - "positionAbsolute": { - "x": 1692.2322233423606, - "y": 1821.9077961087607 - }, - "selected": false, - "type": "noteNode", - "width": 324 - }, { "data": { "id": "OpenSearchHybrid-Ve6bS", @@ -1307,6 +729,10 @@ { "key": "owner", "value": "OWNER" + }, + { + "key": "owner_email", + "value": "OWNER_EMAIL" } ] }, @@ -1710,7 +1136,7 @@ "x": 2694.183983837566, "y": 1425.777807367294 }, - "selected": false, + "selected": true, "type": "genericNode" }, { @@ -2382,7 +1808,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-10-03T14:31:53.355Z", + "last_updated": "2025-10-03T20:31:36.023Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -2798,7 +2224,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-10-03T14:31:53.355Z", + "last_updated": "2025-10-03T20:31:36.025Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -3471,7 +2897,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-10-03T14:40:28.935Z", + "last_updated": "2025-10-03T20:31:36.026Z", "legacy": false, "metadata": { "code_hash": "b4d6b19b6eef", @@ -3853,14 +3279,325 @@ "x": 2773.2060092972047, "y": 2337.54590413581 }, - "selected": true, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "EmbeddingModel-eC65s", + "node": { + "base_classes": [ + "Embeddings" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Generate embeddings using a specified provider.", + "display_name": "Embedding Model", + "documentation": "https://docs.langflow.org/components-embedding-models", + "edited": false, + "field_order": [ + "provider", + "model", + "api_key", + "api_base", + "dimensions", + "chunk_size", + "request_timeout", + "max_retries", + "show_progress_bar", + "model_kwargs" + ], + "frozen": false, + "icon": "binary", + "last_updated": "2025-10-03T20:31:47.177Z", + "legacy": false, + "metadata": { + "code_hash": "8607e963fdef", + "dependencies": { + "dependencies": [ + { + "name": "langchain_openai", + "version": "0.3.23" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 2 + }, + "module": "lfx.components.models.embedding_model.EmbeddingModelComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Embedding Model", + "group_outputs": false, + "method": "build_embeddings", + "name": "embeddings", + "options": null, + "required_inputs": null, + "selected": "Embeddings", + "tool_mode": true, + "types": [ + "Embeddings" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "api_base": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "API Base URL", + "dynamic": false, + "info": "Base URL for the API. Leave empty for default.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "api_base", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "type": "str", + "value": "" + }, + "api_key": { + "_input_type": "SecretStrInput", + "advanced": false, + "display_name": "OpenAI API Key", + "dynamic": false, + "info": "Model Provider API key", + "input_types": [], + "load_from_db": true, + "name": "api_key", + "password": true, + "placeholder": "", + "real_time_refresh": true, + "required": true, + "show": true, + "title_case": false, + "type": "str", + "value": "OPENAI_API_KEY" + }, + "chunk_size": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Chunk Size", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "chunk_size", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 1000 + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from typing import Any\n\nfrom langchain_openai import OpenAIEmbeddings\n\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import (\n BoolInput,\n DictInput,\n DropdownInput,\n FloatInput,\n IntInput,\n MessageTextInput,\n SecretStrInput,\n)\nfrom lfx.schema.dotdict import dotdict\n\n\nclass EmbeddingModelComponent(LCEmbeddingsModel):\n display_name = \"Embedding Model\"\n description = \"Generate embeddings using a specified provider.\"\n documentation: str = \"https://docs.langflow.org/components-embedding-models\"\n icon = \"binary\"\n name = \"EmbeddingModel\"\n category = \"models\"\n\n inputs = [\n DropdownInput(\n name=\"provider\",\n display_name=\"Model Provider\",\n options=[\"OpenAI\"],\n value=\"OpenAI\",\n info=\"Select the embedding model provider\",\n real_time_refresh=True,\n options_metadata=[{\"icon\": \"OpenAI\"}],\n ),\n DropdownInput(\n name=\"model\",\n display_name=\"Model Name\",\n options=OPENAI_EMBEDDING_MODEL_NAMES,\n value=OPENAI_EMBEDDING_MODEL_NAMES[0],\n info=\"Select the embedding model to use\",\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"OpenAI API Key\",\n info=\"Model Provider API key\",\n required=True,\n show=True,\n real_time_refresh=True,\n ),\n MessageTextInput(\n name=\"api_base\",\n display_name=\"API Base URL\",\n info=\"Base URL for the API. Leave empty for default.\",\n advanced=True,\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n IntInput(name=\"chunk_size\", display_name=\"Chunk Size\", advanced=True, value=1000),\n FloatInput(name=\"request_timeout\", display_name=\"Request Timeout\", advanced=True),\n IntInput(name=\"max_retries\", display_name=\"Max Retries\", advanced=True, value=3),\n BoolInput(name=\"show_progress_bar\", display_name=\"Show Progress Bar\", advanced=True),\n DictInput(\n name=\"model_kwargs\",\n display_name=\"Model Kwargs\",\n advanced=True,\n info=\"Additional keyword arguments to pass to the model.\",\n ),\n ]\n\n def build_embeddings(self) -> Embeddings:\n provider = self.provider\n model = self.model\n api_key = self.api_key\n api_base = self.api_base\n dimensions = self.dimensions\n chunk_size = self.chunk_size\n request_timeout = self.request_timeout\n max_retries = self.max_retries\n show_progress_bar = self.show_progress_bar\n model_kwargs = self.model_kwargs or {}\n\n if provider == \"OpenAI\":\n if not api_key:\n msg = \"OpenAI API key is required when using OpenAI provider\"\n raise ValueError(msg)\n return OpenAIEmbeddings(\n model=model,\n dimensions=dimensions or None,\n base_url=api_base or None,\n api_key=api_key,\n chunk_size=chunk_size,\n max_retries=max_retries,\n timeout=request_timeout or None,\n show_progress_bar=show_progress_bar,\n model_kwargs=model_kwargs,\n )\n msg = f\"Unknown provider: {provider}\"\n raise ValueError(msg)\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n if field_name == \"provider\" and field_value == \"OpenAI\":\n build_config[\"model\"][\"options\"] = OPENAI_EMBEDDING_MODEL_NAMES\n build_config[\"model\"][\"value\"] = OPENAI_EMBEDDING_MODEL_NAMES[0]\n build_config[\"api_key\"][\"display_name\"] = \"OpenAI API Key\"\n build_config[\"api_base\"][\"display_name\"] = \"OpenAI API Base URL\"\n return build_config\n" + }, + "dimensions": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Dimensions", + "dynamic": false, + "info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.", + "list": false, + "list_add_label": "Add More", + "name": "dimensions", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": "" + }, + "max_retries": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Max Retries", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "max_retries", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "int", + "value": 3 + }, + "model": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Model Name", + "dynamic": false, + "external_options": {}, + "info": "Select the embedding model to use", + "name": "model", + "options": [ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002" + ], + "options_metadata": [], + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "text-embedding-3-small" + }, + "model_kwargs": { + "_input_type": "DictInput", + "advanced": true, + "display_name": "Model Kwargs", + "dynamic": false, + "info": "Additional keyword arguments to pass to the model.", + "list": false, + "list_add_label": "Add More", + "name": "model_kwargs", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "type": "dict", + "value": {} + }, + "provider": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Model Provider", + "dynamic": false, + "external_options": {}, + "info": "Select the embedding model provider", + "name": "provider", + "options": [ + "OpenAI" + ], + "options_metadata": [ + { + "icon": "OpenAI" + } + ], + "placeholder": "", + "real_time_refresh": true, + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "str", + "value": "OpenAI" + }, + "request_timeout": { + "_input_type": "FloatInput", + "advanced": true, + "display_name": "Request Timeout", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "request_timeout", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "float", + "value": "" + }, + "show_progress_bar": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Show Progress Bar", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "show_progress_bar", + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "type": "bool", + "value": false + } + }, + "tool_mode": false + }, + "showNode": true, + "type": "EmbeddingModel" + }, + "dragging": false, + "id": "EmbeddingModel-eC65s", + "measured": { + "height": 369, + "width": 320 + }, + "position": { + "x": 2160.1718548185536, + "y": 2003.7747198162415 + }, + "selected": false, "type": "genericNode" } ], "viewport": { - "x": -1399.3594540015256, - "y": -1163.4584036787048, - "zoom": 0.7595394813583742 + "x": -407.1633937626607, + "y": -577.5291936220412, + "zoom": 0.5347553210574026 } }, "description": "This flow is to ingest the URL to open search.", @@ -3869,7 +3606,6 @@ "is_component": false, "last_tested_version": "1.6.0", "name": "OpenSearch URL Ingestion Flow", - "mcp_enabled": true, "tags": [ "openai", "astradb", From fe708e4fd6a69961b4bfb6787a61555484af33e9 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 17:50:46 -0300 Subject: [PATCH 38/55] made openrag url mcp be mcp enabled --- flows/openrag_url_mcp.json | 1 + 1 file changed, 1 insertion(+) diff --git a/flows/openrag_url_mcp.json b/flows/openrag_url_mcp.json index 729e2145..69dbc85d 100644 --- a/flows/openrag_url_mcp.json +++ b/flows/openrag_url_mcp.json @@ -3602,6 +3602,7 @@ }, "description": "This flow is to ingest the URL to open search.", "endpoint_name": null, + "mcp_enabled": true, "id": "72c3d17c-2dac-4a73-b48a-6518473d7830", "is_component": false, "last_tested_version": "1.6.0", From 84a43141d90e06acbc6a0d9f223a652ef66ed072 Mon Sep 17 00:00:00 2001 From: Mike Fortman Date: Fri, 3 Oct 2025 15:54:57 -0500 Subject: [PATCH 39/55] set table structure to true by default --- frontend/src/app/settings/page.tsx | 2 +- frontend/src/lib/constants.ts | 2 +- src/config/config_manager.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/settings/page.tsx b/frontend/src/app/settings/page.tsx index 7a444a02..a6e22ffd 100644 --- a/frontend/src/app/settings/page.tsx +++ b/frontend/src/app/settings/page.tsx @@ -149,7 +149,7 @@ function KnowledgeSourcesPage() { const [systemPrompt, setSystemPrompt] = useState(""); const [chunkSize, setChunkSize] = useState(1024); const [chunkOverlap, setChunkOverlap] = useState(50); - const [tableStructure, setTableStructure] = useState(false); + const [tableStructure, setTableStructure] = useState(true); const [ocr, setOcr] = useState(false); const [pictureDescriptions, setPictureDescriptions] = useState(false); diff --git a/frontend/src/lib/constants.ts b/frontend/src/lib/constants.ts index 8e7770fb..9ce34634 100644 --- a/frontend/src/lib/constants.ts +++ b/frontend/src/lib/constants.ts @@ -12,7 +12,7 @@ export const DEFAULT_AGENT_SETTINGS = { export const DEFAULT_KNOWLEDGE_SETTINGS = { chunk_size: 1000, chunk_overlap: 200, - table_structure: false, + table_structure: true, ocr: false, picture_descriptions: false } as const; diff --git a/src/config/config_manager.py b/src/config/config_manager.py index da059d0d..93fb86c5 100644 --- a/src/config/config_manager.py +++ b/src/config/config_manager.py @@ -27,7 +27,7 @@ class KnowledgeConfig: embedding_model: str = "text-embedding-3-small" chunk_size: int = 1000 chunk_overlap: int = 200 - table_structure: bool = False + table_structure: bool = True ocr: bool = False picture_descriptions: bool = False From 9284000859e7098f6d228d7995fc0f3ed1c400f9 Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Fri, 3 Oct 2025 17:55:13 -0300 Subject: [PATCH 40/55] updated json --- flows/openrag_agent.json | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/flows/openrag_agent.json b/flows/openrag_agent.json index 3b83f107..63eadb08 100644 --- a/flows/openrag_agent.json +++ b/flows/openrag_agent.json @@ -172,6 +172,8 @@ "targetHandle": "{œfieldNameœ:œtoolsœ,œidœ:œAgent-crjWfœ,œinputTypesœ:[œToolœ],œtypeœ:œotherœ}" }, { + "animated": false, + "className": "", "data": { "sourceHandle": { "dataType": "TextInput", @@ -191,6 +193,7 @@ } }, "id": "xy-edge__TextInput-aHsQb{œdataTypeœ:œTextInputœ,œidœ:œTextInput-aHsQbœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearch-iYfjf{œfieldNameœ:œfilter_expressionœ,œidœ:œOpenSearch-iYfjfœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}", + "selected": true, "source": "TextInput-aHsQb", "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-aHsQbœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", "target": "OpenSearch-iYfjf", @@ -755,7 +758,7 @@ ], "frozen": false, "icon": "OpenSearch", - "last_updated": "2025-10-03T15:07:58.068Z", + "last_updated": "2025-10-03T20:53:32.145Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -1409,7 +1412,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-10-03T15:07:58.070Z", + "last_updated": "2025-10-03T20:53:32.146Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -1690,7 +1693,7 @@ "x": 727.4791597769406, "y": 416.82609966052854 }, - "selected": true, + "selected": false, "type": "genericNode" }, { @@ -1734,7 +1737,7 @@ ], "frozen": false, "icon": "bot", - "last_updated": "2025-10-03T15:07:58.132Z", + "last_updated": "2025-10-03T20:53:32.216Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -2273,7 +2276,7 @@ ], "frozen": false, "icon": "brain-circuit", - "last_updated": "2025-10-03T15:07:58.071Z", + "last_updated": "2025-10-03T20:53:32.147Z", "legacy": false, "lf_version": "1.6.0", "metadata": { @@ -2601,7 +2604,7 @@ "frozen": false, "icon": "Mcp", "key": "mcp_lf-starter_project", - "last_updated": "2025-10-03T15:07:58.072Z", + "last_updated": "2025-10-03T20:53:32.149Z", "legacy": false, "mcpServerName": "lf-starter_project", "metadata": { @@ -2839,9 +2842,9 @@ } ], "viewport": { - "x": -184.98015964664273, - "y": 154.6885920024542, - "zoom": 0.602433700773958 + "x": -125.84420166020391, + "y": 161.94501551784947, + "zoom": 0.5780225995341656 } }, "description": "OpenRAG Open Search Agent", From 81c82cea411437252fc2f098bc6284aa4860d25d Mon Sep 17 00:00:00 2001 From: Mike Fortman Date: Fri, 3 Oct 2025 16:53:48 -0500 Subject: [PATCH 41/55] tweak id fix --- src/services/langflow_file_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py index 7ffdd3aa..14d2f42d 100644 --- a/src/services/langflow_file_service.py +++ b/src/services/langflow_file_service.py @@ -86,12 +86,12 @@ class LangflowFileService: # Pass files via tweaks to File component (File-PSU37 from the flow) if file_paths: - tweaks["File-PSU37"] = {"path": file_paths} + tweaks["DoclingRemote-78KoX"] = {"path": file_paths} # Pass JWT token via tweaks using the x-langflow-global-var- pattern if jwt_token: # Using the global variable pattern that Langflow expects for OpenSearch components - tweaks["OpenSearchHybrid-Ve6bS"] = {"jwt_token": jwt_token} + tweaks["OpenSearchHybrid-XtKoA"] = {"jwt_token": jwt_token} logger.debug("[LF] Added JWT token to tweaks for OpenSearch components") else: logger.warning("[LF] No JWT token provided") From 336e7bb5017d1926515cc487d1bd26021f2f80b0 Mon Sep 17 00:00:00 2001 From: Edwin Jose Date: Sat, 4 Oct 2025 02:09:29 -0400 Subject: [PATCH 42/55] Integrate OpenSearch hybrid search and update ingestion flow Replaces the File component with a new OpenSearch hybrid search component in the ingestion flow, adds support for document metadata, and updates flow edges for DataFrame operations. Updates OpenSearch component implementation with advanced authentication, metadata handling, and vector store features. Docker Compose files and related service references are also updated to support the new OpenSearch integration. --- flows/ingestion_flow.json | 2579 +++++++++++++++++++----- flows/openrag_agent.json | 306 +-- src/services/langflow_file_service.py | 12 +- src/tui/_assets/docker-compose-cpu.yml | 13 +- src/tui/_assets/docker-compose.yml | 13 +- 5 files changed, 2089 insertions(+), 834 deletions(-) diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json index 12cf5b63..911c3e38 100644 --- a/flows/ingestion_flow.json +++ b/flows/ingestion_flow.json @@ -30,36 +30,6 @@ "target": "OpenSearchHybrid-Ve6bS", "targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" }, - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "File", - "id": "File-PSU37", - "name": "message", - "output_types": [ - "Message" - ] - }, - "targetHandle": { - "fieldName": "data_inputs", - "id": "SplitText-QIKhg", - "inputTypes": [ - "Data", - "DataFrame", - "Message" - ], - "type": "other" - } - }, - "id": "xy-edge__File-PSU37{œdataTypeœ:œFileœ,œidœ:œFile-PSU37œ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}-SplitText-QIKhg{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}", - "selected": false, - "source": "File-PSU37", - "sourceHandle": "{œdataTypeœ:œFileœ,œidœ:œFile-PSU37œ,œnameœ:œmessageœ,œoutput_typesœ:[œMessageœ]}", - "target": "SplitText-QIKhg", - "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" - }, { "animated": false, "className": "", @@ -206,7 +176,7 @@ }, { "animated": false, - "className": "not-running", + "className": "", "data": { "sourceHandle": { "dataType": "AdvancedDynamicFormBuilder", @@ -231,6 +201,149 @@ "sourceHandle": "{œdataTypeœ:œAdvancedDynamicFormBuilderœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œnameœ:œform_dataœ,œoutput_typesœ:[œDataœ]}", "target": "OpenSearchHybrid-Ve6bS", "targetHandle": "{œfieldNameœ:œdocs_metadataœ,œidœ:œOpenSearchHybrid-Ve6bSœ,œinputTypesœ:[œDataœ],œtypeœ:œtableœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DoclingRemote", + "id": "DoclingRemote-Dp3PX", + "name": "dataframe", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "data_inputs", + "id": "ExportDoclingDocument-zZdRg", + "inputTypes": [ + "Data", + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__DoclingRemote-Dp3PX{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-ExportDoclingDocument-zZdRg{œfieldNameœ:œdata_inputsœ,œidœ:œExportDoclingDocument-zZdRgœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DoclingRemote-Dp3PX", + "sourceHandle": "{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "ExportDoclingDocument-zZdRg", + "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œExportDoclingDocument-zZdRgœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "ExportDoclingDocument", + "id": "ExportDoclingDocument-zZdRg", + "name": "dataframe", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "df", + "id": "DataFrameOperations-1BWXB", + "inputTypes": [ + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__ExportDoclingDocument-zZdRg{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-1BWXB{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "ExportDoclingDocument-zZdRg", + "sourceHandle": "{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "DataFrameOperations-1BWXB", + "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DataFrameOperations", + "id": "DataFrameOperations-N80fC", + "name": "output", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "data_inputs", + "id": "SplitText-QIKhg", + "inputTypes": [ + "Data", + "DataFrame", + "Message" + ], + "type": "other" + } + }, + "id": "xy-edge__DataFrameOperations-N80fC{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-SplitText-QIKhg{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DataFrameOperations-N80fC", + "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "SplitText-QIKhg", + "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DataFrameOperations", + "id": "DataFrameOperations-1BWXB", + "name": "output", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "df", + "id": "DataFrameOperations-9vMrp", + "inputTypes": [ + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__DataFrameOperations-1BWXB{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-1BWXBœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-9vMrp{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DataFrameOperations-1BWXB", + "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-1BWXBœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "DataFrameOperations-9vMrp", + "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DataFrameOperations", + "id": "DataFrameOperations-9vMrp", + "name": "output", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "df", + "id": "DataFrameOperations-N80fC", + "inputTypes": [ + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__DataFrameOperations-9vMrp{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-9vMrpœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-N80fC{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-N80fCœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DataFrameOperations-9vMrp", + "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-9vMrpœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "DataFrameOperations-N80fC", + "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-N80fCœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" } ], "nodes": [ @@ -261,7 +374,7 @@ "frozen": false, "icon": "scissors-line-dashed", "legacy": false, - "lf_version": "1.6.0", + "lf_version": "1.6.3.dev0", "metadata": { "code_hash": "f2867efda61f", "dependencies": { @@ -466,8 +579,8 @@ "width": 320 }, "position": { - "x": 1729.1788373023007, - "y": 1330.8003441546418 + "x": 1704.25352249077, + "y": 1199.364065218893 }, "positionAbsolute": { "x": 1683.4543896546102, @@ -477,481 +590,6 @@ "type": "genericNode", "width": 320 }, - { - "data": { - "id": "File-PSU37", - "node": { - "base_classes": [ - "Message" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Loads content from one or more files.", - "display_name": "File", - "documentation": "https://docs.langflow.org/components-data#file", - "edited": true, - "field_order": [ - "path", - "file_path", - "separator", - "silent_errors", - "delete_server_file_after_processing", - "ignore_unsupported_extensions", - "ignore_unspecified_files", - "advanced_mode", - "pipeline", - "ocr_engine", - "md_image_placeholder", - "md_page_break_placeholder", - "doc_key", - "use_multithreading", - "concurrency_multithreading", - "markdown" - ], - "frozen": false, - "icon": "file-text", - "last_updated": "2025-09-26T14:37:42.811Z", - "legacy": false, - "lf_version": "1.6.0", - "metadata": { - "code_hash": "9a1d497f4f91", - "dependencies": { - "dependencies": [ - { - "name": "lfx", - "version": null - } - ], - "total_dependencies": 1 - }, - "module": "custom_components.file" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "Raw Content", - "group_outputs": false, - "hidden": null, - "method": "load_files_message", - "name": "message", - "options": null, - "required_inputs": null, - "selected": "Message", - "tool_mode": true, - "types": [ - "Message" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "advanced_mode": { - "_input_type": "BoolInput", - "advanced": false, - "display_name": "Advanced Parser", - "dynamic": false, - "info": "Enable advanced document processing and export with Docling for PDFs, images, and office documents. Available only for single file processing.Note that advanced document processing can consume significant resources.", - "list": false, - "list_add_label": "Add More", - "name": "advanced_mode", - "placeholder": "", - "real_time_refresh": true, - "required": false, - "show": false, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "bool", - "value": false - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "\"\"\"Enhanced file component with Docling support and process isolation.\n\nNotes:\n-----\n- ALL Docling parsing/export runs in a separate OS process to prevent memory\n growth and native library state from impacting the main Langflow process.\n- Standard text/structured parsing continues to use existing BaseFileComponent\n utilities (and optional threading via `parallel_load_data`).\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport subprocess\nimport sys\nimport textwrap\nfrom copy import deepcopy\nfrom typing import Any\n\nfrom lfx.base.data.base_file import BaseFileComponent\nfrom lfx.base.data.utils import TEXT_FILE_TYPES, parallel_load_data, parse_text_file_to_data\nfrom lfx.inputs.inputs import DropdownInput, MessageTextInput, StrInput\nfrom lfx.io import BoolInput, FileInput, IntInput, Output\nfrom lfx.schema import DataFrame # noqa: TC001\nfrom lfx.schema.data import Data\nfrom lfx.schema.message import Message\n\n\nclass FileComponent(BaseFileComponent):\n \"\"\"File component with optional Docling processing (isolated in a subprocess).\"\"\"\n\n display_name = \"File\"\n description = \"Loads content from one or more files.\"\n documentation: str = \"https://docs.langflow.org/components-data#file\"\n icon = \"file-text\"\n name = \"File\"\n\n # Docling-supported/compatible extensions; TEXT_FILE_TYPES are supported by the base loader.\n VALID_EXTENSIONS = [\n *TEXT_FILE_TYPES,\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"jpeg\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"webp\",\n ]\n\n # Fixed export settings used when markdown export is requested.\n EXPORT_FORMAT = \"Markdown\"\n IMAGE_MODE = \"placeholder\"\n\n _base_inputs = deepcopy(BaseFileComponent.get_base_inputs())\n\n for input_item in _base_inputs:\n if isinstance(input_item, FileInput) and input_item.name == \"path\":\n input_item.real_time_refresh = True\n break\n\n inputs = [\n *_base_inputs,\n BoolInput(\n name=\"advanced_mode\",\n display_name=\"Advanced Parser\",\n value=False,\n real_time_refresh=True,\n info=(\n \"Enable advanced document processing and export with Docling for PDFs, images, and office documents. \"\n \"Available only for single file processing.\"\n \"Note that advanced document processing can consume significant resources.\"\n ),\n show=False,\n ),\n DropdownInput(\n name=\"pipeline\",\n display_name=\"Pipeline\",\n info=\"Docling pipeline to use\",\n options=[\"standard\", \"vlm\"],\n value=\"standard\",\n advanced=True,\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"ocr_engine\",\n display_name=\"OCR Engine\",\n info=\"OCR engine to use. Only available when pipeline is set to 'standard'.\",\n options=[\"None\", \"easyocr\"],\n value=\"easyocr\",\n show=False,\n advanced=True,\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"\",\n advanced=True,\n show=False,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder between pages in the markdown output.\",\n value=\"\",\n advanced=True,\n show=False,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n show=False,\n ),\n # Deprecated input retained for backward-compatibility.\n BoolInput(\n name=\"use_multithreading\",\n display_name=\"[Deprecated] Use Multithreading\",\n advanced=True,\n value=True,\n info=\"Set 'Processing Concurrency' greater than 1 to enable multithreading.\",\n ),\n IntInput(\n name=\"concurrency_multithreading\",\n display_name=\"Processing Concurrency\",\n advanced=True,\n info=\"When multiple files are being processed, the number of files to process concurrently.\",\n value=1,\n ),\n BoolInput(\n name=\"markdown\",\n display_name=\"Markdown Export\",\n info=\"Export processed documents to Markdown format. Only available when advanced mode is enabled.\",\n value=False,\n show=False,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n ]\n\n # ------------------------------ UI helpers --------------------------------------\n\n def _path_value(self, template: dict) -> list[str]:\n \"\"\"Return the list of currently selected file paths from the template.\"\"\"\n return template.get(\"path\", {}).get(\"file_path\", [])\n\n def update_build_config(\n self,\n build_config: dict[str, Any],\n field_value: Any,\n field_name: str | None = None,\n ) -> dict[str, Any]:\n \"\"\"Show/hide Advanced Parser and related fields based on selection context.\"\"\"\n if field_name == \"path\":\n paths = self._path_value(build_config)\n file_path = paths[0] if paths else \"\"\n file_count = len(field_value) if field_value else 0\n\n # Advanced mode only for single (non-tabular) file\n allow_advanced = file_count == 1 and not file_path.endswith((\".csv\", \".xlsx\", \".parquet\"))\n build_config[\"advanced_mode\"][\"show\"] = allow_advanced\n if not allow_advanced:\n build_config[\"advanced_mode\"][\"value\"] = False\n for f in (\"pipeline\", \"ocr_engine\", \"doc_key\", \"md_image_placeholder\", \"md_page_break_placeholder\"):\n if f in build_config:\n build_config[f][\"show\"] = False\n\n # Docling Processing\n elif field_name == \"advanced_mode\":\n for f in (\"pipeline\", \"ocr_engine\", \"doc_key\", \"md_image_placeholder\", \"md_page_break_placeholder\"):\n if f in build_config:\n build_config[f][\"show\"] = bool(field_value)\n\n elif field_name == \"pipeline\":\n if field_value == \"standard\":\n build_config[\"ocr_engine\"][\"show\"] = True\n build_config[\"ocr_engine\"][\"value\"] = \"easyocr\"\n else:\n build_config[\"ocr_engine\"][\"show\"] = False\n build_config[\"ocr_engine\"][\"value\"] = \"None\"\n\n return build_config\n\n def update_outputs(self, frontend_node: dict[str, Any], field_name: str, field_value: Any) -> dict[str, Any]: # noqa: ARG002\n \"\"\"Dynamically show outputs based on file count/type and advanced mode.\"\"\"\n if field_name not in [\"path\", \"advanced_mode\", \"pipeline\"]:\n return frontend_node\n\n template = frontend_node.get(\"template\", {})\n paths = self._path_value(template)\n if not paths:\n return frontend_node\n\n frontend_node[\"outputs\"] = []\n if len(paths) == 1:\n file_path = paths[0] if field_name == \"path\" else frontend_node[\"template\"][\"path\"][\"file_path\"][0]\n if file_path.endswith((\".csv\", \".xlsx\", \".parquet\")):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"dataframe\", method=\"load_files_structured\"),\n )\n elif file_path.endswith(\".json\"):\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Content\", name=\"json\", method=\"load_files_json\"),\n )\n\n advanced_mode = frontend_node.get(\"template\", {}).get(\"advanced_mode\", {}).get(\"value\", False)\n if advanced_mode:\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Structured Output\", name=\"advanced_dataframe\", method=\"load_files_dataframe\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Markdown\", name=\"advanced_markdown\", method=\"load_files_markdown\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n frontend_node[\"outputs\"].append(\n Output(display_name=\"Raw Content\", name=\"message\", method=\"load_files_message\"),\n )\n frontend_node[\"outputs\"].append(\n Output(display_name=\"File Path\", name=\"path\", method=\"load_files_path\"),\n )\n else:\n # Multiple files => DataFrame output; advanced parser disabled\n frontend_node[\"outputs\"].append(Output(display_name=\"Files\", name=\"dataframe\", method=\"load_files\"))\n\n return frontend_node\n\n # ------------------------------ Core processing ----------------------------------\n\n def _is_docling_compatible(self, file_path: str) -> bool:\n \"\"\"Lightweight extension gate for Docling-compatible types.\"\"\"\n docling_exts = (\n \".adoc\",\n \".asciidoc\",\n \".asc\",\n \".bmp\",\n \".csv\",\n \".dotx\",\n \".dotm\",\n \".docm\",\n \".docx\",\n \".htm\",\n \".html\",\n \".jpeg\",\n \".json\",\n \".md\",\n \".pdf\",\n \".png\",\n \".potx\",\n \".ppsx\",\n \".pptm\",\n \".potm\",\n \".ppsm\",\n \".pptx\",\n \".tiff\",\n \".txt\",\n \".xls\",\n \".xlsx\",\n \".xhtml\",\n \".xml\",\n \".webp\",\n )\n return file_path.lower().endswith(docling_exts)\n\n def _process_docling_in_subprocess(self, file_path: str) -> Data | None:\n \"\"\"Run Docling in a separate OS process and map the result to a Data object.\n\n We avoid multiprocessing pickling by launching `python -c \"