From e9de598a0fc87badae648df4cca027b57d9ce71e Mon Sep 17 00:00:00 2001 From: Deon Sanchez <69873175+deon-sanchez@users.noreply.github.com> Date: Fri, 19 Sep 2025 09:26:12 -0600 Subject: [PATCH] Implement ingest settings in UnifiedCloudPicker and IngestSettings components - Added IngestSettings interface to define settings structure. - Integrated ingest settings state management in UnifiedCloudPicker. - Updated IngestSettings component to handle user input for chunk size, chunk overlap, OCR, and picture descriptions. - Passed settings and change handler to IngestSettings from UnifiedCloudPicker. - Enhanced UploadProviderPage to utilize ingest settings during file upload. --- frontend/src/app/upload/[provider]/page.tsx | 11 ++ .../cloud-picker/ingest-settings.tsx | 169 ++++++++++++------ frontend/src/components/cloud-picker/types.ts | 10 ++ .../cloud-picker/unified-cloud-picker.tsx | 24 ++- 4 files changed, 157 insertions(+), 57 deletions(-) diff --git a/frontend/src/app/upload/[provider]/page.tsx b/frontend/src/app/upload/[provider]/page.tsx index 3957a9f9..522a45b7 100644 --- a/frontend/src/app/upload/[provider]/page.tsx +++ b/frontend/src/app/upload/[provider]/page.tsx @@ -5,6 +5,7 @@ import { useParams, useRouter } from "next/navigation"; import { Button } from "@/components/ui/button"; import { ArrowLeft, AlertCircle } from "lucide-react"; import { UnifiedCloudPicker, CloudFile } from "@/components/cloud-picker"; +import type { IngestSettings } from "@/components/cloud-picker/types"; import { useTask } from "@/contexts/task-context"; import { Toast } from "@/components/ui/toast"; @@ -38,6 +39,13 @@ export default function UploadProviderPage() { null ); const [showSuccessToast, setShowSuccessToast] = useState(false); + const [ingestSettings, setIngestSettings] = useState({ + chunkSize: 1000, + chunkOverlap: 200, + ocr: false, + pictureDescriptions: false, + embeddingModel: "text-embedding-3-small", + }); useEffect(() => { const fetchConnectorInfo = async () => { @@ -167,9 +175,11 @@ export default function UploadProviderPage() { connection_id: string; max_files?: number; selected_files?: string[]; + settings?: IngestSettings; } = { connection_id: connector.connectionId, selected_files: selectedFiles.map(file => file.id), + settings: ingestSettings, }; const response = await fetch(`/api/connectors/${connector.type}/sync`, { @@ -337,6 +347,7 @@ export default function UploadProviderPage() { isAuthenticated={true} accessToken={accessToken || undefined} clientId={connector.clientId} + onSettingsChange={setIngestSettings} /> diff --git a/frontend/src/components/cloud-picker/ingest-settings.tsx b/frontend/src/components/cloud-picker/ingest-settings.tsx index d594d86d..d5843a2a 100644 --- a/frontend/src/components/cloud-picker/ingest-settings.tsx +++ b/frontend/src/components/cloud-picker/ingest-settings.tsx @@ -8,75 +8,132 @@ import { CollapsibleTrigger, } from "@/components/ui/collapsible"; import { ChevronRight, Info } from "lucide-react"; +import { IngestSettings as IngestSettingsType } from "./types"; interface IngestSettingsProps { isOpen: boolean; onOpenChange: (open: boolean) => void; + settings?: IngestSettingsType; + onSettingsChange?: (settings: IngestSettingsType) => void; } export const IngestSettings = ({ isOpen, onOpenChange, -}: IngestSettingsProps) => ( - - -
- - Ingest settings -
- + settings, + onSettingsChange, +}: IngestSettingsProps) => { + // Default settings + const defaultSettings: IngestSettingsType = { + chunkSize: 1000, + chunkOverlap: 200, + ocr: false, + pictureDescriptions: false, + embeddingModel: "text-embedding-3-small", + }; - -
-
-
-
Chunk size
- -
-
-
Chunk overlap
- -
+ // Use provided settings or defaults + const currentSettings = settings || defaultSettings; + + const handleSettingsChange = (newSettings: Partial) => { + const updatedSettings = { ...currentSettings, ...newSettings }; + onSettingsChange?.(updatedSettings); + }; + + return ( + + +
+ + Ingest settings
+ + + +
+
+
+
Chunk size
+ + handleSettingsChange({ + chunkSize: parseInt(e.target.value) || 0, + }) + } + /> +
+
+
Chunk overlap
+ + handleSettingsChange({ + chunkOverlap: parseInt(e.target.value) || 0, + }) + } + /> +
+
+ +
+
+
OCR
+
+ Extracts text from images/PDFs. Ingest is slower when enabled. +
+
+ + handleSettingsChange({ ocr: checked }) + } + /> +
+ +
+
+
+ Picture descriptions +
+
+ Adds captions for images. Ingest is more expensive when enabled. +
+
+ + handleSettingsChange({ pictureDescriptions: checked }) + } + /> +
-
-
OCR
-
- Extracts text from images/PDFs. Ingest is slower when enabled. +
+ Embedding model +
+ + handleSettingsChange({ embeddingModel: e.target.value }) + } + placeholder="text-embedding-3-small" + />
-
- -
-
-
- Picture descriptions -
-
- Adds captions for images. Ingest is more expensive when enabled. -
-
- -
- -
-
- Embedding model - -
- -
-
- - -); + + + ); +}; diff --git a/frontend/src/components/cloud-picker/types.ts b/frontend/src/components/cloud-picker/types.ts index 568cb3d5..ca346bf0 100644 --- a/frontend/src/components/cloud-picker/types.ts +++ b/frontend/src/components/cloud-picker/types.ts @@ -23,6 +23,8 @@ export interface UnifiedCloudPickerProps { // OneDrive/SharePoint specific props clientId?: string; baseUrl?: string; + // Ingest settings + onSettingsChange?: (settings: IngestSettings) => void; } export interface GoogleAPI { @@ -94,3 +96,11 @@ export interface GooglePickerBuilder { export interface GooglePicker { setVisible: (visible: boolean) => void; } + +export interface IngestSettings { + chunkSize: number; + chunkOverlap: number; + ocr: boolean; + pictureDescriptions: boolean; + embeddingModel: string; +} diff --git a/frontend/src/components/cloud-picker/unified-cloud-picker.tsx b/frontend/src/components/cloud-picker/unified-cloud-picker.tsx index 24f3e7ae..fd77698f 100644 --- a/frontend/src/components/cloud-picker/unified-cloud-picker.tsx +++ b/frontend/src/components/cloud-picker/unified-cloud-picker.tsx @@ -1,7 +1,11 @@ "use client"; import { useState, useEffect } from "react"; -import { UnifiedCloudPickerProps, CloudFile } from "./types"; +import { + UnifiedCloudPickerProps, + CloudFile, + IngestSettings as IngestSettingsType, +} from "./types"; import { PickerHeader } from "./picker-header"; import { FileList } from "./file-list"; import { IngestSettings } from "./ingest-settings"; @@ -16,6 +20,7 @@ export const UnifiedCloudPicker = ({ onPickerStateChange, clientId, baseUrl, + onSettingsChange, }: UnifiedCloudPickerProps) => { const [isPickerLoaded, setIsPickerLoaded] = useState(false); const [isPickerOpen, setIsPickerOpen] = useState(false); @@ -23,6 +28,21 @@ export const UnifiedCloudPicker = ({ const [isLoadingBaseUrl, setIsLoadingBaseUrl] = useState(false); const [autoBaseUrl, setAutoBaseUrl] = useState(undefined); + // Settings state with defaults + const [ingestSettings, setIngestSettings] = useState({ + chunkSize: 1000, + chunkOverlap: 200, + ocr: false, + pictureDescriptions: false, + embeddingModel: "text-embedding-3-small", + }); + + // Handle settings changes and notify parent + const handleSettingsChange = (newSettings: IngestSettingsType) => { + setIngestSettings(newSettings); + onSettingsChange?.(newSettings); + }; + const effectiveBaseUrl = baseUrl || autoBaseUrl; // Auto-detect base URL for OneDrive personal accounts @@ -167,6 +187,8 @@ export const UnifiedCloudPicker = ({
);