ingesting polish
This commit is contained in:
Mike Fortman
parent
5425e41d49
commit
1d3c5459d2
4 changed files with 183 additions and 103 deletions
|
|
@ -3,11 +3,11 @@
|
|||
import { useState, useEffect } from "react"
|
||||
import { useParams, useRouter } from "next/navigation"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { ArrowLeft, AlertCircle } from "lucide-react"
|
||||
import { GoogleDrivePicker } from "@/components/google-drive-picker"
|
||||
import { OneDrivePicker } from "@/components/onedrive-picker"
|
||||
import { useTask } from "@/contexts/task-context"
|
||||
import { Toast } from "@/components/ui/toast"
|
||||
|
||||
interface GoogleDriveFile {
|
||||
id: string
|
||||
|
|
@ -43,15 +43,16 @@ export default function UploadProviderPage() {
|
|||
const params = useParams()
|
||||
const router = useRouter()
|
||||
const provider = params.provider as string
|
||||
const { addTask } = useTask()
|
||||
const { addTask, tasks } = useTask()
|
||||
|
||||
const [connector, setConnector] = useState<CloudConnector | null>(null)
|
||||
const [isLoading, setIsLoading] = useState(true)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [accessToken, setAccessToken] = useState<string | null>(null)
|
||||
const [selectedFiles, setSelectedFiles] = useState<GoogleDriveFile[] | OneDriveFile[]>([])
|
||||
const [isSyncing, setIsSyncing] = useState<boolean>(false)
|
||||
const [syncResult, setSyncResult] = useState<any>(null)
|
||||
const [isIngesting, setIsIngesting] = useState<boolean>(false)
|
||||
const [currentSyncTaskId, setCurrentSyncTaskId] = useState<string | null>(null)
|
||||
const [showSuccessToast, setShowSuccessToast] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
const fetchConnectorInfo = async () => {
|
||||
|
|
@ -130,6 +131,26 @@ export default function UploadProviderPage() {
|
|||
}
|
||||
}, [provider])
|
||||
|
||||
// Watch for sync task completion and redirect
|
||||
useEffect(() => {
|
||||
if (!currentSyncTaskId) return
|
||||
|
||||
const currentTask = tasks.find(task => task.task_id === currentSyncTaskId)
|
||||
|
||||
if (currentTask && currentTask.status === 'completed') {
|
||||
// Task completed successfully, show toast and redirect
|
||||
setIsIngesting(false)
|
||||
setShowSuccessToast(true)
|
||||
setTimeout(() => {
|
||||
router.push('/knowledge')
|
||||
}, 2000) // 2 second delay to let user see toast
|
||||
} else if (currentTask && currentTask.status === 'failed') {
|
||||
// Task failed, clear the tracking but don't redirect
|
||||
setIsIngesting(false)
|
||||
setCurrentSyncTaskId(null)
|
||||
}
|
||||
}, [tasks, currentSyncTaskId, router])
|
||||
|
||||
const handleFileSelected = (files: GoogleDriveFile[] | OneDriveFile[]) => {
|
||||
setSelectedFiles(files)
|
||||
console.log(`Selected ${files.length} files from ${provider}:`, files)
|
||||
|
|
@ -139,8 +160,7 @@ export default function UploadProviderPage() {
|
|||
const handleSync = async (connector: CloudConnector) => {
|
||||
if (!connector.connectionId || selectedFiles.length === 0) return
|
||||
|
||||
setIsSyncing(true)
|
||||
setSyncResult(null)
|
||||
setIsIngesting(true)
|
||||
|
||||
try {
|
||||
const syncBody: {
|
||||
|
|
@ -163,26 +183,18 @@ export default function UploadProviderPage() {
|
|||
const result = await response.json()
|
||||
|
||||
if (response.status === 201) {
|
||||
const taskId = result.task_id
|
||||
if (taskId) {
|
||||
const taskIds = result.task_ids
|
||||
if (taskIds && taskIds.length > 0) {
|
||||
const taskId = taskIds[0] // Use the first task ID
|
||||
addTask(taskId)
|
||||
setSyncResult({
|
||||
processed: 0,
|
||||
total: selectedFiles.length,
|
||||
status: 'started'
|
||||
})
|
||||
setCurrentSyncTaskId(taskId)
|
||||
}
|
||||
} else if (response.ok) {
|
||||
setSyncResult(result)
|
||||
} else {
|
||||
console.error('Sync failed:', result.error)
|
||||
setSyncResult({ error: result.error || 'Sync failed' })
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Sync error:', error)
|
||||
setSyncResult({ error: 'Network error occurred' })
|
||||
} finally {
|
||||
setIsSyncing(false)
|
||||
setIsIngesting(false)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -297,26 +309,18 @@ export default function UploadProviderPage() {
|
|||
}
|
||||
|
||||
return (
|
||||
<div className="container mx-auto p-6">
|
||||
<div className="mb-6">
|
||||
<div className="container mx-auto max-w-3xl p-6">
|
||||
<div className="mb-6 flex gap-2 items-center">
|
||||
<Button
|
||||
variant="ghost"
|
||||
onClick={() => router.back()}
|
||||
className="mb-4"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4 mr-2" />
|
||||
Back
|
||||
<ArrowLeft className="h-4 w-4 scale-125 mr-2" />
|
||||
</Button>
|
||||
|
||||
<div className="mb-6 max-w-4xl mx-auto">
|
||||
<h1 className="text-2xl font-bold mb-2">Select Files from {connector.name}</h1>
|
||||
<p className="text-muted-foreground">
|
||||
Choose specific files from your {connector.name} account to add to your knowledge base.
|
||||
</p>
|
||||
</div>
|
||||
<h2 className="text-2xl font-bold">Add Cloud Knowledge</h2>
|
||||
</div>
|
||||
|
||||
<div className="max-w-4xl mx-auto">
|
||||
<div className="max-w-3xl mx-auto">
|
||||
{connector.type === "google_drive" && (
|
||||
<GoogleDrivePicker
|
||||
onFileSelected={handleFileSelected}
|
||||
|
|
@ -338,44 +342,29 @@ export default function UploadProviderPage() {
|
|||
</div>
|
||||
|
||||
{selectedFiles.length > 0 && (
|
||||
<div className="max-w-4xl mx-auto mt-8">
|
||||
<div className="flex justify-center gap-3 mb-4">
|
||||
<div className="max-w-3xl mx-auto mt-8">
|
||||
<div className="flex justify-end gap-3 mb-4">
|
||||
<Button
|
||||
onClick={() => handleSync(connector)}
|
||||
disabled={selectedFiles.length === 0 || isSyncing}
|
||||
disabled={selectedFiles.length === 0 || isIngesting}
|
||||
>
|
||||
{isSyncing ? (
|
||||
<>Syncing {selectedFiles.length} Selected Files...</>
|
||||
{isIngesting ? (
|
||||
<>Ingesting {selectedFiles.length} Files...</>
|
||||
) : (
|
||||
<>Sync Selected Files ({selectedFiles.length})</>
|
||||
<>Ingest Files ({selectedFiles.length})</>
|
||||
)}
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => setSelectedFiles([])}>
|
||||
Clear Selection
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{syncResult && (
|
||||
<div className="p-3 bg-gray-100 rounded text-sm text-center">
|
||||
{syncResult.error ? (
|
||||
<div className="text-red-600">Error: {syncResult.error}</div>
|
||||
) : syncResult.status === 'started' ? (
|
||||
<div className="text-blue-600">
|
||||
Sync started for {syncResult.total} files. Check the task notification for progress.
|
||||
</div>
|
||||
) : (
|
||||
<div className="text-green-600">
|
||||
<div>Processed: {syncResult.processed || 0}</div>
|
||||
<div>Added: {syncResult.added || 0}</div>
|
||||
{syncResult.errors && <div>Errors: {syncResult.errors}</div>}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Success toast notification */}
|
||||
<Toast
|
||||
message="Ingested successfully!."
|
||||
show={showSuccessToast}
|
||||
onHide={() => setShowSuccessToast(false)}
|
||||
duration={20000}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -3,7 +3,8 @@
|
|||
import { useState, useEffect } from "react"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { FileText, Folder, X } from "lucide-react"
|
||||
import { FileText, Folder, Plus, Trash2 } from "lucide-react"
|
||||
import { Card, CardContent } from "@/components/ui/card"
|
||||
|
||||
interface GoogleDrivePickerProps {
|
||||
onFileSelected: (files: GoogleDriveFile[]) => void
|
||||
|
|
@ -19,6 +20,9 @@ interface GoogleDriveFile {
|
|||
mimeType: string
|
||||
webViewLink?: string
|
||||
iconLink?: string
|
||||
size?: number
|
||||
modifiedTime?: string
|
||||
isFolder?: boolean
|
||||
}
|
||||
|
||||
interface GoogleAPI {
|
||||
|
|
@ -174,17 +178,52 @@ export function GoogleDrivePicker({
|
|||
}
|
||||
}
|
||||
|
||||
const pickerCallback = (data: GooglePickerData) => {
|
||||
const pickerCallback = async (data: GooglePickerData) => {
|
||||
if (data.action === window.google.picker.Action.PICKED) {
|
||||
const files: GoogleDriveFile[] = data.docs.map((doc: GooglePickerDocument) => ({
|
||||
id: doc[window.google.picker.Document.ID],
|
||||
name: doc[window.google.picker.Document.NAME],
|
||||
mimeType: doc[window.google.picker.Document.MIME_TYPE],
|
||||
webViewLink: doc[window.google.picker.Document.URL],
|
||||
iconLink: doc[window.google.picker.Document.ICON_URL]
|
||||
iconLink: doc[window.google.picker.Document.ICON_URL],
|
||||
size: doc['sizeBytes'] ? parseInt(doc['sizeBytes']) : undefined,
|
||||
modifiedTime: doc['lastEditedUtc'],
|
||||
isFolder: doc[window.google.picker.Document.MIME_TYPE] === 'application/vnd.google-apps.folder'
|
||||
}))
|
||||
|
||||
onFileSelected(files)
|
||||
// If size is still missing, try to fetch it via Google Drive API
|
||||
if (accessToken && files.some(f => !f.size && !f.isFolder)) {
|
||||
try {
|
||||
const enrichedFiles = await Promise.all(files.map(async (file) => {
|
||||
if (!file.size && !file.isFolder) {
|
||||
try {
|
||||
const response = await fetch(`https://www.googleapis.com/drive/v3/files/${file.id}?fields=size,modifiedTime`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${accessToken}`
|
||||
}
|
||||
})
|
||||
if (response.ok) {
|
||||
const fileDetails = await response.json()
|
||||
return {
|
||||
...file,
|
||||
size: fileDetails.size ? parseInt(fileDetails.size) : undefined,
|
||||
modifiedTime: fileDetails.modifiedTime || file.modifiedTime
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Failed to fetch file details:', error)
|
||||
}
|
||||
}
|
||||
return file
|
||||
}))
|
||||
onFileSelected(enrichedFiles)
|
||||
} catch (error) {
|
||||
console.warn('Failed to enrich file data:', error)
|
||||
onFileSelected(files)
|
||||
}
|
||||
} else {
|
||||
onFileSelected(files)
|
||||
}
|
||||
}
|
||||
|
||||
setIsPickerOpen(false)
|
||||
|
|
@ -218,6 +257,14 @@ export function GoogleDrivePicker({
|
|||
return typeMap[mimeType] || 'Document'
|
||||
}
|
||||
|
||||
const formatFileSize = (bytes?: number) => {
|
||||
if (!bytes) return ''
|
||||
const sizes = ['B', 'KB', 'MB', 'GB', 'TB']
|
||||
if (bytes === 0) return '0 B'
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(1024))
|
||||
return `${(bytes / Math.pow(1024, i)).toFixed(1)} ${sizes[i]}`
|
||||
}
|
||||
|
||||
if (!isAuthenticated) {
|
||||
return (
|
||||
<div className="text-sm text-muted-foreground p-4 bg-muted/20 rounded-md">
|
||||
|
|
@ -228,29 +275,38 @@ export function GoogleDrivePicker({
|
|||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h4 className="text-sm font-medium">File Selection</h4>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Choose specific files to sync instead of syncing everything
|
||||
<Card>
|
||||
<CardContent className="flex flex-col items-center text-center p-6">
|
||||
<p className="text-sm text-muted-foreground mb-4">
|
||||
Select files from Google Drive to ingest.
|
||||
</p>
|
||||
</div>
|
||||
<Button
|
||||
onClick={openPicker}
|
||||
disabled={!isPickerLoaded || isPickerOpen || !accessToken}
|
||||
size="sm"
|
||||
variant="outline"
|
||||
>
|
||||
{isPickerOpen ? 'Opening Picker...' : 'Add Files'}
|
||||
</Button>
|
||||
</div>
|
||||
<Button
|
||||
onClick={openPicker}
|
||||
disabled={!isPickerLoaded || isPickerOpen || !accessToken}
|
||||
className="bg-foreground text-background hover:bg-foreground/90"
|
||||
>
|
||||
<Plus className="h-4 w-4" />
|
||||
{isPickerOpen ? 'Opening Picker...' : 'Add Files'}
|
||||
</Button>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{selectedFiles.length > 0 && (
|
||||
<div className="space-y-2">
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Selected files ({selectedFiles.length}):
|
||||
</p>
|
||||
<div className="max-h-32 overflow-y-auto space-y-1">
|
||||
<div className="flex items-center justify-between">
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Added files
|
||||
</p>
|
||||
<Button
|
||||
onClick={() => onFileSelected([])}
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="text-xs h-6"
|
||||
>
|
||||
Clear all
|
||||
</Button>
|
||||
</div>
|
||||
<div className="max-h-64 overflow-y-auto space-y-1">
|
||||
{selectedFiles.map((file) => (
|
||||
<div
|
||||
key={file.id}
|
||||
|
|
@ -263,25 +319,21 @@ export function GoogleDrivePicker({
|
|||
{getMimeTypeLabel(file.mimeType)}
|
||||
</Badge>
|
||||
</div>
|
||||
<Button
|
||||
onClick={() => removeFile(file.id)}
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="h-6 w-6 p-0"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
</Button>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-xs text-muted-foreground">{formatFileSize(file.size)}</span>
|
||||
<Button
|
||||
onClick={() => removeFile(file.id)}
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="h-6 w-6 p-0"
|
||||
>
|
||||
<Trash2 className="h-3 w-3" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<Button
|
||||
onClick={() => onFileSelected([])}
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="text-xs h-6"
|
||||
>
|
||||
Clear all
|
||||
</Button>
|
||||
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
import { useState, useEffect } from "react"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { FileText, Folder, X } from "lucide-react"
|
||||
import { FileText, Folder, Trash2, X } from "lucide-react"
|
||||
|
||||
interface OneDrivePickerProps {
|
||||
onFileSelected: (files: OneDriveFile[]) => void
|
||||
|
|
@ -283,7 +283,7 @@ export function OneDrivePicker({
|
|||
<p className="text-xs text-muted-foreground">
|
||||
Selected files ({selectedFiles.length}):
|
||||
</p>
|
||||
<div className="max-h-32 overflow-y-auto space-y-1">
|
||||
<div className="max-h-48 overflow-y-auto space-y-1">
|
||||
{selectedFiles.map((file) => (
|
||||
<div
|
||||
key={file.id}
|
||||
|
|
@ -302,7 +302,7 @@ export function OneDrivePicker({
|
|||
variant="ghost"
|
||||
className="h-6 w-6 p-0"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
<Trash2 className="h-3 w-3" />
|
||||
</Button>
|
||||
</div>
|
||||
))}
|
||||
|
|
|
|||
39
frontend/src/components/ui/toast.tsx
Normal file
39
frontend/src/components/ui/toast.tsx
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
"use client"
|
||||
|
||||
import { useState, useEffect } from 'react'
|
||||
import { Check } from 'lucide-react'
|
||||
|
||||
interface ToastProps {
|
||||
message: string
|
||||
show: boolean
|
||||
onHide?: () => void
|
||||
duration?: number
|
||||
}
|
||||
|
||||
export function Toast({ message, show, onHide, duration = 3000 }: ToastProps) {
|
||||
const [isVisible, setIsVisible] = useState(show)
|
||||
|
||||
useEffect(() => {
|
||||
setIsVisible(show)
|
||||
|
||||
if (show && duration > 0) {
|
||||
const timer = setTimeout(() => {
|
||||
setIsVisible(false)
|
||||
onHide?.()
|
||||
}, duration)
|
||||
|
||||
return () => clearTimeout(timer)
|
||||
}
|
||||
}, [show, duration, onHide])
|
||||
|
||||
if (!isVisible) return null
|
||||
|
||||
return (
|
||||
<div className="fixed bottom-4 left-4 z-50 animate-in slide-in-from-bottom-full">
|
||||
<div className="bg-green-600 text-white px-4 py-3 rounded-lg shadow-lg flex items-center gap-2 max-w-md">
|
||||
<Check className="h-4 w-4 flex-shrink-0" />
|
||||
<span className="text-sm font-medium">{message}</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue