design sweep for ingest for connector page

This commit is contained in:
Brent O'Neill 2025-10-03 15:00:53 -06:00
parent 886297a40d
commit 1eb0fb3593
2 changed files with 332 additions and 335 deletions

View file

@ -12,367 +12,370 @@ import { useTask } from "@/contexts/task-context";
// CloudFile interface is now imported from the unified cloud picker // CloudFile interface is now imported from the unified cloud picker
interface CloudConnector { interface CloudConnector {
id: string; id: string;
name: string; name: string;
description: string; description: string;
status: "not_connected" | "connecting" | "connected" | "error"; status: "not_connected" | "connecting" | "connected" | "error";
type: string; type: string;
connectionId?: string; connectionId?: string;
clientId: string; clientId: string;
hasAccessToken: boolean; hasAccessToken: boolean;
accessTokenError?: string; accessTokenError?: string;
} }
export default function UploadProviderPage() { export default function UploadProviderPage() {
const params = useParams(); const params = useParams();
const router = useRouter(); const router = useRouter();
const provider = params.provider as string; const provider = params.provider as string;
const { addTask, tasks } = useTask(); const { addTask, tasks } = useTask();
const [connector, setConnector] = useState<CloudConnector | null>(null); const [connector, setConnector] = useState<CloudConnector | null>(null);
const [isLoading, setIsLoading] = useState(true); const [isLoading, setIsLoading] = useState(true);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const [accessToken, setAccessToken] = useState<string | null>(null); const [accessToken, setAccessToken] = useState<string | null>(null);
const [selectedFiles, setSelectedFiles] = useState<CloudFile[]>([]); const [selectedFiles, setSelectedFiles] = useState<CloudFile[]>([]);
const [isIngesting, setIsIngesting] = useState<boolean>(false); const [isIngesting, setIsIngesting] = useState<boolean>(false);
const [currentSyncTaskId, setCurrentSyncTaskId] = useState<string | null>( const [currentSyncTaskId, setCurrentSyncTaskId] = useState<string | null>(
null, null
); );
const [ingestSettings, setIngestSettings] = useState<IngestSettings>({ const [ingestSettings, setIngestSettings] = useState<IngestSettings>({
chunkSize: 1000, chunkSize: 1000,
chunkOverlap: 200, chunkOverlap: 200,
ocr: false, ocr: false,
pictureDescriptions: false, pictureDescriptions: false,
embeddingModel: "text-embedding-3-small", embeddingModel: "text-embedding-3-small",
}); });
useEffect(() => { useEffect(() => {
const fetchConnectorInfo = async () => { const fetchConnectorInfo = async () => {
setIsLoading(true); setIsLoading(true);
setError(null); setError(null);
try { try {
// Fetch available connectors to validate the provider // Fetch available connectors to validate the provider
const connectorsResponse = await fetch("/api/connectors"); const connectorsResponse = await fetch("/api/connectors");
if (!connectorsResponse.ok) { if (!connectorsResponse.ok) {
throw new Error("Failed to load connectors"); throw new Error("Failed to load connectors");
} }
const connectorsResult = await connectorsResponse.json(); const connectorsResult = await connectorsResponse.json();
const providerInfo = connectorsResult.connectors[provider]; const providerInfo = connectorsResult.connectors[provider];
if (!providerInfo || !providerInfo.available) { if (!providerInfo || !providerInfo.available) {
setError( setError(
`Cloud provider "${provider}" is not available or configured.`, `Cloud provider "${provider}" is not available or configured.`
); );
return; return;
} }
// Check connector status // Check connector status
const statusResponse = await fetch( const statusResponse = await fetch(
`/api/connectors/${provider}/status`, `/api/connectors/${provider}/status`
); );
if (!statusResponse.ok) { if (!statusResponse.ok) {
throw new Error(`Failed to check ${provider} status`); throw new Error(`Failed to check ${provider} status`);
} }
const statusData = await statusResponse.json(); const statusData = await statusResponse.json();
const connections = statusData.connections || []; const connections = statusData.connections || [];
const activeConnection = connections.find( const activeConnection = connections.find(
(conn: { is_active: boolean; connection_id: string }) => (conn: { is_active: boolean; connection_id: string }) =>
conn.is_active, conn.is_active
); );
const isConnected = activeConnection !== undefined; const isConnected = activeConnection !== undefined;
let hasAccessToken = false; let hasAccessToken = false;
let accessTokenError: string | undefined; let accessTokenError: string | undefined;
// Try to get access token for connected connectors // Try to get access token for connected connectors
if (isConnected && activeConnection) { if (isConnected && activeConnection) {
try { try {
const tokenResponse = await fetch( const tokenResponse = await fetch(
`/api/connectors/${provider}/token?connection_id=${activeConnection.connection_id}`, `/api/connectors/${provider}/token?connection_id=${activeConnection.connection_id}`
); );
if (tokenResponse.ok) { if (tokenResponse.ok) {
const tokenData = await tokenResponse.json(); const tokenData = await tokenResponse.json();
if (tokenData.access_token) { if (tokenData.access_token) {
hasAccessToken = true; hasAccessToken = true;
setAccessToken(tokenData.access_token); setAccessToken(tokenData.access_token);
} }
} else { } else {
const errorData = await tokenResponse const errorData = await tokenResponse
.json() .json()
.catch(() => ({ error: "Token unavailable" })); .catch(() => ({ error: "Token unavailable" }));
accessTokenError = errorData.error || "Access token unavailable"; accessTokenError = errorData.error || "Access token unavailable";
} }
} catch { } catch {
accessTokenError = "Failed to fetch access token"; accessTokenError = "Failed to fetch access token";
} }
} }
setConnector({ setConnector({
id: provider, id: provider,
name: providerInfo.name, name: providerInfo.name,
description: providerInfo.description, description: providerInfo.description,
status: isConnected ? "connected" : "not_connected", status: isConnected ? "connected" : "not_connected",
type: provider, type: provider,
connectionId: activeConnection?.connection_id, connectionId: activeConnection?.connection_id,
clientId: activeConnection?.client_id, clientId: activeConnection?.client_id,
hasAccessToken, hasAccessToken,
accessTokenError, accessTokenError,
}); });
} catch (error) { } catch (error) {
console.error("Failed to load connector info:", error); console.error("Failed to load connector info:", error);
setError( setError(
error instanceof Error error instanceof Error
? error.message ? error.message
: "Failed to load connector information", : "Failed to load connector information"
); );
} finally { } finally {
setIsLoading(false); setIsLoading(false);
} }
}; };
if (provider) { if (provider) {
fetchConnectorInfo(); fetchConnectorInfo();
} }
}, [provider]); }, [provider]);
// Watch for sync task completion and redirect // Watch for sync task completion and redirect
useEffect(() => { useEffect(() => {
if (!currentSyncTaskId) return; if (!currentSyncTaskId) return;
const currentTask = tasks.find( const currentTask = tasks.find(
(task) => task.task_id === currentSyncTaskId, (task) => task.task_id === currentSyncTaskId
); );
if (currentTask && currentTask.status === "completed") { if (currentTask && currentTask.status === "completed") {
// Task completed successfully, show toast and redirect // Task completed successfully, show toast and redirect
setIsIngesting(false); setIsIngesting(false);
setTimeout(() => { setTimeout(() => {
router.push("/knowledge"); router.push("/knowledge");
}, 2000); // 2 second delay to let user see toast }, 2000); // 2 second delay to let user see toast
} else if (currentTask && currentTask.status === "failed") { } else if (currentTask && currentTask.status === "failed") {
// Task failed, clear the tracking but don't redirect // Task failed, clear the tracking but don't redirect
setIsIngesting(false); setIsIngesting(false);
setCurrentSyncTaskId(null); setCurrentSyncTaskId(null);
} }
}, [tasks, currentSyncTaskId, router]); }, [tasks, currentSyncTaskId, router]);
const handleFileSelected = (files: CloudFile[]) => { const handleFileSelected = (files: CloudFile[]) => {
setSelectedFiles(files); setSelectedFiles(files);
console.log(`Selected ${files.length} files from ${provider}:`, files); console.log(`Selected ${files.length} files from ${provider}:`, files);
// You can add additional handling here like triggering sync, etc. // You can add additional handling here like triggering sync, etc.
}; };
const handleSync = async (connector: CloudConnector) => { const handleSync = async (connector: CloudConnector) => {
if (!connector.connectionId || selectedFiles.length === 0) return; if (!connector.connectionId || selectedFiles.length === 0) return;
setIsIngesting(true); setIsIngesting(true);
try { try {
const syncBody: { const syncBody: {
connection_id: string; connection_id: string;
max_files?: number; max_files?: number;
selected_files?: string[]; selected_files?: string[];
settings?: IngestSettings; settings?: IngestSettings;
} = { } = {
connection_id: connector.connectionId, connection_id: connector.connectionId,
selected_files: selectedFiles.map((file) => file.id), selected_files: selectedFiles.map((file) => file.id),
settings: ingestSettings, settings: ingestSettings,
}; };
const response = await fetch(`/api/connectors/${connector.type}/sync`, { const response = await fetch(`/api/connectors/${connector.type}/sync`, {
method: "POST", method: "POST",
headers: { headers: {
"Content-Type": "application/json", "Content-Type": "application/json",
}, },
body: JSON.stringify(syncBody), body: JSON.stringify(syncBody),
}); });
const result = await response.json(); const result = await response.json();
if (response.status === 201) { if (response.status === 201) {
const taskIds = result.task_ids; const taskIds = result.task_ids;
if (taskIds && taskIds.length > 0) { if (taskIds && taskIds.length > 0) {
const taskId = taskIds[0]; // Use the first task ID const taskId = taskIds[0]; // Use the first task ID
addTask(taskId); addTask(taskId);
setCurrentSyncTaskId(taskId); setCurrentSyncTaskId(taskId);
} }
} else { } else {
console.error("Sync failed:", result.error); console.error("Sync failed:", result.error);
} }
} catch (error) { } catch (error) {
console.error("Sync error:", error); console.error("Sync error:", error);
setIsIngesting(false); setIsIngesting(false);
} }
}; };
const getProviderDisplayName = () => { const getProviderDisplayName = () => {
const nameMap: { [key: string]: string } = { const nameMap: { [key: string]: string } = {
google_drive: "Google Drive", google_drive: "Google Drive",
onedrive: "OneDrive", onedrive: "OneDrive",
sharepoint: "SharePoint", sharepoint: "SharePoint",
}; };
return nameMap[provider] || provider; return nameMap[provider] || provider;
}; };
if (isLoading) { if (isLoading) {
return ( return (
<div className="container mx-auto p-6"> <div className="container mx-auto p-6">
<div className="flex items-center justify-center py-12"> <div className="flex items-center justify-center py-12">
<div className="text-center"> <div className="text-center">
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary mx-auto mb-4"></div> <div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary mx-auto mb-4"></div>
<p>Loading {getProviderDisplayName()} connector...</p> <p>Loading {getProviderDisplayName()} connector...</p>
</div> </div>
</div> </div>
</div> </div>
); );
} }
if (error || !connector) { if (error || !connector) {
return ( return (
<div className="container mx-auto p-6"> <div className="container mx-auto p-6">
<div className="mb-6"> <div className="mb-6">
<Button <Button
variant="ghost" variant="ghost"
onClick={() => router.back()} onClick={() => router.back()}
className="mb-4" className="mb-4"
> >
<ArrowLeft className="h-4 w-4 mr-2" /> <ArrowLeft className="h-4 w-4 mr-2" />
Back Back
</Button> </Button>
</div> </div>
<div className="flex items-center justify-center py-12"> <div className="flex items-center justify-center py-12">
<div className="text-center max-w-md"> <div className="text-center max-w-md">
<AlertCircle className="h-12 w-12 text-red-500 mx-auto mb-4" /> <AlertCircle className="h-12 w-12 text-red-500 mx-auto mb-4" />
<h2 className="text-xl font-semibold mb-2"> <h2 className="text-xl font-semibold mb-2">
Provider Not Available Provider Not Available
</h2> </h2>
<p className="text-muted-foreground mb-4">{error}</p> <p className="text-muted-foreground mb-4">{error}</p>
<Button onClick={() => router.push("/settings")}> <Button onClick={() => router.push("/settings")}>
Configure Connectors Configure Connectors
</Button> </Button>
</div> </div>
</div> </div>
</div> </div>
); );
} }
if (connector.status !== "connected") { if (connector.status !== "connected") {
return ( return (
<div className="container mx-auto p-6"> <div className="container mx-auto p-6">
<div className="mb-6"> <div className="mb-6">
<Button <Button
variant="ghost" variant="ghost"
onClick={() => router.back()} onClick={() => router.back()}
className="mb-4" className="mb-4"
> >
<ArrowLeft className="h-4 w-4 mr-2" /> <ArrowLeft className="h-4 w-4 mr-2" />
Back Back
</Button> </Button>
</div> </div>
<div className="flex items-center justify-center py-12"> <div className="flex items-center justify-center py-12">
<div className="text-center max-w-md"> <div className="text-center max-w-md">
<AlertCircle className="h-12 w-12 text-yellow-500 mx-auto mb-4" /> <AlertCircle className="h-12 w-12 text-yellow-500 mx-auto mb-4" />
<h2 className="text-xl font-semibold mb-2"> <h2 className="text-xl font-semibold mb-2">
{connector.name} Not Connected {connector.name} Not Connected
</h2> </h2>
<p className="text-muted-foreground mb-4"> <p className="text-muted-foreground mb-4">
You need to connect your {connector.name} account before you can You need to connect your {connector.name} account before you can
select files. select files.
</p> </p>
<Button onClick={() => router.push("/settings")}> <Button onClick={() => router.push("/settings")}>
Connect {connector.name} Connect {connector.name}
</Button> </Button>
</div> </div>
</div> </div>
</div> </div>
); );
} }
if (!connector.hasAccessToken) { if (!connector.hasAccessToken) {
return ( return (
<div className="container mx-auto p-6"> <div className="container mx-auto p-6">
<div className="mb-6"> <div className="mb-6">
<Button <Button
variant="ghost" variant="ghost"
onClick={() => router.back()} onClick={() => router.back()}
className="mb-4" className="mb-4"
> >
<ArrowLeft className="h-4 w-4 mr-2" /> <ArrowLeft className="h-4 w-4 mr-2" />
Back Back
</Button> </Button>
</div> </div>
<div className="flex items-center justify-center py-12"> <div className="flex items-center justify-center py-12">
<div className="text-center max-w-md"> <div className="text-center max-w-md">
<AlertCircle className="h-12 w-12 text-red-500 mx-auto mb-4" /> <AlertCircle className="h-12 w-12 text-red-500 mx-auto mb-4" />
<h2 className="text-xl font-semibold mb-2"> <h2 className="text-xl font-semibold mb-2">
Access Token Required Access Token Required
</h2> </h2>
<p className="text-muted-foreground mb-4"> <p className="text-muted-foreground mb-4">
{connector.accessTokenError || {connector.accessTokenError ||
`Unable to get access token for ${connector.name}. Try reconnecting your account.`} `Unable to get access token for ${connector.name}. Try reconnecting your account.`}
</p> </p>
<Button onClick={() => router.push("/settings")}> <Button onClick={() => router.push("/settings")}>
Reconnect {connector.name} Reconnect {connector.name}
</Button> </Button>
</div> </div>
</div> </div>
</div> </div>
); );
} }
return ( return (
<div className="container mx-auto max-w-3xl p-6"> <div className="container mx-auto max-w-3xl p-6">
<div className="mb-6 flex gap-2 items-center"> <div className="mb-8 flex gap-2 items-center">
<Button variant="ghost" onClick={() => router.back()}> <Button variant="ghost" onClick={() => router.back()} size="icon">
<ArrowLeft className="h-4 w-4 scale-125" /> <ArrowLeft size={18} />
</Button> </Button>
<h2 className="text-2xl font-bold"> <h2 className="text-xl text-[18px] font-semibold">
Add from {getProviderDisplayName()} Add from {getProviderDisplayName()}
</h2> </h2>
</div> </div>
<div className="max-w-3xl mx-auto"> <div className="max-w-3xl mx-auto">
<UnifiedCloudPicker <UnifiedCloudPicker
provider={ provider={
connector.type as "google_drive" | "onedrive" | "sharepoint" connector.type as "google_drive" | "onedrive" | "sharepoint"
} }
onFileSelected={handleFileSelected} onFileSelected={handleFileSelected}
selectedFiles={selectedFiles} selectedFiles={selectedFiles}
isAuthenticated={true} isAuthenticated={true}
accessToken={accessToken || undefined} accessToken={accessToken || undefined}
clientId={connector.clientId} clientId={connector.clientId}
onSettingsChange={setIngestSettings} onSettingsChange={setIngestSettings}
/> />
</div> </div>
<div className="max-w-3xl mx-auto mt-6"> <div className="max-w-3xl mx-auto mt-6">
<div className="flex justify-between gap-3 mb-4"> <div className="flex justify-between gap-3 mb-4">
<Button <Button
variant="ghost" variant="ghost"
className=" border bg-transparent border-border rounded-lg text-secondary-foreground" className=" border bg-transparent border-border rounded-lg text-secondary-foreground"
onClick={() => router.back()} onClick={() => router.back()}
> >
Back Back
</Button> </Button>
<Button <Button
variant="secondary" variant="secondary"
onClick={() => handleSync(connector)} onClick={() => handleSync(connector)}
disabled={selectedFiles.length === 0 || isIngesting} disabled={selectedFiles.length === 0 || isIngesting}
> >
{isIngesting ? ( {isIngesting ? (
<>Ingesting {selectedFiles.length} Files...</> <>
) : ( Ingesting {selectedFiles.length} file
<>Start ingest</> {selectedFiles.length > 1 ? "s" : ""}...
)} </>
</Button> ) : (
</div> <>Ingest files</>
</div> )}
</div> </Button>
); </div>
</div>
</div>
);
} }

View file

@ -51,19 +51,13 @@ export const PickerHeader = ({
Select files from {getProviderName(provider)} to ingest. Select files from {getProviderName(provider)} to ingest.
</p> </p>
<Button <Button
size="sm"
onClick={onAddFiles} onClick={onAddFiles}
disabled={!isPickerLoaded || isPickerOpen || !accessToken} disabled={!isPickerLoaded || isPickerOpen || !accessToken}
className="bg-foreground text-background hover:bg-foreground/90 font-semibold" className="bg-foreground text-background hover:bg-foreground/90 font-semibold"
> >
<Plus className="h-4 w-4" /> <Plus className="h-4 w-4" />
Add Files {isPickerOpen ? "Opening picker..." : "Add files"}
</Button> </Button>
<div className="text-xs text-muted-foreground pt-4">
csv, json, pdf,{" "}
<a className="underline dark:text-pink-400 text-pink-600">+16 more</a>{" "}
<b>150 MB</b> max
</div>
</CardContent> </CardContent>
</Card> </Card>
); );