diff --git a/docs/.gitignore b/docs/.gitignore index 0247a7ea..c010f4db 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -23,3 +23,4 @@ yarn-error.log* !package.json !package-lock.json !yarn.lock +!scraper.config.json diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index ab33c338..f0767fa6 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -25,6 +25,9 @@ const config = { // For GitHub pages deployment, it is often '//' baseUrl: process.env.BASE_URL ? process.env.BASE_URL : '/', + // Control search engine indexing - set to true to prevent indexing + noIndex: true, + // GitHub pages deployment config. // If you aren't using GitHub pages, you don't need these. organizationName: 'langflow-ai', // Usually your GitHub org/user name. diff --git a/docs/package.json b/docs/package.json index 0bbba668..0ca96c29 100644 --- a/docs/package.json +++ b/docs/package.json @@ -6,6 +6,7 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "docusaurus build", + "build:pdf": "npm run build && npm run serve & sleep 10 && npx docusaurus-to-pdf && pkill -f 'docusaurus serve'", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", diff --git a/docs/pdf/openrag-documentation.pdf b/docs/pdf/openrag-documentation.pdf new file mode 100644 index 00000000..cec6d2aa Binary files /dev/null and b/docs/pdf/openrag-documentation.pdf differ diff --git a/docs/scraper.config.json b/docs/scraper.config.json new file mode 100644 index 00000000..94c57806 --- /dev/null +++ b/docs/scraper.config.json @@ -0,0 +1,7 @@ +{ + "baseUrl": "http://localhost:3000", + "entryPoint": "http://localhost:3000", + "outputDir": "./pdf/openrag-documentation.pdf", + "customStyles": "table { max-width: 3500px !important; } .navbar, .footer, .breadcrumbs { display: none !important; }", + "forceImages": true +} \ No newline at end of file diff --git a/docs/static/robots.txt b/docs/static/robots.txt new file mode 100644 index 00000000..783e3e94 --- /dev/null +++ b/docs/static/robots.txt @@ -0,0 +1,12 @@ +# Robots.txt for OpenRAG Documentation + +# Block all crawlers by default +User-agent: * +Disallow: / + +# Allow specific crawlers if needed (uncomment when ready for launch) +# User-agent: Googlebot +# Allow: / + +# Sitemap location (uncomment when ready for launch) +# Sitemap: https://docs.openr.ag/sitemap.xml diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx index 19ddc387..fc522f9b 100644 --- a/frontend/components/knowledge-dropdown.tsx +++ b/frontend/components/knowledge-dropdown.tsx @@ -26,6 +26,10 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { useTask } from "@/contexts/task-context"; import { cn } from "@/lib/utils"; +import { + duplicateCheck, + uploadFile as uploadFileUtil, +} from "@/lib/upload-utils"; import type { File as SearchFile } from "@/src/app/api/queries/useGetSearchQuery"; export function KnowledgeDropdown() { @@ -163,8 +167,17 @@ export function KnowledgeDropdown() { fileInputRef.current?.click(); }; - const handleFileChange = async (e: React.ChangeEvent) => { - const files = e.target.files; + const resetFileInput = () => { + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } + }; + + const handleFileChange = async ( + event: React.ChangeEvent + ) => { + const files = event.target.files; + if (files && files.length > 0) { const file = files[0]; @@ -172,37 +185,16 @@ export function KnowledgeDropdown() { setIsOpen(false); try { - // Check if filename already exists (using ORIGINAL filename) console.log("[Duplicate Check] Checking file:", file.name); - const checkResponse = await fetch( - `/api/documents/check-filename?filename=${encodeURIComponent( - file.name - )}` - ); - - console.log("[Duplicate Check] Response status:", checkResponse.status); - - if (!checkResponse.ok) { - const errorText = await checkResponse.text(); - console.error("[Duplicate Check] Error response:", errorText); - throw new Error( - `Failed to check duplicates: ${checkResponse.statusText}` - ); - } - - const checkData = await checkResponse.json(); + const checkData = await duplicateCheck(file); console.log("[Duplicate Check] Result:", checkData); if (checkData.exists) { - // Show duplicate handling dialog console.log("[Duplicate Check] Duplicate detected, showing dialog"); setPendingFile(file); setDuplicateFilename(file.name); setShowDuplicateDialog(true); - // Reset file input - if (fileInputRef.current) { - fileInputRef.current.value = ""; - } + resetFileInput(); return; } @@ -217,105 +209,20 @@ export function KnowledgeDropdown() { } } - // Reset file input - if (fileInputRef.current) { - fileInputRef.current.value = ""; - } + resetFileInput(); }; const uploadFile = async (file: File, replace: boolean) => { setFileUploading(true); - // Trigger the same file upload event as the chat page - window.dispatchEvent( - new CustomEvent("fileUploadStart", { - detail: { filename: file.name }, - }) - ); - try { - const formData = new FormData(); - formData.append("file", file); - formData.append("replace_duplicates", replace.toString()); - - // Use router upload and ingest endpoint (automatically routes based on configuration) - const uploadIngestRes = await fetch("/api/router/upload_ingest", { - method: "POST", - body: formData, - }); - - const uploadIngestJson = await uploadIngestRes.json(); - - if (!uploadIngestRes.ok) { - throw new Error(uploadIngestJson?.error || "Upload and ingest failed"); - } - - // Extract results from the response - handle both unified and simple formats - const fileId = - uploadIngestJson?.upload?.id || - uploadIngestJson?.id || - uploadIngestJson?.task_id; - const filePath = - uploadIngestJson?.upload?.path || uploadIngestJson?.path || "uploaded"; - const runJson = uploadIngestJson?.ingestion; - const deleteResult = uploadIngestJson?.deletion; - console.log("c", uploadIngestJson); - if (!fileId) { - throw new Error("Upload successful but no file id returned"); - } - // Check if ingestion actually succeeded - if ( - runJson && - runJson.status !== "COMPLETED" && - runJson.status !== "SUCCESS" - ) { - const errorMsg = runJson.error || "Ingestion pipeline failed"; - throw new Error( - `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.` - ); - } - // Log deletion status if provided - if (deleteResult) { - if (deleteResult.status === "deleted") { - console.log( - "File successfully cleaned up from Langflow:", - deleteResult.file_id - ); - } else if (deleteResult.status === "delete_failed") { - console.warn( - "Failed to cleanup file from Langflow:", - deleteResult.error - ); - } - } - // Notify UI - window.dispatchEvent( - new CustomEvent("fileUploaded", { - detail: { - file: file, - result: { - file_id: fileId, - file_path: filePath, - run: runJson, - deletion: deleteResult, - unified: true, - }, - }, - }) - ); - + await uploadFileUtil(file, replace); refetchTasks(); } catch (error) { - window.dispatchEvent( - new CustomEvent("fileUploadError", { - detail: { - filename: file.name, - error: error instanceof Error ? error.message : "Upload failed", - }, - }) - ); + toast.error("Upload failed", { + description: error instanceof Error ? error.message : "Unknown error", + }); } finally { - window.dispatchEvent(new CustomEvent("fileUploadComplete")); setFileUploading(false); } }; @@ -332,6 +239,7 @@ export function KnowledgeDropdown() { }); await uploadFile(pendingFile, true); + setPendingFile(null); setDuplicateFilename(""); } diff --git a/frontend/lib/upload-utils.ts b/frontend/lib/upload-utils.ts new file mode 100644 index 00000000..6a7e7301 --- /dev/null +++ b/frontend/lib/upload-utils.ts @@ -0,0 +1,138 @@ +export interface DuplicateCheckResponse { + exists: boolean; + [key: string]: unknown; +} + +export interface UploadFileResult { + fileId: string; + filePath: string; + run: unknown; + deletion: unknown; + unified: boolean; + raw: unknown; +} + +export async function duplicateCheck( + file: File +): Promise { + const response = await fetch( + `/api/documents/check-filename?filename=${encodeURIComponent(file.name)}` + ); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + errorText || `Failed to check duplicates: ${response.statusText}` + ); + } + + return response.json(); +} + +export async function uploadFile( + file: File, + replace = false +): Promise { + window.dispatchEvent( + new CustomEvent("fileUploadStart", { + detail: { filename: file.name }, + }) + ); + + try { + const formData = new FormData(); + formData.append("file", file); + formData.append("replace_duplicates", replace.toString()); + + const uploadResponse = await fetch("/api/router/upload_ingest", { + method: "POST", + body: formData, + }); + + let payload: unknown; + try { + payload = await uploadResponse.json(); + } catch (error) { + throw new Error("Upload failed: unable to parse server response"); + } + + const uploadIngestJson = + typeof payload === "object" && payload !== null ? payload : {}; + + if (!uploadResponse.ok) { + const errorMessage = + (uploadIngestJson as { error?: string }).error || + "Upload and ingest failed"; + throw new Error(errorMessage); + } + + const fileId = + (uploadIngestJson as { upload?: { id?: string } }).upload?.id || + (uploadIngestJson as { id?: string }).id || + (uploadIngestJson as { task_id?: string }).task_id; + const filePath = + (uploadIngestJson as { upload?: { path?: string } }).upload?.path || + (uploadIngestJson as { path?: string }).path || + "uploaded"; + const runJson = (uploadIngestJson as { ingestion?: unknown }).ingestion; + const deletionJson = (uploadIngestJson as { deletion?: unknown }).deletion; + + if (!fileId) { + throw new Error("Upload successful but no file id returned"); + } + + if ( + runJson && + typeof runJson === "object" && + "status" in (runJson as Record) && + (runJson as { status?: string }).status !== "COMPLETED" && + (runJson as { status?: string }).status !== "SUCCESS" + ) { + const errorMsg = + (runJson as { error?: string }).error || + "Ingestion pipeline failed"; + throw new Error( + `Ingestion failed: ${errorMsg}. Try setting DISABLE_INGEST_WITH_LANGFLOW=true if you're experiencing Langflow component issues.` + ); + } + + const result: UploadFileResult = { + fileId, + filePath, + run: runJson, + deletion: deletionJson, + unified: true, + raw: uploadIngestJson, + }; + + window.dispatchEvent( + new CustomEvent("fileUploaded", { + detail: { + file, + result: { + file_id: fileId, + file_path: filePath, + run: runJson, + deletion: deletionJson, + unified: true, + }, + }, + }) + ); + + return result; + } catch (error) { + window.dispatchEvent( + new CustomEvent("fileUploadError", { + detail: { + filename: file.name, + error: + error instanceof Error ? error.message : "Upload failed", + }, + }) + ); + throw error; + } finally { + window.dispatchEvent(new CustomEvent("fileUploadComplete")); + } +} diff --git a/frontend/src/app/new-onboarding/components/onboarding-content.tsx b/frontend/src/app/new-onboarding/components/onboarding-content.tsx index 46886f47..2a5b9542 100644 --- a/frontend/src/app/new-onboarding/components/onboarding-content.tsx +++ b/frontend/src/app/new-onboarding/components/onboarding-content.tsx @@ -1,6 +1,6 @@ "use client"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { StickToBottom } from "use-stick-to-bottom"; import { AssistantMessage } from "@/app/chat/components/assistant-message"; import { UserMessage } from "@/app/chat/components/user-message"; @@ -8,7 +8,9 @@ import Nudges from "@/app/chat/nudges"; import type { Message } from "@/app/chat/types"; import OnboardingCard from "@/app/onboarding/components/onboarding-card"; import { useChatStreaming } from "@/hooks/useChatStreaming"; + import { OnboardingStep } from "./onboarding-step"; +import OnboardingUpload from "./onboarding-upload"; export function OnboardingContent({ handleStepComplete, @@ -57,6 +59,12 @@ export function OnboardingContent({ // Determine which message to show (streaming takes precedence) const displayMessage = streamingMessage || assistantMessage; + useEffect(() => { + if (currentStep === 1 && !isLoading && !!displayMessage) { + handleStepComplete(); + } + }, [isLoading, displayMessage, handleStepComplete, currentStep]); + return (
+ {/* Step 1 */} = 0} isCompleted={currentStep > 0} @@ -74,6 +83,7 @@ export function OnboardingContent({ + {/* Step 2 */} = 1} isCompleted={currentStep > 1 || !!selectedNudge} @@ -92,7 +102,7 @@ export function OnboardingContent({ {currentStep >= 1 && !!selectedNudge && ( 1} + isCompleted={currentStep > 2} /> )} @@ -100,65 +110,41 @@ export function OnboardingContent({ {currentStep >= 1 && !!selectedNudge && (displayMessage || isLoading) && ( - <> - {}} - isStreaming={!!streamingMessage} - isCompleted={currentStep > 1} - /> - {!isLoading && displayMessage && currentStep === 1 && ( -
- -
- )} - + {}} + isStreaming={!!streamingMessage} + isCompleted={currentStep > 2} + /> )} - + + {/* Step 3 */} = 2} + isVisible={currentStep >= 2 && !isLoading && !!displayMessage} isCompleted={currentStep > 2} - text="Step 2: Connect your model" + text="Now, let's add your data." + hideIcon={true} > -
-

- Choose and connect your preferred AI model provider. -

- -
+ + {/* Step 4 */} = 3} isCompleted={currentStep > 3} text="Step 3: You're all set!" >
-

- Your account is ready to use. Let's start chatting! -

- +
diff --git a/frontend/src/app/new-onboarding/components/onboarding-step.tsx b/frontend/src/app/new-onboarding/components/onboarding-step.tsx index 613f08a9..84cfd6f2 100644 --- a/frontend/src/app/new-onboarding/components/onboarding-step.tsx +++ b/frontend/src/app/new-onboarding/components/onboarding-step.tsx @@ -12,6 +12,7 @@ interface OnboardingStepProps { isCompleted?: boolean; icon?: ReactNode; isMarkdown?: boolean; + hideIcon?: boolean; } export function OnboardingStep({ @@ -21,6 +22,7 @@ export function OnboardingStep({ isCompleted = false, icon, isMarkdown = false, + hideIcon = false, }: OnboardingStepProps) { const [displayedText, setDisplayedText] = useState(""); const [showChildren, setShowChildren] = useState(false); @@ -66,13 +68,17 @@ export function OnboardingStep({ > - - + hideIcon ? ( +
+ ) : ( + icon || ( +
+ +
+ ) ) } > diff --git a/frontend/src/app/new-onboarding/components/onboarding-upload.tsx b/frontend/src/app/new-onboarding/components/onboarding-upload.tsx new file mode 100644 index 00000000..00d90097 --- /dev/null +++ b/frontend/src/app/new-onboarding/components/onboarding-upload.tsx @@ -0,0 +1,114 @@ +import { ChangeEvent, useRef, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { duplicateCheck, uploadFile } from "@/lib/upload-utils"; +import { AnimatePresence, motion } from "motion/react"; +import { AnimatedProviderSteps } from "@/app/onboarding/components/animated-provider-steps"; + +interface OnboardingUploadProps { + onComplete: () => void; +} + +const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => { + const fileInputRef = useRef(null); + const [isUploading, setIsUploading] = useState(false); + const [currentStep, setCurrentStep] = useState(null); + + const STEP_LIST = [ + "Analyzing your document", + "Ingesting your document", + ]; + + const resetFileInput = () => { + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } + }; + + const handleUploadClick = () => { + fileInputRef.current?.click(); + }; + + + const performUpload = async (file: File, replace = false) => { + setIsUploading(true); + try { + setCurrentStep(1); + await uploadFile(file, replace); + console.log("Document uploaded successfully"); + } catch (error) { + console.error("Upload failed", (error as Error).message); + } finally { + setIsUploading(false); + setCurrentStep(STEP_LIST.length); + onComplete(); + } + }; + + const handleFileChange = async (event: ChangeEvent) => { + const selectedFile = event.target.files?.[0]; + if (!selectedFile) { + resetFileInput(); + return; + } + + try { + setCurrentStep(0); + const duplicateInfo = await duplicateCheck(selectedFile); + if (duplicateInfo.exists) { + console.log("Duplicate file detected"); + return; + } + + await performUpload(selectedFile, false); + } catch (error) { + console.error("Unable to prepare file for upload", (error as Error).message); + } finally { + resetFileInput(); + } + }; + + + return ( + + {currentStep === null ? ( + + + + + ) : ( + + + + )} + + ) +} + +export default OnboardingUpload; diff --git a/frontend/src/app/new-onboarding/page.tsx b/frontend/src/app/new-onboarding/page.tsx index d7e122d5..9c1aeec7 100644 --- a/frontend/src/app/new-onboarding/page.tsx +++ b/frontend/src/app/new-onboarding/page.tsx @@ -3,11 +3,7 @@ import { Suspense, useState } from "react"; import { DoclingHealthBanner } from "@/components/docling-health-banner"; import { ProtectedRoute } from "@/components/protected-route"; -import { DotPattern } from "@/components/ui/dot-pattern"; -import { cn } from "@/lib/utils"; -import OnboardingCard from "../onboarding/components/onboarding-card"; import { OnboardingContent } from "./components/onboarding-content"; -import { OnboardingStep } from "./components/onboarding-step"; import { ProgressBar } from "./components/progress-bar"; const TOTAL_STEPS = 4; @@ -28,7 +24,7 @@ function NewOnboardingPage() { {/* Chat-like content area */}
- +
diff --git a/frontend/src/app/onboarding/components/animated-provider-steps.tsx b/frontend/src/app/onboarding/components/animated-provider-steps.tsx index cc52b783..f2b48c99 100644 --- a/frontend/src/app/onboarding/components/animated-provider-steps.tsx +++ b/frontend/src/app/onboarding/components/animated-provider-steps.tsx @@ -9,16 +9,12 @@ import { cn } from "@/lib/utils"; export function AnimatedProviderSteps({ currentStep, setCurrentStep, + steps, }: { currentStep: number; setCurrentStep: (step: number) => void; + steps: string[]; }) { - const steps = [ - "Setting up your model provider", - "Defining schema", - "Configuring Langflow", - "Ingesting sample data", - ]; useEffect(() => { if (currentStep < steps.length - 1) { @@ -27,7 +23,7 @@ export function AnimatedProviderSteps({ }, 1000); return () => clearInterval(interval); } - }, [currentStep, setCurrentStep]); + }, [currentStep, setCurrentStep, steps]); const isDone = currentStep >= steps.length; diff --git a/frontend/src/app/onboarding/components/onboarding-card.tsx b/frontend/src/app/onboarding/components/onboarding-card.tsx index 61ee85d8..745dd076 100644 --- a/frontend/src/app/onboarding/components/onboarding-card.tsx +++ b/frontend/src/app/onboarding/components/onboarding-card.tsx @@ -35,7 +35,15 @@ interface OnboardingCardProps { onComplete: () => void; } -const TOTAL_PROVIDER_STEPS = 4; + +const STEP_LIST = [ + "Setting up your model provider", + "Defining schema", + "Configuring Langflow", + "Ingesting sample data", +]; + +const TOTAL_PROVIDER_STEPS = STEP_LIST.length; const OnboardingCard = ({ onComplete }: OnboardingCardProps) => { const updatedOnboarding = process.env.UPDATED_ONBOARDING === "true"; @@ -250,9 +258,10 @@ const OnboardingCard = ({ onComplete }: OnboardingCardProps) => { transition={{ duration: 0.4, ease: "easeInOut" }} > + currentStep={currentStep} + setCurrentStep={setCurrentStep} + steps={STEP_LIST} + /> )}