gmakstutis/openrag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Merge branch 'main' into 175-design-sweep-polish-update-the-no-auth-cloud-connector-warning-state-to-match-designs

Browse source
This commit is contained in:
boneill-ds 2025-10-06 12:22:11 -06:00 • committed by GitHub
commit 17efcd416b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
47 changed files with 8253 additions and 2165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
.env.example
View file

@ -1,60 +1,43 @@
# Ingestion Configuration # Ingestion Configuration
# Set to true to disable Langflow ingestion and use the traditional OpenRAG processor. # Set to true to disable Langflow ingestion and use traditional OpenRAG processor
# If unset or false, the Langflow pipeline is used (default: upload -> ingest -> delete). # If unset or false, Langflow pipeline will be used (default: upload -> ingest -> delete)
DISABLE_INGEST_WITH_LANGFLOW=false DISABLE_INGEST_WITH_LANGFLOW=false
# make one like so https://docs.langflow.org/api-keys-and-authentication#langflow-secret-key
# Create a Langflow secret key:
# https://docs.langflow.org/api-keys-and-authentication#langflow-secret-key
LANGFLOW_SECRET_KEY= LANGFLOW_SECRET_KEY=
# flow ids for chat and ingestion flows
# Flow IDs for chat and ingestion
LANGFLOW_CHAT_FLOW_ID=1098eea1-6649-4e1d-aed1-b77249fb8dd0 LANGFLOW_CHAT_FLOW_ID=1098eea1-6649-4e1d-aed1-b77249fb8dd0
LANGFLOW_INGEST_FLOW_ID=5488df7c-b93f-4f87-a446-b67028bc0813 LANGFLOW_INGEST_FLOW_ID=5488df7c-b93f-4f87-a446-b67028bc0813
# Ingest flow using Docling LANGFLOW_URL_INGEST_FLOW_ID=72c3d17c-2dac-4a73-b48a-6518473d7830
# Ingest flow using docling
# LANGFLOW_INGEST_FLOW_ID=1402618b-e6d1-4ff2-9a11-d6ce71186915 # LANGFLOW_INGEST_FLOW_ID=1402618b-e6d1-4ff2-9a11-d6ce71186915
NUDGES_FLOW_ID=ebc01d31-1976-46ce-a385-b0240327226c NUDGES_FLOW_ID=ebc01d31-1976-46ce-a385-b0240327226c
# Set a strong admin password for OpenSearch; a bcrypt hash is generated at
# OpenSearch Auth # container startup from this value. Do not commit real secrets.
# Set a strong admin password for OpenSearch. # must match the hashed password in secureconfig, must change for secure deployment!!!
# A bcrypt hash is generated at container startup from this value.
# Do not commit real secrets.
# Must be changed for secure deployments.
OPENSEARCH_PASSWORD= OPENSEARCH_PASSWORD=
# make here https://console.cloud.google.com/apis/credentials
# Google OAuth
# Create credentials here:
# https://console.cloud.google.com/apis/credentials
GOOGLE_OAUTH_CLIENT_ID= GOOGLE_OAUTH_CLIENT_ID=
GOOGLE_OAUTH_CLIENT_SECRET= GOOGLE_OAUTH_CLIENT_SECRET=
# Azure app registration credentials for SharePoint/OneDrive
# Microsoft (SharePoint/OneDrive) OAuth
# Azure app registration credentials.
MICROSOFT_GRAPH_OAUTH_CLIENT_ID= MICROSOFT_GRAPH_OAUTH_CLIENT_ID=
MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET= MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET=
# OPTIONAL: dns routable from google (etc.) to handle continous ingest (something like ngrok works). This enables continous ingestion
# Webhooks (optional)
# Public, DNS-resolvable base URL (e.g., via ngrok) for continuous ingestion.
WEBHOOK_BASE_URL= WEBHOOK_BASE_URL=
# API Keys
OPENAI_API_KEY= OPENAI_API_KEY=
AWS_ACCESS_KEY_ID= AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY= AWS_SECRET_ACCESS_KEY=
# OPTIONAL url for openrag link to langflow in the UI
# Langflow UI URL (optional)
# Public URL to link OpenRAG to Langflow in the UI.
LANGFLOW_PUBLIC_URL= LANGFLOW_PUBLIC_URL=
# Langflow auth
# Langflow Auth
LANGFLOW_AUTO_LOGIN=False LANGFLOW_AUTO_LOGIN=False
LANGFLOW_SUPERUSER= LANGFLOW_SUPERUSER=
LANGFLOW_SUPERUSER_PASSWORD= LANGFLOW_SUPERUSER_PASSWORD=

1
.gitignore vendored
View file

@ -17,6 +17,7 @@ wheels/
1001*.pdf 1001*.pdf
*.json *.json
!flows/*.json
.DS_Store .DS_Store
config/ config/

48
Dockerfile.langflow
View file

@ -1,49 +1,5 @@
FROM python:3.12-slim FROM langflowai/langflow-nightly:1.6.3.dev0
# Set environment variables
ENV DEBIAN_FRONTEND=noninteractive
ENV PYTHONUNBUFFERED=1
ENV RUSTFLAGS="--cfg reqwest_unstable"
# Accept build arguments for git repository and branch
ARG GIT_REPO=https://github.com/langflow-ai/langflow.git
ARG GIT_BRANCH=test-openai-responses
WORKDIR /app
# Install system dependencies
RUN apt-get update && apt-get install -y \
build-essential \
curl \
git \
ca-certificates \
gnupg \
npm \
rustc cargo pkg-config libssl-dev \
&& rm -rf /var/lib/apt/lists/*
# Install uv for faster Python package management
RUN pip install uv
# Clone the repository and checkout the specified branch
RUN git clone --depth 1 --branch ${GIT_BRANCH} ${GIT_REPO} /app
# Install backend dependencies
RUN uv sync --frozen --no-install-project --no-editable --extra postgresql
# Build frontend
WORKDIR /app/src/frontend
RUN NODE_OPTIONS=--max_old_space_size=4096 npm ci && \
NODE_OPTIONS=--max_old_space_size=4096 npm run build && \
mkdir -p /app/src/backend/base/langflow/frontend && \
cp -r build/* /app/src/backend/base/langflow/frontend/
# Return to app directory and install the project
WORKDIR /app
RUN uv sync --frozen --no-dev --no-editable --extra postgresql
# Expose ports
EXPOSE 7860 EXPOSE 7860
# Start the backend server CMD ["langflow", "run", "--host", "0.0.0.0", "--port", "7860"]
CMD ["uv", "run", "langflow", "run", "--host", "0.0.0.0", "--port", "7860"]

32
docker-compose-cpu.yml
View file

@ -40,10 +40,10 @@ services:
openrag-backend: openrag-backend:
image: phact/openrag-backend:${OPENRAG_VERSION:-latest} image: phact/openrag-backend:${OPENRAG_VERSION:-latest}
#build: # build:
#context: . # context: .
#dockerfile: Dockerfile.backend # dockerfile: Dockerfile.backend
container_name: openrag-backend # container_name: openrag-backend
depends_on: depends_on:
- langflow - langflow
environment: environment:
@ -55,6 +55,7 @@ services:
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
- LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID}
- LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID}
- LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID}
- DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false} - DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false}
- NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID}
- OPENSEARCH_PORT=9200 - OPENSEARCH_PORT=9200
@ -77,9 +78,9 @@ services:
openrag-frontend: openrag-frontend:
image: phact/openrag-frontend:${OPENRAG_VERSION:-latest} image: phact/openrag-frontend:${OPENRAG_VERSION:-latest}
#build: # build:
#context: . # context: .
#dockerfile: Dockerfile.frontend # dockerfile: Dockerfile.frontend
container_name: openrag-frontend container_name: openrag-frontend
depends_on: depends_on:
- openrag-backend - openrag-backend
@ -92,6 +93,9 @@ services:
volumes: volumes:
- ./flows:/app/flows:Z - ./flows:/app/flows:Z
image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest} image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest}
# build:
# context: .
# dockerfile: Dockerfile.langflow
container_name: langflow container_name: langflow
ports: ports:
- "7860:7860" - "7860:7860"
@ -99,15 +103,23 @@ services:
- OPENAI_API_KEY=${OPENAI_API_KEY} - OPENAI_API_KEY=${OPENAI_API_KEY}
- LANGFLOW_LOAD_FLOWS_PATH=/app/flows - LANGFLOW_LOAD_FLOWS_PATH=/app/flows
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY} - LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
- JWT="dummy" - JWT=None
- OWNER=None
- OWNER_NAME=None
- OWNER_EMAIL=None
- CONNECTOR_TYPE=system
- OPENRAG-QUERY-FILTER="{}" - OPENRAG-QUERY-FILTER="{}"
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD - FILENAME=None
- MIMETYPE=None
- FILESIZE=0
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE
- LANGFLOW_LOG_LEVEL=DEBUG - LANGFLOW_LOG_LEVEL=DEBUG
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
- LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE} - LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE}
- LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI} - LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI}
- DEFAULT_FOLDER_NAME="OpenRAG" # - DEFAULT_FOLDER_NAME=OpenRAG
- HIDE_GETTING_STARTED_PROGRESS=true - HIDE_GETTING_STARTED_PROGRESS=true

12
docker-compose.yml
View file

@ -43,7 +43,7 @@ services:
# build: # build:
# context: . # context: .
# dockerfile: Dockerfile.backend # dockerfile: Dockerfile.backend
container_name: openrag-backend # container_name: openrag-backend
depends_on: depends_on:
- langflow - langflow
environment: environment:
@ -54,6 +54,7 @@ services:
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
- LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID}
- LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID}
- LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID}
- DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false} - DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false}
- NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID}
- OPENSEARCH_PORT=9200 - OPENSEARCH_PORT=9200
@ -80,7 +81,7 @@ services:
# build: # build:
# context: . # context: .
# dockerfile: Dockerfile.frontend # dockerfile: Dockerfile.frontend
# #dockerfile: Dockerfile.frontend #dockerfile: Dockerfile.frontend
container_name: openrag-frontend container_name: openrag-frontend
depends_on: depends_on:
- openrag-backend - openrag-backend
@ -109,13 +110,16 @@ services:
- OWNER_EMAIL=None - OWNER_EMAIL=None
- CONNECTOR_TYPE=system - CONNECTOR_TYPE=system
- OPENRAG-QUERY-FILTER="{}" - OPENRAG-QUERY-FILTER="{}"
- FILENAME=None
- MIMETYPE=None
- FILESIZE=0
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE
- LANGFLOW_LOG_LEVEL=DEBUG - LANGFLOW_LOG_LEVEL=DEBUG
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
- LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE} - LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE}
- LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI} - LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI}
- DEFAULT_FOLDER_NAME="OpenRAG" # - DEFAULT_FOLDER_NAME=OpenRAG
- HIDE_GETTING_STARTED_PROGRESS=true - HIDE_GETTING_STARTED_PROGRESS=true

2579
flows/ingestion_flow.json
View file

File diff suppressed because one or more lines are too long

21
flows/openrag_agent.json
View file

@ -144,6 +144,8 @@
"targetHandle": "{œfieldNameœ:œagent_llmœ,œidœ:œAgent-crjWfœ,œinputTypesœ:[œLanguageModelœ],œtypeœ:œstrœ}" "targetHandle": "{œfieldNameœ:œagent_llmœ,œidœ:œAgent-crjWfœ,œinputTypesœ:[œLanguageModelœ],œtypeœ:œstrœ}"
}, },
{ {
"animated": false,
"className": "",
"data": { "data": {
"sourceHandle": { "sourceHandle": {
"dataType": "TextInput", "dataType": "TextInput",
@ -163,6 +165,7 @@
} }
}, },
"id": "xy-edge__TextInput-aHsQb{œdataTypeœ:œTextInputœ,œidœ:œTextInput-aHsQbœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearch-iYfjf{œfieldNameœ:œfilter_expressionœ,œidœ:œOpenSearch-iYfjfœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}", "id": "xy-edge__TextInput-aHsQb{œdataTypeœ:œTextInputœ,œidœ:œTextInput-aHsQbœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearch-iYfjf{œfieldNameœ:œfilter_expressionœ,œidœ:œOpenSearch-iYfjfœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}",
"selected": false,
"source": "TextInput-aHsQb", "source": "TextInput-aHsQb",
"sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-aHsQbœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-aHsQbœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",
"target": "OpenSearch-iYfjf", "target": "OpenSearch-iYfjf",
@ -727,7 +730,7 @@
], ],
"frozen": false, "frozen": false,
"icon": "OpenSearch", "icon": "OpenSearch",
"last_updated": "2025-10-02T20:05:34.814Z", "last_updated": "2025-10-04T05:41:33.344Z",
"legacy": false, "legacy": false,
"lf_version": "1.6.0", "lf_version": "1.6.0",
"metadata": { "metadata": {
@ -1381,7 +1384,7 @@
], ],
"frozen": false, "frozen": false,
"icon": "binary", "icon": "binary",
"last_updated": "2025-10-02T20:05:34.815Z", "last_updated": "2025-10-04T05:41:33.345Z",
"legacy": false, "legacy": false,
"lf_version": "1.6.0", "lf_version": "1.6.0",
"metadata": { "metadata": {
@ -1660,7 +1663,7 @@
}, },
"position": { "position": {
"x": 727.4791597769406, "x": 727.4791597769406,
"y": 518.0820551650631 "y": 416.82609966052854
}, },
"selected": false, "selected": false,
"type": "genericNode" "type": "genericNode"
@ -1706,7 +1709,7 @@
], ],
"frozen": false, "frozen": false,
"icon": "bot", "icon": "bot",
"last_updated": "2025-10-02T20:05:34.872Z", "last_updated": "2025-10-04T05:41:33.399Z",
"legacy": false, "legacy": false,
"lf_version": "1.6.0", "lf_version": "1.6.0",
"metadata": { "metadata": {
@ -2245,7 +2248,7 @@
], ],
"frozen": false, "frozen": false,
"icon": "brain-circuit", "icon": "brain-circuit",
"last_updated": "2025-10-02T20:05:34.815Z", "last_updated": "2025-10-04T05:41:33.347Z",
"legacy": false, "legacy": false,
"lf_version": "1.6.0", "lf_version": "1.6.0",
"metadata": { "metadata": {
@ -2551,17 +2554,17 @@
} }
], ],
"viewport": { "viewport": {
"x": -237.0727605845459, "x": -149.48015964664273,
"y": 154.6885920024542, "y": 154.6885920024542,
"zoom": 0.602433700773958 "zoom": 0.602433700773958
} }
}, },
"description": "OpenRAG Open Search Agent", "description": "OpenRAG OpenSearch Agent",
"endpoint_name": null, "endpoint_name": null,
"id": "1098eea1-6649-4e1d-aed1-b77249fb8dd0", "id": "1098eea1-6649-4e1d-aed1-b77249fb8dd0",
"is_component": false, "is_component": false,
"last_tested_version": "1.6.0", "last_tested_version": "1.6.3.dev0",
"name": "OpenRAG Open Search Agent", "name": "OpenRAG OpenSearch Agent",
"tags": [ "tags": [
"assistants", "assistants",
"agents" "agents"

4
flows/openrag_nudges.json
View file

@ -2337,12 +2337,12 @@
"zoom": 0.5380793988167256 "zoom": 0.5380793988167256
} }
}, },
"description": "OpenRAG Open Search Nudges generator, based on the Open Search documents and the chat history.", "description": "OpenRAG OpenSearch Nudges generator, based on the OpenSearch documents and the chat history.",
"endpoint_name": null, "endpoint_name": null,
"id": "ebc01d31-1976-46ce-a385-b0240327226c", "id": "ebc01d31-1976-46ce-a385-b0240327226c",
"is_component": false, "is_component": false,
"last_tested_version": "1.6.0", "last_tested_version": "1.6.0",
"name": "OpenRAG Open Search Nudges", "name": "OpenRAG OpenSearch Nudges",
"tags": [ "tags": [
"assistants", "assistants",
"agents" "agents"

3616
flows/openrag_url_mcp.json Normal file
View file

File diff suppressed because one or more lines are too long

134
frontend/components/docling-health-banner.tsx Normal file
View file

@ -0,0 +1,134 @@
"use client";
import { AlertTriangle, ExternalLink, Copy } from "lucide-react";
import { useDoclingHealthQuery } from "@/src/app/api/queries/useDoclingHealthQuery";
import { Banner, BannerIcon, BannerTitle, BannerAction } from "@/components/ui/banner";
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogHeader,
DialogTitle,
DialogDescription,
DialogFooter
} from "@/components/ui/dialog";
import { cn } from "@/lib/utils";
import { useState } from "react";
interface DoclingHealthBannerProps {
className?: string;
}
// DoclingSetupDialog component
interface DoclingSetupDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
className?: string;
}
function DoclingSetupDialog({
open,
onOpenChange,
className
}: DoclingSetupDialogProps) {
const [copied, setCopied] = useState(false);
const handleCopy = async () => {
await navigator.clipboard.writeText("uv run openrag");
setCopied(true);
setTimeout(() => setCopied(false), 2000);
};
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className={cn("max-w-lg", className)}>
<DialogHeader>
<DialogTitle className="flex items-center gap-2 text-base">
<AlertTriangle className="h-4 w-4 text-amber-600 dark:text-amber-400" />
docling-serve is stopped. Knowledge ingest is unavailable.
</DialogTitle>
<DialogDescription>
Start docling-serve by running:
</DialogDescription>
</DialogHeader>
<div className="space-y-4">
<div className="flex items-center gap-2">
<code className="flex-1 bg-muted px-3 py-2.5 rounded-md text-sm font-mono">
uv run openrag
</code>
<Button
variant="ghost"
size="icon"
onClick={handleCopy}
className="shrink-0"
title={copied ? "Copied!" : "Copy to clipboard"}
>
<Copy className="h-4 w-4" />
</Button>
</div>
<DialogDescription>
Then, select <span className="font-semibold text-foreground">Start Native Services</span> in the TUI. Once docling-serve is running, refresh OpenRAG.
</DialogDescription>
</div>
<DialogFooter>
<Button
variant="default"
onClick={() => onOpenChange(false)}
>
Close
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
);
}
export function DoclingHealthBanner({ className }: DoclingHealthBannerProps) {
const { data: health, isLoading, isError } = useDoclingHealthQuery();
const [showDialog, setShowDialog] = useState(false);
const isHealthy = health?.status === "healthy" && !isError;
const isUnhealthy = health?.status === "unhealthy" || isError;
// Only show banner when service is unhealthy
if (isLoading || isHealthy) {
return null;
}
if (isUnhealthy) {
return (
<>
<Banner
className={cn(
"bg-amber-50 text-amber-900 dark:bg-amber-950 dark:text-amber-200 border-amber-200 dark:border-amber-800",
className
)}
>
<BannerIcon
icon={AlertTriangle}
/>
<BannerTitle className="font-medium">
docling-serve native service is stopped. Knowledge ingest is unavailable.
</BannerTitle>
<BannerAction
onClick={() => setShowDialog(true)}
className="bg-foreground text-background hover:bg-primary/90"
>
Setup Docling Serve
<ExternalLink className="h-3 w-3 ml-1" />
</BannerAction>
</Banner>
<DoclingSetupDialog
open={showDialog}
onOpenChange={setShowDialog}
/>
</>
);
}
return null;
}

66
frontend/components/duplicate-handling-dialog.tsx Normal file
View file

@ -0,0 +1,66 @@
"use client";
import { RotateCcw } from "lucide-react";
import type React from "react";
import { Button } from "./ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "./ui/dialog";
interface DuplicateHandlingDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
onOverwrite: () => void | Promise<void>;
isLoading?: boolean;
}
export const DuplicateHandlingDialog: React.FC<
DuplicateHandlingDialogProps
> = ({ open, onOpenChange, onOverwrite, isLoading = false }) => {
const handleOverwrite = async () => {
await onOverwrite();
onOpenChange(false);
};
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-[450px]">
<DialogHeader>
<DialogTitle>Overwrite document</DialogTitle>
<DialogDescription className="pt-2 text-muted-foreground">
Overwriting will replace the existing document with another version.
This can't be undone.
</DialogDescription>
</DialogHeader>
<DialogFooter className="flex-row gap-2 justify-end">
<Button
type="button"
variant="ghost"
onClick={() => onOpenChange(false)}
disabled={isLoading}
size="sm"
>
Cancel
</Button>
<Button
type="button"
variant="default"
size="sm"
onClick={handleOverwrite}
disabled={isLoading}
className="flex items-center gap-2 !bg-accent-amber-foreground hover:!bg-foreground text-primary-foreground"
>
<RotateCcw className="h-3.5 w-3.5" />
Overwrite
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
);
};

1189
frontend/components/knowledge-dropdown.tsx
View file

File diff suppressed because it is too large Load diff

2
frontend/components/logo/ibm-logo.tsx
View file

@ -9,7 +9,7 @@ export default function IBMLogo(props: React.SVGProps<SVGSVGElement>) {
{...props} {...props}
> >
<title>IBM watsonx.ai Logo</title> <title>IBM watsonx.ai Logo</title>
<g clip-path="url(#clip0_2620_2081)"> <g clipPath="url(#clip0_2620_2081)">
<path <path
d="M13 12.0007C12.4477 12.0007 12 12.4484 12 13.0007C12 13.0389 12.0071 13.0751 12.0112 13.1122C10.8708 14.0103 9.47165 14.5007 8 14.5007C5.86915 14.5007 4 12.5146 4 10.2507C4 7.90722 5.9065 6.00072 8.25 6.00072H8.5V5.00072H8.25C5.3552 5.00072 3 7.35592 3 10.2507C3 11.1927 3.2652 12.0955 3.71855 12.879C2.3619 11.6868 1.5 9.94447 1.5 8.00072C1.5 6.94312 1.74585 5.93432 2.23095 5.00292L1.34375 4.54102C0.79175 5.60157 0.5 6.79787 0.5 8.00072C0.5 12.1362 3.8645 15.5007 8 15.5007C9.6872 15.5007 11.2909 14.9411 12.6024 13.9176C12.7244 13.9706 12.8586 14.0007 13 14.0007C13.5523 14.0007 14 13.553 14 13.0007C14 12.4484 13.5523 12.0007 13 12.0007Z" d="M13 12.0007C12.4477 12.0007 12 12.4484 12 13.0007C12 13.0389 12.0071 13.0751 12.0112 13.1122C10.8708 14.0103 9.47165 14.5007 8 14.5007C5.86915 14.5007 4 12.5146 4 10.2507C4 7.90722 5.9065 6.00072 8.25 6.00072H8.5V5.00072H8.25C5.3552 5.00072 3 7.35592 3 10.2507C3 11.1927 3.2652 12.0955 3.71855 12.879C2.3619 11.6868 1.5 9.94447 1.5 8.00072C1.5 6.94312 1.74585 5.93432 2.23095 5.00292L1.34375 4.54102C0.79175 5.60157 0.5 6.79787 0.5 8.00072C0.5 12.1362 3.8645 15.5007 8 15.5007C9.6872 15.5007 11.2909 14.9411 12.6024 13.9176C12.7244 13.9706 12.8586 14.0007 13 14.0007C13.5523 14.0007 14 13.553 14 13.0007C14 12.4484 13.5523 12.0007 13 12.0007Z"
fill="currentColor" fill="currentColor"

141
frontend/components/ui/banner.tsx Normal file
View file

@ -0,0 +1,141 @@
'use client';
import { useControllableState } from '@radix-ui/react-use-controllable-state';
import { type LucideIcon, XIcon } from 'lucide-react';
import {
type ComponentProps,
createContext,
type HTMLAttributes,
type MouseEventHandler,
useContext,
} from 'react';
import { Button } from '@/components/ui/button';
import { cn } from '@/lib/utils';
type BannerContextProps = {
show: boolean;
setShow: (show: boolean) => void;
};
export const BannerContext = createContext<BannerContextProps>({
show: true,
setShow: () => {},
});
export type BannerProps = HTMLAttributes<HTMLDivElement> & {
visible?: boolean;
defaultVisible?: boolean;
onClose?: () => void;
inset?: boolean;
};
export const Banner = ({
children,
visible,
defaultVisible = true,
onClose,
className,
inset = false,
...props
}: BannerProps) => {
const [show, setShow] = useControllableState({
defaultProp: defaultVisible,
prop: visible,
onChange: onClose,
});
if (!show) {
return null;
}
return (
<BannerContext.Provider value={{ show, setShow }}>
<div
className={cn(
'flex w-full items-center justify-between gap-2 bg-primary px-4 py-2 text-primary-foreground',
inset && 'rounded-lg',
className
)}
{...props}
>
{children}
</div>
</BannerContext.Provider>
);
};
export type BannerIconProps = HTMLAttributes<HTMLDivElement> & {
icon: LucideIcon;
};
export const BannerIcon = ({
icon: Icon,
className,
...props
}: BannerIconProps) => (
<div
className={cn(
'p-1',
className
)}
{...props}
>
<Icon size={16} />
</div>
);
export type BannerTitleProps = HTMLAttributes<HTMLParagraphElement>;
export const BannerTitle = ({ className, ...props }: BannerTitleProps) => (
<p className={cn('flex-1 text-sm', className)} {...props} />
);
export type BannerActionProps = ComponentProps<typeof Button>;
export const BannerAction = ({
variant = 'outline',
size = 'sm',
className,
...props
}: BannerActionProps) => (
<Button
className={cn(
'shrink-0 bg-transparent hover:bg-background/10 hover:text-background',
className
)}
size={size}
variant={variant}
{...props}
/>
);
export type BannerCloseProps = ComponentProps<typeof Button>;
export const BannerClose = ({
variant = 'ghost',
size = 'icon',
onClick,
className,
...props
}: BannerCloseProps) => {
const { setShow } = useContext(BannerContext);
const handleClick: MouseEventHandler<HTMLButtonElement> = (e) => {
setShow(false);
onClick?.(e);
};
return (
<Button
className={cn(
'shrink-0 bg-transparent hover:bg-background/10 hover:text-background',
className
)}
onClick={handleClick}
size={size}
variant={variant}
{...props}
>
<XIcon size={18} />
</Button>
);
};

2
frontend/components/ui/input.tsx
View file

@ -44,7 +44,7 @@ const Input = React.forwardRef<HTMLInputElement, InputProps>(
placeholder={placeholder} placeholder={placeholder}
className={cn( className={cn(
"primary-input", "primary-input",
icon && "pl-9", icon && "!pl-9",
type === "password" && "!pr-8", type === "password" && "!pr-8",
icon ? inputClassName : className icon ? inputClassName : className
)} )}

47
frontend/src/app/api/mutations/useCancelTaskMutation.ts Normal file
View file

@ -0,0 +1,47 @@
import {
type UseMutationOptions,
useMutation,
useQueryClient,
} from "@tanstack/react-query";
export interface CancelTaskRequest {
taskId: string;
}
export interface CancelTaskResponse {
status: string;
task_id: string;
}
export const useCancelTaskMutation = (
options?: Omit<
UseMutationOptions<CancelTaskResponse, Error, CancelTaskRequest>,
"mutationFn"
>
) => {
const queryClient = useQueryClient();
async function cancelTask(
variables: CancelTaskRequest,
): Promise<CancelTaskResponse> {
const response = await fetch(`/api/tasks/${variables.taskId}/cancel`, {
method: "POST",
});
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
throw new Error(errorData.error || "Failed to cancel task");
}
return response.json();
}
return useMutation({
mutationFn: cancelTask,
onSuccess: () => {
// Invalidate tasks query to refresh the list
queryClient.invalidateQueries({ queryKey: ["tasks"] });
},
...options,
});
};

55
frontend/src/app/api/queries/useDoclingHealthQuery.ts Normal file
View file

@ -0,0 +1,55 @@
import {
type UseQueryOptions,
useQuery,
useQueryClient,
} from "@tanstack/react-query";
export interface DoclingHealthResponse {
status: "healthy" | "unhealthy";
message?: string;
}
export const useDoclingHealthQuery = (
options?: Omit<UseQueryOptions<DoclingHealthResponse>, "queryKey" | "queryFn">,
) => {
const queryClient = useQueryClient();
async function checkDoclingHealth(): Promise<DoclingHealthResponse> {
try {
const response = await fetch("http://127.0.0.1:5001/health", {
method: "GET",
headers: {
"Content-Type": "application/json",
},
});
if (response.ok) {
return { status: "healthy" };
} else {
return {
status: "unhealthy",
message: `Health check failed with status: ${response.status}`,
};
}
} catch (error) {
return {
status: "unhealthy",
message: error instanceof Error ? error.message : "Connection failed",
};
}
}
const queryResult = useQuery(
{
queryKey: ["docling-health"],
queryFn: checkDoclingHealth,
retry: 1,
refetchInterval: 30000, // Check every 30 seconds
staleTime: 25000, // Consider data stale after 25 seconds
...options,
},
queryClient,
);
return queryResult;
};

5
frontend/src/app/api/queries/useGetSearchQuery.ts
View file

@ -29,6 +29,7 @@ export interface ChunkResult {
owner_email?: string; owner_email?: string;
file_size?: number; file_size?: number;
connector_type?: string; connector_type?: string;
index?: number;
} }
export interface File { export interface File {
@ -55,7 +56,7 @@ export interface File {
export const useGetSearchQuery = ( export const useGetSearchQuery = (
query: string, query: string,
queryData?: ParsedQueryData | null, queryData?: ParsedQueryData | null,
options?: Omit<UseQueryOptions, "queryKey" | "queryFn">, options?: Omit<UseQueryOptions, "queryKey" | "queryFn">
) => { ) => {
const queryClient = useQueryClient(); const queryClient = useQueryClient();
@ -184,7 +185,7 @@ export const useGetSearchQuery = (
queryFn: getFiles, queryFn: getFiles,
...options, ...options,
}, },
queryClient, queryClient
); );
return queryResult; return queryResult;

79
frontend/src/app/api/queries/useGetTasksQuery.ts Normal file
View file

@ -0,0 +1,79 @@
import {
type UseQueryOptions,
useQuery,
useQueryClient,
} from "@tanstack/react-query";
export interface Task {
task_id: string;
status:
| "pending"
| "running"
| "processing"
| "completed"
| "failed"
| "error";
total_files?: number;
processed_files?: number;
successful_files?: number;
failed_files?: number;
running_files?: number;
pending_files?: number;
created_at: string;
updated_at: string;
duration_seconds?: number;
result?: Record<string, unknown>;
error?: string;
files?: Record<string, Record<string, unknown>>;
}
export interface TasksResponse {
tasks: Task[];
}
export const useGetTasksQuery = (
options?: Omit<UseQueryOptions<Task[]>, "queryKey" | "queryFn">
) => {
const queryClient = useQueryClient();
async function getTasks(): Promise<Task[]> {
const response = await fetch("/api/tasks");
if (!response.ok) {
throw new Error("Failed to fetch tasks");
}
const data: TasksResponse = await response.json();
return data.tasks || [];
}
const queryResult = useQuery(
{
queryKey: ["tasks"],
queryFn: getTasks,
refetchInterval: (query) => {
// Only poll if there are tasks with pending or running status
const data = query.state.data;
if (!data || data.length === 0) {
return false; // Stop polling if no tasks
}
const hasActiveTasks = data.some(
(task: Task) =>
task.status === "pending" ||
task.status === "running" ||
task.status === "processing"
);
return hasActiveTasks ? 3000 : false; // Poll every 3 seconds if active tasks exist
},
refetchIntervalInBackground: true,
staleTime: 0, // Always consider data stale to ensure fresh updates
gcTime: 5 * 60 * 1000, // Keep in cache for 5 minutes
...options,
},
queryClient,
);
return queryResult;
};

5
frontend/src/app/chat/page.tsx
View file

@ -31,6 +31,7 @@ import {
import { useAuth } from "@/contexts/auth-context"; import { useAuth } from "@/contexts/auth-context";
import { type EndpointType, useChat } from "@/contexts/chat-context"; import { type EndpointType, useChat } from "@/contexts/chat-context";
import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";
import { useLayout } from "@/contexts/layout-context";
import { useTask } from "@/contexts/task-context"; import { useTask } from "@/contexts/task-context";
import { useLoadingStore } from "@/stores/loadingStore"; import { useLoadingStore } from "@/stores/loadingStore";
import { useGetNudgesQuery } from "../api/queries/useGetNudgesQuery"; import { useGetNudgesQuery } from "../api/queries/useGetNudgesQuery";
@ -151,6 +152,7 @@ function ChatPage() {
const streamIdRef = useRef(0); const streamIdRef = useRef(0);
const lastLoadedConversationRef = useRef<string | null>(null); const lastLoadedConversationRef = useRef<string | null>(null);
const { addTask, isMenuOpen } = useTask(); const { addTask, isMenuOpen } = useTask();
const { totalTopOffset } = useLayout();
const { selectedFilter, parsedFilterData, isPanelOpen, setSelectedFilter } = const { selectedFilter, parsedFilterData, isPanelOpen, setSelectedFilter } =
useKnowledgeFilter(); useKnowledgeFilter();
@ -2046,7 +2048,7 @@ function ChatPage() {
return ( return (
<div <div
className={`fixed inset-0 md:left-72 top-[53px] flex flex-col transition-all duration-300 ${ className={`fixed inset-0 md:left-72 flex flex-col transition-all duration-300 ${
isMenuOpen && isPanelOpen isMenuOpen && isPanelOpen
? "md:right-[704px]" // Both open: 384px (menu) + 320px (KF panel) ? "md:right-[704px]" // Both open: 384px (menu) + 320px (KF panel)
: isMenuOpen : isMenuOpen
@ -2055,6 +2057,7 @@ function ChatPage() {
? "md:right-80" // Only KF panel open: 320px ? "md:right-80" // Only KF panel open: 320px
: "md:right-6" // Neither open: 24px : "md:right-6" // Neither open: 24px
}`} }`}
style={{ top: `${totalTopOffset}px` }}
> >
{/* Debug header - only show in debug mode */} {/* Debug header - only show in debug mode */}
{isDebugMode && ( {isDebugMode && (

607
frontend/src/app/knowledge/chunks/page.tsx
View file

@ -1,176 +1,204 @@
"use client"; "use client";
import { ArrowLeft, Check, Copy, Loader2, Search } from "lucide-react"; import { ArrowLeft, Check, Copy, Loader2, Search, X } from "lucide-react";
import { Suspense, useCallback, useEffect, useMemo, useState } from "react";
import { useRouter, useSearchParams } from "next/navigation"; import { useRouter, useSearchParams } from "next/navigation";
import { Suspense, useCallback, useEffect, useMemo, useState } from "react";
// import { Label } from "@/components/ui/label";
// import { Checkbox } from "@/components/ui/checkbox";
import { filterAccentClasses } from "@/components/knowledge-filter-panel";
import { ProtectedRoute } from "@/components/protected-route"; import { ProtectedRoute } from "@/components/protected-route";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";
import { useTask } from "@/contexts/task-context";
import {
type ChunkResult,
type File,
useGetSearchQuery,
} from "../../api/queries/useGetSearchQuery";
import { Label } from "@/components/ui/label";
import { Checkbox } from "@/components/ui/checkbox"; import { Checkbox } from "@/components/ui/checkbox";
import { Input } from "@/components/ui/input"; import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";
import { useLayout } from "@/contexts/layout-context";
import { useTask } from "@/contexts/task-context";
import {
type ChunkResult,
type File,
useGetSearchQuery,
} from "../../api/queries/useGetSearchQuery";
const getFileTypeLabel = (mimetype: string) => { const getFileTypeLabel = (mimetype: string) => {
if (mimetype === "application/pdf") return "PDF"; if (mimetype === "application/pdf") return "PDF";
if (mimetype === "text/plain") return "Text"; if (mimetype === "text/plain") return "Text";
if (mimetype === "application/msword") return "Word Document"; if (mimetype === "application/msword") return "Word Document";
return "Unknown"; return "Unknown";
}; };
function ChunksPageContent() { function ChunksPageContent() {
const router = useRouter(); const router = useRouter();
const searchParams = useSearchParams(); const searchParams = useSearchParams();
const { isMenuOpen } = useTask(); const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } =
const { parsedFilterData, isPanelOpen } = useKnowledgeFilter(); useKnowledgeFilter();
const { isMenuOpen } = useTask();
const { totalTopOffset } = useLayout();
const filename = searchParams.get("filename"); const filename = searchParams.get("filename");
const [chunks, setChunks] = useState<ChunkResult[]>([]); const [chunks, setChunks] = useState<ChunkResult[]>([]);
const [chunksFilteredByQuery, setChunksFilteredByQuery] = useState< const [chunksFilteredByQuery, setChunksFilteredByQuery] = useState<
ChunkResult[] ChunkResult[]
>([]); >([]);
const [selectedChunks, setSelectedChunks] = useState<Set<number>>(new Set()); const [selectedChunks, setSelectedChunks] = useState<Set<number>>(new Set());
const [activeCopiedChunkIndex, setActiveCopiedChunkIndex] = useState< const [activeCopiedChunkIndex, setActiveCopiedChunkIndex] = useState<
number | null number | null
>(null); >(null);
// Calculate average chunk length // Calculate average chunk length
const averageChunkLength = useMemo( const averageChunkLength = useMemo(
() => () =>
chunks.reduce((acc, chunk) => acc + chunk.text.length, 0) / chunks.reduce((acc, chunk) => acc + chunk.text.length, 0) /
chunks.length || 0, chunks.length || 0,
[chunks] [chunks],
); );
const [selectAll, setSelectAll] = useState(false); const [selectAll, setSelectAll] = useState(false);
const [queryInputText, setQueryInputText] = useState( const [queryInputText, setQueryInputText] = useState(
parsedFilterData?.query ?? "" parsedFilterData?.query ?? "",
); );
// Use the same search query as the knowledge page, but we'll filter for the specific file // Use the same search query as the knowledge page, but we'll filter for the specific file
const { data = [], isFetching } = useGetSearchQuery("*", parsedFilterData); const { data = [], isFetching } = useGetSearchQuery("*", parsedFilterData);
useEffect(() => { useEffect(() => {
if (queryInputText === "") { if (queryInputText === "") {
setChunksFilteredByQuery(chunks); setChunksFilteredByQuery(chunks);
} else { } else {
setChunksFilteredByQuery( setChunksFilteredByQuery(
chunks.filter((chunk) => chunks.filter((chunk) =>
chunk.text.toLowerCase().includes(queryInputText.toLowerCase()) chunk.text.toLowerCase().includes(queryInputText.toLowerCase()),
) ),
); );
} }
}, [queryInputText, chunks]); }, [queryInputText, chunks]);
const handleCopy = useCallback((text: string, index: number) => { const handleCopy = useCallback((text: string, index: number) => {
// Trim whitespace and remove new lines/tabs for cleaner copy // Trim whitespace and remove new lines/tabs for cleaner copy
navigator.clipboard.writeText(text.trim().replace(/[\n\r\t]/gm, "")); navigator.clipboard.writeText(text.trim().replace(/[\n\r\t]/gm, ""));
setActiveCopiedChunkIndex(index); setActiveCopiedChunkIndex(index);
setTimeout(() => setActiveCopiedChunkIndex(null), 10 * 1000); // 10 seconds setTimeout(() => setActiveCopiedChunkIndex(null), 10 * 1000); // 10 seconds
}, []); }, []);
const fileData = (data as File[]).find( const fileData = (data as File[]).find(
(file: File) => file.filename === filename (file: File) => file.filename === filename,
); );
// Extract chunks for the specific file // Extract chunks for the specific file
useEffect(() => { useEffect(() => {
if (!filename || !(data as File[]).length) { if (!filename || !(data as File[]).length) {
setChunks([]); setChunks([]);
return; return;
} }
setChunks(fileData?.chunks || []); setChunks(
}, [data, filename]); fileData?.chunks?.map((chunk, i) => ({ ...chunk, index: i + 1 })) || [],
);
}, [data, filename]);
// Set selected state for all checkboxes when selectAll changes // Set selected state for all checkboxes when selectAll changes
useEffect(() => { useEffect(() => {
if (selectAll) { if (selectAll) {
setSelectedChunks(new Set(chunks.map((_, index) => index))); setSelectedChunks(new Set(chunks.map((_, index) => index)));
} else { } else {
setSelectedChunks(new Set()); setSelectedChunks(new Set());
} }
}, [selectAll, setSelectedChunks, chunks]); }, [selectAll, setSelectedChunks, chunks]);
const handleBack = useCallback(() => { const handleBack = useCallback(() => {
router.push("/knowledge"); router.push("/knowledge");
}, [router]); }, [router]);
const handleChunkCardCheckboxChange = useCallback( // const handleChunkCardCheckboxChange = useCallback(
(index: number) => { // (index: number) => {
setSelectedChunks((prevSelected) => { // setSelectedChunks((prevSelected) => {
const newSelected = new Set(prevSelected); // const newSelected = new Set(prevSelected);
if (newSelected.has(index)) { // if (newSelected.has(index)) {
newSelected.delete(index); // newSelected.delete(index);
} else { // } else {
newSelected.add(index); // newSelected.add(index);
} // }
return newSelected; // return newSelected;
}); // });
}, // },
[setSelectedChunks] // [setSelectedChunks]
); // );
if (!filename) { if (!filename) {
return ( return (
<div className="flex items-center justify-center h-64"> <div className="flex items-center justify-center h-64">
<div className="text-center"> <div className="text-center">
<Search className="h-12 w-12 mx-auto mb-4 text-muted-foreground/50" /> <Search className="h-12 w-12 mx-auto mb-4 text-muted-foreground/50" />
<p className="text-lg text-muted-foreground">No file specified</p> <p className="text-lg text-muted-foreground">No file specified</p>
<p className="text-sm text-muted-foreground/70 mt-2"> <p className="text-sm text-muted-foreground/70 mt-2">
Please select a file from the knowledge page Please select a file from the knowledge page
</p> </p>
</div> </div>
</div> </div>
); );
} }
return ( return (
<div <div
className={`fixed inset-0 md:left-72 top-[53px] flex flex-row transition-all duration-300 ${ className={`fixed inset-0 md:left-72 flex flex-row transition-all duration-300 ${
isMenuOpen && isPanelOpen isMenuOpen && isPanelOpen
? "md:right-[704px]" ? "md:right-[704px]"
: // Both open: 384px (menu) + 320px (KF panel) : // Both open: 384px (menu) + 320px (KF panel)
isMenuOpen isMenuOpen
? "md:right-96" ? "md:right-96"
: // Only menu open: 384px : // Only menu open: 384px
isPanelOpen isPanelOpen
? "md:right-80" ? "md:right-80"
: // Only KF panel open: 320px : // Only KF panel open: 320px
"md:right-6" // Neither open: 24px "md:right-6" // Neither open: 24px
}`} }`}
> style={{ top: `${totalTopOffset}px` }}
<div className="flex-1 flex flex-col min-h-0 px-6 py-6"> >
{/* Header */} <div className="flex-1 flex flex-col min-h-0 px-6 py-6">
<div className="flex flex-col mb-6"> {/* Header */}
<div className="flex flex-row items-center gap-3 mb-6"> <div className="flex flex-col mb-6">
<Button variant="ghost" onClick={handleBack} size="sm"> <div className="flex flex-row items-center gap-3 mb-6">
<ArrowLeft size={24} /> <Button variant="ghost" onClick={handleBack} size="sm">
</Button> <ArrowLeft size={24} />
<h1 className="text-lg font-semibold"> </Button>
{/* Removes file extension from filename */} <h1 className="text-lg font-semibold">
{filename.replace(/\.[^/.]+$/, "")} {/* Removes file extension from filename */}
</h1> {filename.replace(/\.[^/.]+$/, "")}
</div> </h1>
<div className="flex flex-col items-start mt-2"> </div>
<div className="flex-1 flex items-center gap-2 w-full max-w-[616px] mb-8"> <div className="flex flex-col items-start mt-2">
<Input <div className="flex-1 flex items-center gap-2 w-full max-w-[640px]">
name="search-query" <div className="primary-input min-h-10 !flex items-center flex-nowrap focus-within:border-foreground transition-colors !p-[0.3rem]">
icon={!queryInputText.length ? <Search size={18} /> : null} {selectedFilter?.name && (
id="search-query" <div
type="text" className={`flex items-center gap-1 h-full px-1.5 py-0.5 mr-1 rounded max-w-[25%] ${
defaultValue={parsedFilterData?.query} filterAccentClasses[parsedFilterData?.color || "zinc"]
value={queryInputText} }`}
onChange={(e) => setQueryInputText(e.target.value)} >
placeholder="Search chunks..." <span className="truncate">{selectedFilter?.name}</span>
/> <X
</div> aria-label="Remove filter"
<div className="flex items-center pl-4 gap-2"> className="h-4 w-4 flex-shrink-0 cursor-pointer"
onClick={() => setSelectedFilter(null)}
/>
</div>
)}
<Search
className="h-4 w-4 ml-1 flex-shrink-0 text-placeholder-foreground"
strokeWidth={1.5}
/>
<input
className="bg-transparent w-full h-full ml-2 focus:outline-none focus-visible:outline-none font-mono placeholder:font-mono"
name="search-query"
id="search-query"
type="text"
placeholder="Enter your search query..."
onChange={(e) => setQueryInputText(e.target.value)}
value={queryInputText}
/>
</div>
</div>
{/* <div className="flex items-center pl-4 gap-2">
<Checkbox <Checkbox
id="selectAllChunks" id="selectAllChunks"
checked={selectAll} checked={selectAll}
@ -184,106 +212,114 @@ function ChunksPageContent() {
> >
Select all Select all
</Label> </Label>
</div> </div> */}
</div> </div>
</div> </div>
{/* Content Area - matches knowledge page structure */} {/* Content Area - matches knowledge page structure */}
<div className="flex-1 overflow-scroll pr-6"> <div className="flex-1 overflow-scroll pr-6">
{isFetching ? ( {isFetching ? (
<div className="flex items-center justify-center h-64"> <div className="flex items-center justify-center h-64">
<div className="text-center"> <div className="text-center">
<Loader2 className="h-12 w-12 mx-auto mb-4 text-muted-foreground/50 animate-spin" /> <Loader2 className="h-12 w-12 mx-auto mb-4 text-muted-foreground/50 animate-spin" />
<p className="text-lg text-muted-foreground"> <p className="text-lg text-muted-foreground">
Loading chunks... Loading chunks...
</p> </p>
</div> </div>
</div> </div>
) : chunks.length === 0 ? ( ) : chunks.length === 0 ? (
<div className="flex items-center justify-center h-64"> <div className="flex items-center justify-center h-64">
<div className="text-center"> <div className="text-center">
<Search className="h-12 w-12 mx-auto mb-4 text-muted-foreground/50" /> <p className="text-xl font-semibold mb-2">No knowledge</p>
<p className="text-lg text-muted-foreground">No chunks found</p> <p className="text-sm text-secondary-foreground">
<p className="text-sm text-muted-foreground/70 mt-2"> Clear the knowledge filter or return to the knowledge page
This file may not have been indexed yet </p>
</p> </div>
</div> </div>
</div> ) : (
) : ( <div className="space-y-4 pb-6">
<div className="space-y-4 pb-6"> {chunksFilteredByQuery.map((chunk, index) => (
{chunksFilteredByQuery.map((chunk, index) => ( <div
<div key={chunk.filename + index}
key={chunk.filename + index} className="bg-muted rounded-lg p-4 border border-border/50"
className="bg-muted rounded-lg p-4 border border-border/50" >
> <div className="flex items-center justify-between mb-2">
<div className="flex items-center justify-between mb-2"> <div className="flex items-center gap-3">
<div className="flex items-center gap-3"> {/* <div>
<div>
<Checkbox <Checkbox
checked={selectedChunks.has(index)} checked={selectedChunks.has(index)}
onCheckedChange={() => onCheckedChange={() =>
handleChunkCardCheckboxChange(index) handleChunkCardCheckboxChange(index)
} }
/> />
</div> </div> */}
<span className="text-sm font-bold"> <span className="text-sm font-bold">
Chunk {chunk.page} Chunk {chunk.index}
</span> </span>
<span className="bg-background p-1 rounded text-xs text-muted-foreground/70"> <span className="bg-background p-1 rounded text-xs text-muted-foreground/70">
{chunk.text.length} chars {chunk.text.length} chars
</span> </span>
<div className="py-1"> <div className="py-1">
<Button <Button
onClick={() => handleCopy(chunk.text, index)} onClick={() => handleCopy(chunk.text, index)}
variant="ghost" variant="ghost"
size="sm" size="sm"
> >
{activeCopiedChunkIndex === index ? ( {activeCopiedChunkIndex === index ? (
<Check className="text-muted-foreground" /> <Check className="text-muted-foreground" />
) : ( ) : (
<Copy className="text-muted-foreground" /> <Copy className="text-muted-foreground" />
)} )}
</Button> </Button>
</div> </div>
</div> </div>
{/* TODO: Update to use active toggle */} <span className="bg-background p-1 rounded text-xs text-muted-foreground/70">
{/* <span className="px-2 py-1 text-green-500"> {chunk.score.toFixed(2)} score
</span>
{/* TODO: Update to use active toggle */}
{/* <span className="px-2 py-1 text-green-500">
<Switch <Switch
className="ml-2 bg-green-500" className="ml-2 bg-green-500"
checked={true} checked={true}
/> />
Active Active
</span> */} </span> */}
</div> </div>
<blockquote className="text-sm text-muted-foreground leading-relaxed border-l-2 border-input ml-1.5 pl-4"> <blockquote className="text-sm text-muted-foreground leading-relaxed ml-1.5">
{chunk.text} {chunk.text}
</blockquote> </blockquote>
</div> </div>
))} ))}
</div> </div>
)} )}
</div> </div>
</div> </div>
{/* Right panel - Summary (TODO), Technical details, */} {/* Right panel - Summary (TODO), Technical details, */}
<div className="w-[320px] py-20 px-2"> {chunks.length > 0 && (
<div className="mb-8"> <div className="w-[320px] py-20 px-2">
<h2 className="text-xl font-semibold mt-3 mb-4">Technical details</h2> <div className="mb-8">
<dl> <h2 className="text-xl font-semibold mt-3 mb-4">
<div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"> Technical details
<dt className="text-sm/6 text-muted-foreground">Total chunks</dt> </h2>
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"> <dl>
{chunks.length} <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
</dd> <dt className="text-sm/6 text-muted-foreground">
</div> Total chunks
<div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"> </dt>
<dt className="text-sm/6 text-muted-foreground">Avg length</dt> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0">
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"> {chunks.length}
{averageChunkLength.toFixed(0)} chars </dd>
</dd> </div>
</div> <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
{/* TODO: Uncomment after data is available */} <dt className="text-sm/6 text-muted-foreground">Avg length</dt>
{/* <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0">
{averageChunkLength.toFixed(0)} chars
</dd>
</div>
{/* TODO: Uncomment after data is available */}
{/* <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
<dt className="text-sm/6 text-muted-foreground">Process time</dt> <dt className="text-sm/6 text-muted-foreground">Process time</dt>
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0">
</dd> </dd>
@ -293,76 +329,79 @@ function ChunksPageContent() {
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0">
</dd> </dd>
</div> */} </div> */}
</dl> </dl>
</div> </div>
<div className="mb-8"> <div className="mb-8">
<h2 className="text-xl font-semibold mt-2 mb-3">Original document</h2> <h2 className="text-xl font-semibold mt-2 mb-3">
<dl> Original document
<div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"> </h2>
<dl>
{/* <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
<dt className="text-sm/6 text-muted-foreground">Name</dt> <dt className="text-sm/6 text-muted-foreground">Name</dt>
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0">
{fileData?.filename} {fileData?.filename}
</dd> </dd>
</div> </div> */}
<div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"> <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
<dt className="text-sm/6 text-muted-foreground">Type</dt> <dt className="text-sm/6 text-muted-foreground">Type</dt>
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0">
{fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"} {fileData ? getFileTypeLabel(fileData.mimetype) : "Unknown"}
</dd> </dd>
</div> </div>
<div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"> <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
<dt className="text-sm/6 text-muted-foreground">Size</dt> <dt className="text-sm/6 text-muted-foreground">Size</dt>
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0">
{fileData?.size {fileData?.size
? `${Math.round(fileData.size / 1024)} KB` ? `${Math.round(fileData.size / 1024)} KB`
: "Unknown"} : "Unknown"}
</dd> </dd>
</div> </div>
<div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"> {/* <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
<dt className="text-sm/6 text-muted-foreground">Uploaded</dt> <dt className="text-sm/6 text-muted-foreground">Uploaded</dt>
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0">
N/A N/A
</dd> </dd>
</div> </div> */}
{/* TODO: Uncomment after data is available */} {/* TODO: Uncomment after data is available */}
{/* <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"> {/* <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
<dt className="text-sm/6 text-muted-foreground">Source</dt> <dt className="text-sm/6 text-muted-foreground">Source</dt>
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"></dd> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"></dd>
</div> */} </div> */}
<div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"> {/* <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
<dt className="text-sm/6 text-muted-foreground">Updated</dt> <dt className="text-sm/6 text-muted-foreground">Updated</dt>
<dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0"> <dd className="mt-1 text-sm/6 text-gray-100 sm:col-span-2 sm:mt-0">
N/A N/A
</dd> </dd>
</div> </div> */}
</dl> </dl>
</div> </div>
</div> </div>
</div> )}
); </div>
);
} }
function ChunksPage() { function ChunksPage() {
return ( return (
<Suspense <Suspense
fallback={ fallback={
<div className="flex items-center justify-center h-64"> <div className="flex items-center justify-center h-64">
<div className="text-center"> <div className="text-center">
<Loader2 className="h-12 w-12 mx-auto mb-4 text-muted-foreground/50 animate-spin" /> <Loader2 className="h-12 w-12 mx-auto mb-4 text-muted-foreground/50 animate-spin" />
<p className="text-lg text-muted-foreground">Loading...</p> <p className="text-lg text-muted-foreground">Loading...</p>
</div> </div>
</div> </div>
} }
> >
<ChunksPageContent /> <ChunksPageContent />
</Suspense> </Suspense>
); );
} }
export default function ProtectedChunksPage() { export default function ProtectedChunksPage() {
return ( return (
<ProtectedRoute> <ProtectedRoute>
<ChunksPage /> <ChunksPage />
</ProtectedRoute> </ProtectedRoute>
); );
} }

608
frontend/src/app/knowledge/page.tsx
View file

@ -1,236 +1,284 @@
"use client"; "use client";
import type { ColDef } from "ag-grid-community"; import type { ColDef, GetRowIdParams } from "ag-grid-community";
import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react"; import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react";
import { Building2, Cloud, HardDrive, Search, Trash2, X } from "lucide-react"; import { Building2, Cloud, HardDrive, Search, Trash2, X } from "lucide-react";
import { useRouter } from "next/navigation"; import { useRouter } from "next/navigation";
import { type ChangeEvent, useCallback, useRef, useState } from "react"; import {
type ChangeEvent,
useCallback,
useEffect,
useRef,
useState,
} from "react";
import { SiGoogledrive } from "react-icons/si"; import { SiGoogledrive } from "react-icons/si";
import { TbBrandOnedrive } from "react-icons/tb"; import { TbBrandOnedrive } from "react-icons/tb";
import { KnowledgeDropdown } from "@/components/knowledge-dropdown"; import { KnowledgeDropdown } from "@/components/knowledge-dropdown";
import { ProtectedRoute } from "@/components/protected-route"; import { ProtectedRoute } from "@/components/protected-route";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";
import { useLayout } from "@/contexts/layout-context";
import { useTask } from "@/contexts/task-context"; import { useTask } from "@/contexts/task-context";
import { type File, useGetSearchQuery } from "../api/queries/useGetSearchQuery"; import { type File, useGetSearchQuery } from "../api/queries/useGetSearchQuery";
import "@/components/AgGrid/registerAgGridModules"; import "@/components/AgGrid/registerAgGridModules";
import "@/components/AgGrid/agGridStyles.css"; import "@/components/AgGrid/agGridStyles.css";
import { toast } from "sonner"; import { toast } from "sonner";
import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdown"; import { KnowledgeActionsDropdown } from "@/components/knowledge-actions-dropdown";
import { filterAccentClasses } from "@/components/knowledge-filter-panel";
import { StatusBadge } from "@/components/ui/status-badge"; import { StatusBadge } from "@/components/ui/status-badge";
import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog"; import { DeleteConfirmationDialog } from "../../../components/confirmation-dialog";
import { useDeleteDocument } from "../api/mutations/useDeleteDocument"; import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
import { filterAccentClasses } from "@/components/knowledge-filter-panel";
// Function to get the appropriate icon for a connector type // Function to get the appropriate icon for a connector type
function getSourceIcon(connectorType?: string) { function getSourceIcon(connectorType?: string) {
switch (connectorType) { switch (connectorType) {
case "google_drive": case "google_drive":
return ( return (
<SiGoogledrive className="h-4 w-4 text-foreground flex-shrink-0" /> <SiGoogledrive className="h-4 w-4 text-foreground flex-shrink-0" />
); );
case "onedrive": case "onedrive":
return ( return (
<TbBrandOnedrive className="h-4 w-4 text-foreground flex-shrink-0" /> <TbBrandOnedrive className="h-4 w-4 text-foreground flex-shrink-0" />
); );
case "sharepoint": case "sharepoint":
return <Building2 className="h-4 w-4 text-foreground flex-shrink-0" />; return <Building2 className="h-4 w-4 text-foreground flex-shrink-0" />;
case "s3": case "s3":
return <Cloud className="h-4 w-4 text-foreground flex-shrink-0" />; return <Cloud className="h-4 w-4 text-foreground flex-shrink-0" />;
default: default:
return ( return (
<HardDrive className="h-4 w-4 text-muted-foreground flex-shrink-0" /> <HardDrive className="h-4 w-4 text-muted-foreground flex-shrink-0" />
); );
} }
} }
function SearchPage() { function SearchPage() {
const router = useRouter(); const router = useRouter();
const { isMenuOpen, files: taskFiles } = useTask(); const { isMenuOpen, files: taskFiles, refreshTasks } = useTask();
const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } = const { totalTopOffset } = useLayout();
useKnowledgeFilter(); const { selectedFilter, setSelectedFilter, parsedFilterData, isPanelOpen } =
const [selectedRows, setSelectedRows] = useState<File[]>([]); useKnowledgeFilter();
const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false); const [selectedRows, setSelectedRows] = useState<File[]>([]);
const [showBulkDeleteDialog, setShowBulkDeleteDialog] = useState(false);
const deleteDocumentMutation = useDeleteDocument(); const deleteDocumentMutation = useDeleteDocument();
const { data = [], isFetching } = useGetSearchQuery( useEffect(() => {
parsedFilterData?.query || "*", refreshTasks();
parsedFilterData }, [refreshTasks]);
);
const handleTableSearch = (e: ChangeEvent<HTMLInputElement>) => { const { data: searchData = [], isFetching } = useGetSearchQuery(
gridRef.current?.api.setGridOption("quickFilterText", e.target.value); parsedFilterData?.query || "*",
}; parsedFilterData,
);
// Convert TaskFiles to File format and merge with backend results
const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => {
return {
filename: taskFile.filename,
mimetype: taskFile.mimetype,
source_url: taskFile.source_url,
size: taskFile.size,
connector_type: taskFile.connector_type,
status: taskFile.status,
};
});
// Convert TaskFiles to File format and merge with backend results // Create a map of task files by filename for quick lookup
const taskFilesAsFiles: File[] = taskFiles.map((taskFile) => { const taskFileMap = new Map(
return { taskFilesAsFiles.map((file) => [file.filename, file]),
filename: taskFile.filename, );
mimetype: taskFile.mimetype,
source_url: taskFile.source_url,
size: taskFile.size,
connector_type: taskFile.connector_type,
status: taskFile.status,
};
});
const backendFiles = data as File[]; // Override backend files with task file status if they exist
const backendFiles = (searchData as File[])
.map((file) => {
const taskFile = taskFileMap.get(file.filename);
if (taskFile) {
// Override backend file with task file data (includes status)
return { ...file, ...taskFile };
}
return file;
})
.filter((file) => {
// Only filter out files that are currently processing AND in taskFiles
const taskFile = taskFileMap.get(file.filename);
return !taskFile || taskFile.status !== "processing";
});
const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => { const filteredTaskFiles = taskFilesAsFiles.filter((taskFile) => {
return ( return (
taskFile.status !== "active" && taskFile.status !== "active" &&
!backendFiles.some( !backendFiles.some(
(backendFile) => backendFile.filename === taskFile.filename (backendFile) => backendFile.filename === taskFile.filename,
) )
); );
}); });
// Combine task files first, then backend files // Combine task files first, then backend files
const fileResults = [...backendFiles, ...filteredTaskFiles]; const fileResults = [...backendFiles, ...filteredTaskFiles];
const gridRef = useRef<AgGridReact>(null); const handleTableSearch = (e: ChangeEvent<HTMLInputElement>) => {
gridRef.current?.api.setGridOption("quickFilterText", e.target.value);
};
const [columnDefs] = useState<ColDef<File>[]>([ const gridRef = useRef<AgGridReact>(null);
{
field: "filename",
headerName: "Source",
checkboxSelection: true,
headerCheckboxSelection: true,
initialFlex: 2,
minWidth: 220,
cellRenderer: ({ data, value }: CustomCellRendererProps<File>) => {
return (
<button
type="button"
className="flex items-center gap-2 cursor-pointer hover:text-blue-600 transition-colors text-left w-full"
onClick={() => {
router.push(
`/knowledge/chunks?filename=${encodeURIComponent(
data?.filename ?? ""
)}`
);
}}
>
{getSourceIcon(data?.connector_type)}
<span className="font-medium text-foreground truncate">
{value}
</span>
</button>
);
},
},
{
field: "size",
headerName: "Size",
valueFormatter: (params) =>
params.value ? `${Math.round(params.value / 1024)} KB` : "-",
},
{
field: "mimetype",
headerName: "Type",
},
{
field: "owner",
headerName: "Owner",
valueFormatter: (params) =>
params.data?.owner_name || params.data?.owner_email || "—",
},
{
field: "chunkCount",
headerName: "Chunks",
valueFormatter: (params) => params.data?.chunkCount?.toString() || "-",
},
{
field: "avgScore",
headerName: "Avg score",
initialFlex: 0.5,
cellRenderer: ({ value }: CustomCellRendererProps<File>) => {
return (
<span className="text-xs text-accent-emerald-foreground bg-accent-emerald px-2 py-1 rounded">
{value?.toFixed(2) ?? "-"}
</span>
);
},
},
{
field: "status",
headerName: "Status",
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
// Default to 'active' status if no status is provided
const status = data?.status || "active";
return <StatusBadge status={status} />;
},
},
{
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
return <KnowledgeActionsDropdown filename={data?.filename || ""} />;
},
cellStyle: {
alignItems: "center",
display: "flex",
justifyContent: "center",
padding: 0,
},
colId: "actions",
filter: false,
minWidth: 0,
width: 40,
resizable: false,
sortable: false,
initialFlex: 0,
},
]);
const defaultColDef: ColDef<File> = { const columnDefs = [
resizable: false, {
suppressMovable: true, field: "filename",
initialFlex: 1, headerName: "Source",
minWidth: 100, checkboxSelection: (params: CustomCellRendererProps<File>) =>
}; (params?.data?.status || "active") === "active",
headerCheckboxSelection: true,
initialFlex: 2,
minWidth: 220,
cellRenderer: ({ data, value }: CustomCellRendererProps<File>) => {
// Read status directly from data on each render
const status = data?.status || "active";
const isActive = status === "active";
console.log(data?.filename, status, "a");
return (
<div className="flex items-center overflow-hidden w-full">
<div
className={`transition-opacity duration-200 ${isActive ? "w-0" : "w-7"}`}
></div>
<button
type="button"
className="flex items-center gap-2 cursor-pointer hover:text-blue-600 transition-colors text-left flex-1 overflow-hidden"
onClick={() => {
if (!isActive) {
return;
}
router.push(
`/knowledge/chunks?filename=${encodeURIComponent(
data?.filename ?? "",
)}`,
);
}}
>
{getSourceIcon(data?.connector_type)}
<span className="font-medium text-foreground truncate">
{value}
</span>
</button>
</div>
);
},
},
{
field: "size",
headerName: "Size",
valueFormatter: (params: CustomCellRendererProps<File>) =>
params.value ? `${Math.round(params.value / 1024)} KB` : "-",
},
{
field: "mimetype",
headerName: "Type",
},
{
field: "owner",
headerName: "Owner",
valueFormatter: (params: CustomCellRendererProps<File>) =>
params.data?.owner_name || params.data?.owner_email || "—",
},
{
field: "chunkCount",
headerName: "Chunks",
valueFormatter: (params: CustomCellRendererProps<File>) => params.data?.chunkCount?.toString() || "-",
},
{
field: "avgScore",
headerName: "Avg score",
initialFlex: 0.5,
cellRenderer: ({ value }: CustomCellRendererProps<File>) => {
return (
<span className="text-xs text-accent-emerald-foreground bg-accent-emerald px-2 py-1 rounded">
{value?.toFixed(2) ?? "-"}
</span>
);
},
},
{
field: "status",
headerName: "Status",
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
console.log(data?.filename, data?.status, "b");
// Default to 'active' status if no status is provided
const status = data?.status || "active";
return <StatusBadge status={status} />;
},
},
{
cellRenderer: ({ data }: CustomCellRendererProps<File>) => {
const status = data?.status || "active";
if (status !== "active") {
return null;
}
return <KnowledgeActionsDropdown filename={data?.filename || ""} />;
},
cellStyle: {
alignItems: "center",
display: "flex",
justifyContent: "center",
padding: 0,
},
colId: "actions",
filter: false,
minWidth: 0,
width: 40,
resizable: false,
sortable: false,
initialFlex: 0,
},
];
const onSelectionChanged = useCallback(() => { const defaultColDef: ColDef<File> = {
if (gridRef.current) { resizable: false,
const selectedNodes = gridRef.current.api.getSelectedRows(); suppressMovable: true,
setSelectedRows(selectedNodes); initialFlex: 1,
} minWidth: 100,
}, []); };
const handleBulkDelete = async () => { const onSelectionChanged = useCallback(() => {
if (selectedRows.length === 0) return; if (gridRef.current) {
const selectedNodes = gridRef.current.api.getSelectedRows();
setSelectedRows(selectedNodes);
}
}, []);
try { const handleBulkDelete = async () => {
// Delete each file individually since the API expects one filename at a time if (selectedRows.length === 0) return;
const deletePromises = selectedRows.map((row) =>
deleteDocumentMutation.mutateAsync({ filename: row.filename })
);
await Promise.all(deletePromises); try {
// Delete each file individually since the API expects one filename at a time
const deletePromises = selectedRows.map((row) =>
deleteDocumentMutation.mutateAsync({ filename: row.filename }),
);
toast.success( await Promise.all(deletePromises);
`Successfully deleted ${selectedRows.length} document${
selectedRows.length > 1 ? "s" : ""
}`
);
setSelectedRows([]);
setShowBulkDeleteDialog(false);
// Clear selection in the grid toast.success(
if (gridRef.current) { `Successfully deleted ${selectedRows.length} document${
gridRef.current.api.deselectAll(); selectedRows.length > 1 ? "s" : ""
} }`,
} catch (error) { );
toast.error( setSelectedRows([]);
error instanceof Error setShowBulkDeleteDialog(false);
? error.message
: "Failed to delete some documents" // Clear selection in the grid
); if (gridRef.current) {
} gridRef.current.api.deselectAll();
}; }
} catch (error) {
toast.error(
error instanceof Error
? error.message
: "Failed to delete some documents",
);
}
};
return ( return (
<div <div
className={`fixed inset-0 md:left-72 top-[53px] flex flex-col transition-all duration-300 ${ className={`fixed inset-0 md:left-72 flex flex-col transition-all duration-300 ${
isMenuOpen && isPanelOpen isMenuOpen && isPanelOpen
? "md:right-[704px]" ? "md:right-[704px]"
: // Both open: 384px (menu) + 320px (KF panel) : // Both open: 384px (menu) + 320px (KF panel)
@ -242,6 +290,7 @@ function SearchPage() {
: // Only KF panel open: 320px : // Only KF panel open: 320px
"md:right-6" // Neither open: 24px "md:right-6" // Neither open: 24px
}`} }`}
style={{ top: `${totalTopOffset}px` }}
> >
<div className="flex-1 flex flex-col min-h-0 px-6 py-6"> <div className="flex-1 flex flex-col min-h-0 px-6 py-6">
<div className="flex items-center justify-between mb-6"> <div className="flex items-center justify-between mb-6">
@ -249,38 +298,37 @@ function SearchPage() {
<KnowledgeDropdown variant="button" /> <KnowledgeDropdown variant="button" />
</div> </div>
{/* Search Input Area */} {/* Search Input Area */}
<div className="flex-shrink-0 mb-6 xl:max-w-[75%]"> <div className="flex-shrink-0 mb-6 xl:max-w-[75%]">
<form className="flex gap-3"> <form className="flex gap-3">
<div className="primary-input min-h-10 !flex items-center flex-nowrap focus-within:border-foreground transition-colors !p-[0.3rem]"> <div className="primary-input min-h-10 !flex items-center flex-nowrap focus-within:border-foreground transition-colors !p-[0.3rem]">
{selectedFilter?.name && ( {selectedFilter?.name && (
<div <div
className={`flex items-center gap-1 h-full px-1.5 py-0.5 mr-1 rounded max-w-[25%] ${ className={`flex items-center gap-1 h-full px-1.5 py-0.5 mr-1 rounded max-w-[25%] ${
filterAccentClasses[parsedFilterData?.color || "zinc"] filterAccentClasses[parsedFilterData?.color || "zinc"]
}`} }`}
> >
<span className="truncate">{selectedFilter?.name}</span> <span className="truncate">{selectedFilter?.name}</span>
<X <X
aria-label="Remove filter" aria-label="Remove filter"
className="h-4 w-4 flex-shrink-0 cursor-pointer" className="h-4 w-4 flex-shrink-0 cursor-pointer"
onClick={() => setSelectedFilter(null)} onClick={() => setSelectedFilter(null)}
/> />
</div> </div>
)} )}
<Search <Search
className="h-4 w-4 ml-1 flex-shrink-0 text-placeholder-foreground" className="h-4 w-4 ml-1 flex-shrink-0 text-placeholder-foreground"
strokeWidth={1.5} />
/> <input
<input className="bg-transparent w-full h-full ml-2 focus:outline-none focus-visible:outline-none font-mono placeholder:font-mono"
className="bg-transparent w-full h-full ml-2 focus:outline-none focus-visible:outline-none font-mono placeholder:font-mono" name="search-query"
name="search-query" id="search-query"
id="search-query" type="text"
type="text" placeholder="Enter your search query..."
placeholder="Search your documents..." onChange={handleTableSearch}
onChange={handleTableSearch} />
/> </div>
</div> {/* <Button
{/* <Button
type="submit" type="submit"
variant="outline" variant="outline"
className="rounded-lg p-0 flex-shrink-0" className="rounded-lg p-0 flex-shrink-0"
@ -291,8 +339,8 @@ function SearchPage() {
<Search className="h-4 w-4" /> <Search className="h-4 w-4" />
)} )}
</Button> */} </Button> */}
{/* //TODO: Implement sync button */} {/* //TODO: Implement sync button */}
{/* <Button {/* <Button
type="button" type="button"
variant="outline" variant="outline"
className="rounded-lg flex-shrink-0" className="rounded-lg flex-shrink-0"
@ -300,69 +348,69 @@ function SearchPage() {
> >
Sync Sync
</Button> */} </Button> */}
{selectedRows.length > 0 && ( {selectedRows.length > 0 && (
<Button <Button
type="button" type="button"
variant="destructive" variant="destructive"
className="rounded-lg flex-shrink-0" className="rounded-lg flex-shrink-0"
onClick={() => setShowBulkDeleteDialog(true)} onClick={() => setShowBulkDeleteDialog(true)}
> >
<Trash2 className="h-4 w-4" /> Delete <Trash2 className="h-4 w-4" /> Delete
</Button> </Button>
)} )}
</form> </form>
</div> </div>
<AgGridReact <AgGridReact
className="w-full overflow-auto" className="w-full overflow-auto"
columnDefs={columnDefs} columnDefs={columnDefs as ColDef<File>[]}
defaultColDef={defaultColDef} defaultColDef={defaultColDef}
loading={isFetching} loading={isFetching}
ref={gridRef} ref={gridRef}
rowData={fileResults} rowData={fileResults}
rowSelection="multiple" rowSelection="multiple"
rowMultiSelectWithClick={false} rowMultiSelectWithClick={false}
suppressRowClickSelection={true} suppressRowClickSelection={true}
getRowId={(params) => params.data.filename} getRowId={(params: GetRowIdParams<File>) => params.data?.filename}
domLayout="normal" domLayout="normal"
onSelectionChanged={onSelectionChanged} onSelectionChanged={onSelectionChanged}
noRowsOverlayComponent={() => ( noRowsOverlayComponent={() => (
<div className="text-center pb-[45px]"> <div className="text-center pb-[45px]">
<div className="text-lg text-primary font-semibold"> <div className="text-lg text-primary font-semibold">
No knowledge No knowledge
</div> </div>
<div className="text-sm mt-1 text-muted-foreground"> <div className="text-sm mt-1 text-muted-foreground">
Add files from local or your preferred cloud. Add files from local or your preferred cloud.
</div> </div>
</div> </div>
)} )}
/> />
</div> </div>
{/* Bulk Delete Confirmation Dialog */} {/* Bulk Delete Confirmation Dialog */}
<DeleteConfirmationDialog <DeleteConfirmationDialog
open={showBulkDeleteDialog} open={showBulkDeleteDialog}
onOpenChange={setShowBulkDeleteDialog} onOpenChange={setShowBulkDeleteDialog}
title="Delete Documents" title="Delete Documents"
description={`Are you sure you want to delete ${ description={`Are you sure you want to delete ${
selectedRows.length selectedRows.length
} document${ } document${
selectedRows.length > 1 ? "s" : "" selectedRows.length > 1 ? "s" : ""
}? This will remove all chunks and data associated with these documents. This action cannot be undone. }? This will remove all chunks and data associated with these documents. This action cannot be undone.
Documents to be deleted: Documents to be deleted:
${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`} ${selectedRows.map((row) => `• ${row.filename}`).join("\n")}`}
confirmText="Delete All" confirmText="Delete All"
onConfirm={handleBulkDelete} onConfirm={handleBulkDelete}
isLoading={deleteDocumentMutation.isPending} isLoading={deleteDocumentMutation.isPending}
/> />
</div> </div>
); );
} }
export default function ProtectedSearchPage() { export default function ProtectedSearchPage() {
return ( return (
<ProtectedRoute> <ProtectedRoute>
<SearchPage /> <SearchPage />
</ProtectedRoute> </ProtectedRoute>
); );
} }

2
frontend/src/app/settings/page.tsx
View file

@ -149,7 +149,7 @@ function KnowledgeSourcesPage() {
const [systemPrompt, setSystemPrompt] = useState<string>(""); const [systemPrompt, setSystemPrompt] = useState<string>("");
const [chunkSize, setChunkSize] = useState<number>(1024); const [chunkSize, setChunkSize] = useState<number>(1024);
const [chunkOverlap, setChunkOverlap] = useState<number>(50); const [chunkOverlap, setChunkOverlap] = useState<number>(50);
const [tableStructure, setTableStructure] = useState<boolean>(false); const [tableStructure, setTableStructure] = useState<boolean>(true);
const [ocr, setOcr] = useState<boolean>(false); const [ocr, setOcr] = useState<boolean>(false);
const [pictureDescriptions, setPictureDescriptions] = const [pictureDescriptions, setPictureDescriptions] =
useState<boolean>(false); useState<boolean>(false);

49
frontend/src/components/layout-wrapper.tsx
View file

@ -7,6 +7,7 @@ import {
type ChatConversation, type ChatConversation,
} from "@/app/api/queries/useGetConversationsQuery"; } from "@/app/api/queries/useGetConversationsQuery";
import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery"; import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery";
import { DoclingHealthBanner } from "@/components/docling-health-banner";
import { KnowledgeFilterPanel } from "@/components/knowledge-filter-panel"; import { KnowledgeFilterPanel } from "@/components/knowledge-filter-panel";
import Logo from "@/components/logo/logo"; import Logo from "@/components/logo/logo";
import { Navigation } from "@/components/navigation"; import { Navigation } from "@/components/navigation";
@ -16,9 +17,11 @@ import { UserNav } from "@/components/user-nav";
import { useAuth } from "@/contexts/auth-context"; import { useAuth } from "@/contexts/auth-context";
import { useChat } from "@/contexts/chat-context"; import { useChat } from "@/contexts/chat-context";
import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context"; import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";
import { LayoutProvider } from "@/contexts/layout-context";
// import { GitHubStarButton } from "@/components/github-star-button" // import { GitHubStarButton } from "@/components/github-star-button"
// import { DiscordLink } from "@/components/discord-link" // import { DiscordLink } from "@/components/discord-link"
import { useTask } from "@/contexts/task-context"; import { useTask } from "@/contexts/task-context";
import { useDoclingHealthQuery } from "@/src/app/api/queries/useDoclingHealthQuery";
import { cn } from "@/lib/utils"; import { cn } from "@/lib/utils";
export function LayoutWrapper({ children }: { children: React.ReactNode }) { export function LayoutWrapper({ children }: { children: React.ReactNode }) {
@ -35,6 +38,11 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
const { isLoading: isSettingsLoading, data: settings } = useGetSettingsQuery({ const { isLoading: isSettingsLoading, data: settings } = useGetSettingsQuery({
enabled: isAuthenticated || isNoAuthMode, enabled: isAuthenticated || isNoAuthMode,
}); });
const {
data: health,
isLoading: isHealthLoading,
isError,
} = useDoclingHealthQuery();
// Only fetch conversations on chat page // Only fetch conversations on chat page
const isOnChatPage = pathname === "/" || pathname === "/chat"; const isOnChatPage = pathname === "/" || pathname === "/chat";
@ -64,6 +72,17 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
task.status === "processing" task.status === "processing"
); );
const isUnhealthy = health?.status === "unhealthy" || isError;
const isBannerVisible = !isHealthLoading && isUnhealthy;
// Dynamic height calculations based on banner visibility
const headerHeight = 53;
const bannerHeight = 52; // Approximate banner height
const totalTopOffset = isBannerVisible
? headerHeight + bannerHeight
: headerHeight;
const mainContentHeight = `calc(100vh - ${totalTopOffset}px)`;
// Show loading state when backend isn't ready // Show loading state when backend isn't ready
if (isLoading || isSettingsLoading) { if (isLoading || isSettingsLoading) {
return ( return (
@ -76,7 +95,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
); );
} }
if (isAuthPage || (settings && !settings.edited)) { if (isAuthPage) {
// For auth pages, render without navigation // For auth pages, render without navigation
return <div className="h-full">{children}</div>; return <div className="h-full">{children}</div>;
} }
@ -84,6 +103,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
// For all other pages, render with Langflow-styled navigation and task menu // For all other pages, render with Langflow-styled navigation and task menu
return ( return (
<div className="h-full relative"> <div className="h-full relative">
<DoclingHealthBanner className="w-full pt-2" />
<header className="header-arrangement bg-background sticky top-0 z-50 h-10"> <header className="header-arrangement bg-background sticky top-0 z-50 h-10">
<div className="header-start-display px-[16px]"> <div className="header-start-display px-[16px]">
{/* Logo/Title */} {/* Logo/Title */}
@ -124,7 +144,10 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
</div> </div>
</div> </div>
</header> </header>
<div className="side-bar-arrangement bg-background fixed left-0 top-[40px] bottom-0 md:flex hidden pt-1"> <div
className="side-bar-arrangement bg-background fixed left-0 top-[40px] bottom-0 md:flex hidden pt-1"
style={{ top: `${totalTopOffset}px` }}
>
<Navigation <Navigation
conversations={conversations} conversations={conversations}
isConversationsLoading={isConversationsLoading} isConversationsLoading={isConversationsLoading}
@ -132,7 +155,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
/> />
</div> </div>
<main <main
className={`md:pl-72 transition-all duration-300 overflow-y-auto h-[calc(100vh-53px)] ${ className={`md:pl-72 transition-all duration-300 overflow-y-auto ${
isMenuOpen && isPanelOpen isMenuOpen && isPanelOpen
? "md:pr-[728px]" ? "md:pr-[728px]"
: // Both open: 384px (menu) + 320px (KF panel) + 24px (original padding) : // Both open: 384px (menu) + 320px (KF panel) + 24px (original padding)
@ -144,15 +167,21 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
: // Only KF panel open: 320px : // Only KF panel open: 320px
"md:pr-0" // Neither open: 24px "md:pr-0" // Neither open: 24px
}`} }`}
style={{ height: mainContentHeight }}
> >
<div <LayoutProvider
className={cn( headerHeight={headerHeight}
"py-6 lg:py-8 px-4 lg:px-6", totalTopOffset={totalTopOffset}
isSmallWidthPath ? "max-w-[850px]" : "container"
)}
> >
{children} <div
</div> className={cn(
"py-6 lg:py-8 px-4 lg:px-6",
isSmallWidthPath ? "max-w-[850px]" : "container"
)}
>
{children}
</div>
</LayoutProvider>
</main> </main>
<TaskNotificationMenu /> <TaskNotificationMenu />
<KnowledgeFilterPanel /> <KnowledgeFilterPanel />

11
frontend/src/components/task-notification-menu.tsx
View file

@ -1,6 +1,6 @@
"use client" "use client"
import { useState } from 'react' import { useEffect, useState } from 'react'
import { Bell, CheckCircle, XCircle, Clock, Loader2, ChevronDown, ChevronUp, X } from 'lucide-react' import { Bell, CheckCircle, XCircle, Clock, Loader2, ChevronDown, ChevronUp, X } from 'lucide-react'
import { Button } from '@/components/ui/button' import { Button } from '@/components/ui/button'
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card' import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'
@ -8,9 +8,16 @@ import { Badge } from '@/components/ui/badge'
import { useTask, Task } from '@/contexts/task-context' import { useTask, Task } from '@/contexts/task-context'
export function TaskNotificationMenu() { export function TaskNotificationMenu() {
const { tasks, isFetching, isMenuOpen, cancelTask } = useTask() const { tasks, isFetching, isMenuOpen, isRecentTasksExpanded, cancelTask } = useTask()
const [isExpanded, setIsExpanded] = useState(false) const [isExpanded, setIsExpanded] = useState(false)
// Sync local state with context state
useEffect(() => {
if (isRecentTasksExpanded) {
setIsExpanded(true)
}
}, [isRecentTasksExpanded])
// Don't render if menu is closed // Don't render if menu is closed
if (!isMenuOpen) return null if (!isMenuOpen) return null

52
frontend/src/components/ui/animated-processing-icon.tsx
View file

@ -1,26 +1,16 @@
interface AnimatedProcessingIconProps { import type { SVGProps } from "react";
className?: string;
size?: number;
}
export const AnimatedProcessingIcon = ({ export const AnimatedProcessingIcon = (props: SVGProps<SVGSVGElement>) => {
className = "", return (
size = 10, <svg
}: AnimatedProcessingIconProps) => { viewBox="0 0 8 12"
const width = Math.round((size * 6) / 10); fill="none"
const height = size; xmlns="http://www.w3.org/2000/svg"
{...props}
return ( >
<svg <title>Processing</title>
width={width} <style>
height={height} {`
viewBox="0 0 6 10"
fill="none"
xmlns="http://www.w3.org/2000/svg"
className={className}
>
<style>
{`
.dot-1 { animation: pulse-wave 1.5s infinite; animation-delay: 0s; } .dot-1 { animation: pulse-wave 1.5s infinite; animation-delay: 0s; }
.dot-2 { animation: pulse-wave 1.5s infinite; animation-delay: 0.1s; } .dot-2 { animation: pulse-wave 1.5s infinite; animation-delay: 0.1s; }
.dot-3 { animation: pulse-wave 1.5s infinite; animation-delay: 0.2s; } .dot-3 { animation: pulse-wave 1.5s infinite; animation-delay: 0.2s; }
@ -30,20 +20,18 @@ export const AnimatedProcessingIcon = ({
@keyframes pulse-wave { @keyframes pulse-wave {
0%, 60%, 100% { 0%, 60%, 100% {
opacity: 0.25; opacity: 0.25;
transform: scale(1);
} }
30% { 30% {
opacity: 1; opacity: 1;
transform: scale(1.2);
} }
} }
`} `}
</style> </style>
<circle className="dot-1" cx="1" cy="5" r="1" fill="currentColor" /> <circle className="dot-1" cx="2" cy="6" r="1" fill="currentColor" />
<circle className="dot-2" cx="1" cy="9" r="1" fill="currentColor" /> <circle className="dot-2" cx="2" cy="10" r="1" fill="currentColor" />
<circle className="dot-3" cx="5" cy="1" r="1" fill="currentColor" /> <circle className="dot-3" cx="6" cy="2" r="1" fill="currentColor" />
<circle className="dot-4" cx="5" cy="5" r="1" fill="currentColor" /> <circle className="dot-4" cx="6" cy="6" r="1" fill="currentColor" />
<circle className="dot-5" cx="5" cy="9" r="1" fill="currentColor" /> <circle className="dot-5" cx="6" cy="10" r="1" fill="currentColor" />
</svg> </svg>
); );
}; };

2
frontend/src/components/ui/status-badge.tsx
View file

@ -50,7 +50,7 @@ export const StatusBadge = ({ status, className }: StatusBadgeProps) => {
}`} }`}
> >
{status === "processing" && ( {status === "processing" && (
<AnimatedProcessingIcon className="text-current mr-2" size={10} /> <AnimatedProcessingIcon className="text-current h-3 w-3 shrink-0" />
)} )}
{config.label} {config.label}
</div> </div>

34
frontend/src/contexts/layout-context.tsx Normal file
View file

@ -0,0 +1,34 @@
"use client";
import { createContext, useContext } from "react";
interface LayoutContextType {
headerHeight: number;
totalTopOffset: number;
}
const LayoutContext = createContext<LayoutContextType | undefined>(undefined);
export function useLayout() {
const context = useContext(LayoutContext);
if (context === undefined) {
throw new Error("useLayout must be used within a LayoutProvider");
}
return context;
}
export function LayoutProvider({
children,
headerHeight,
totalTopOffset
}: {
children: React.ReactNode;
headerHeight: number;
totalTopOffset: number;
}) {
return (
<LayoutContext.Provider value={{ headerHeight, totalTopOffset }}>
{children}
</LayoutContext.Provider>
);
}

467
frontend/src/contexts/task-context.tsx
View file

@ -7,33 +7,18 @@ import {
useCallback, useCallback,
useContext, useContext,
useEffect, useEffect,
useRef,
useState, useState,
} from "react"; } from "react";
import { toast } from "sonner"; import { toast } from "sonner";
import { useCancelTaskMutation } from "@/app/api/mutations/useCancelTaskMutation";
import {
type Task,
useGetTasksQuery,
} from "@/app/api/queries/useGetTasksQuery";
import { useAuth } from "@/contexts/auth-context"; import { useAuth } from "@/contexts/auth-context";
export interface Task { // Task interface is now imported from useGetTasksQuery
task_id: string;
status:
| "pending"
| "running"
| "processing"
| "completed"
| "failed"
| "error";
total_files?: number;
processed_files?: number;
successful_files?: number;
failed_files?: number;
running_files?: number;
pending_files?: number;
created_at: string;
updated_at: string;
duration_seconds?: number;
result?: Record<string, unknown>;
error?: string;
files?: Record<string, Record<string, unknown>>;
}
export interface TaskFile { export interface TaskFile {
filename: string; filename: string;
@ -51,27 +36,54 @@ interface TaskContextType {
files: TaskFile[]; files: TaskFile[];
addTask: (taskId: string) => void; addTask: (taskId: string) => void;
addFiles: (files: Partial<TaskFile>[], taskId: string) => void; addFiles: (files: Partial<TaskFile>[], taskId: string) => void;
removeTask: (taskId: string) => void;
refreshTasks: () => Promise<void>; refreshTasks: () => Promise<void>;
cancelTask: (taskId: string) => Promise<void>; cancelTask: (taskId: string) => Promise<void>;
isPolling: boolean; isPolling: boolean;
isFetching: boolean; isFetching: boolean;
isMenuOpen: boolean; isMenuOpen: boolean;
toggleMenu: () => void; toggleMenu: () => void;
isRecentTasksExpanded: boolean;
setRecentTasksExpanded: (expanded: boolean) => void;
// React Query states
isLoading: boolean;
error: Error | null;
} }
const TaskContext = createContext<TaskContextType | undefined>(undefined); const TaskContext = createContext<TaskContextType | undefined>(undefined);
export function TaskProvider({ children }: { children: React.ReactNode }) { export function TaskProvider({ children }: { children: React.ReactNode }) {
const [tasks, setTasks] = useState<Task[]>([]);
const [files, setFiles] = useState<TaskFile[]>([]); const [files, setFiles] = useState<TaskFile[]>([]);
const [isPolling, setIsPolling] = useState(false);
const [isFetching, setIsFetching] = useState(false);
const [isMenuOpen, setIsMenuOpen] = useState(false); const [isMenuOpen, setIsMenuOpen] = useState(false);
const [isRecentTasksExpanded, setIsRecentTasksExpanded] = useState(false);
const previousTasksRef = useRef<Task[]>([]);
const { isAuthenticated, isNoAuthMode } = useAuth(); const { isAuthenticated, isNoAuthMode } = useAuth();
const queryClient = useQueryClient(); const queryClient = useQueryClient();
// Use React Query hooks
const {
data: tasks = [],
isLoading,
error,
refetch: refetchTasks,
isFetching,
} = useGetTasksQuery({
enabled: isAuthenticated || isNoAuthMode,
});
const cancelTaskMutation = useCancelTaskMutation({
onSuccess: () => {
toast.success("Task cancelled", {
description: "Task has been cancelled successfully",
});
},
onError: (error) => {
toast.error("Failed to cancel task", {
description: error.message,
});
},
});
const refetchSearch = useCallback(() => { const refetchSearch = useCallback(() => {
queryClient.invalidateQueries({ queryClient.invalidateQueries({
queryKey: ["search"], queryKey: ["search"],
@ -99,265 +111,216 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
[], [],
); );
const fetchTasks = useCallback(async () => { // Handle task status changes and file updates
if (!isAuthenticated && !isNoAuthMode) return; useEffect(() => {
if (tasks.length === 0) {
setIsFetching(true); // Store current tasks as previous for next comparison
try { previousTasksRef.current = tasks;
const response = await fetch("/api/tasks"); return;
if (response.ok) {
const data = await response.json();
const newTasks = data.tasks || [];
// Update tasks and check for status changes in the same state update
setTasks((prevTasks) => {
// Check for newly completed tasks to show toasts
if (prevTasks.length > 0) {
newTasks.forEach((newTask: Task) => {
const oldTask = prevTasks.find(
(t) => t.task_id === newTask.task_id,
);
// Update or add files from task.files if available
if (newTask.files && typeof newTask.files === "object") {
const taskFileEntries = Object.entries(newTask.files);
const now = new Date().toISOString();
taskFileEntries.forEach(([filePath, fileInfo]) => {
if (typeof fileInfo === "object" && fileInfo) {
const fileName = filePath.split("/").pop() || filePath;
const fileStatus = fileInfo.status as string;
// Map backend file status to our TaskFile status
let mappedStatus: TaskFile["status"];
switch (fileStatus) {
case "pending":
case "running":
mappedStatus = "processing";
break;
case "completed":
mappedStatus = "active";
break;
case "failed":
mappedStatus = "failed";
break;
default:
mappedStatus = "processing";
}
setFiles((prevFiles) => {
const existingFileIndex = prevFiles.findIndex(
(f) =>
f.source_url === filePath &&
f.task_id === newTask.task_id,
);
// Detect connector type based on file path or other indicators
let connectorType = "local";
if (filePath.includes("/") && !filePath.startsWith("/")) {
// Likely S3 key format (bucket/path/file.ext)
connectorType = "s3";
}
const fileEntry: TaskFile = {
filename: fileName,
mimetype: "", // We don't have this info from the task
source_url: filePath,
size: 0, // We don't have this info from the task
connector_type: connectorType,
status: mappedStatus,
task_id: newTask.task_id,
created_at:
typeof fileInfo.created_at === "string"
? fileInfo.created_at
: now,
updated_at:
typeof fileInfo.updated_at === "string"
? fileInfo.updated_at
: now,
};
if (existingFileIndex >= 0) {
// Update existing file
const updatedFiles = [...prevFiles];
updatedFiles[existingFileIndex] = fileEntry;
return updatedFiles;
} else {
// Add new file
return [...prevFiles, fileEntry];
}
});
}
});
}
if (
oldTask &&
oldTask.status !== "completed" &&
newTask.status === "completed"
) {
// Task just completed - show success toast
toast.success("Task completed successfully", {
description: `Task ${newTask.task_id} has finished processing.`,
action: {
label: "View",
onClick: () => console.log("View task", newTask.task_id),
},
});
refetchSearch();
// Dispatch knowledge updated event for all knowledge-related pages
console.log(
"Task completed successfully, dispatching knowledgeUpdated event",
);
window.dispatchEvent(new CustomEvent("knowledgeUpdated"));
// Remove files for this completed task from the files list
setFiles((prevFiles) =>
prevFiles.filter((file) => file.task_id !== newTask.task_id),
);
} else if (
oldTask &&
oldTask.status !== "failed" &&
oldTask.status !== "error" &&
(newTask.status === "failed" || newTask.status === "error")
) {
// Task just failed - show error toast
toast.error("Task failed", {
description: `Task ${newTask.task_id} failed: ${
newTask.error || "Unknown error"
}`,
});
// Files will be updated to failed status by the file parsing logic above
}
});
}
return newTasks;
});
}
} catch (error) {
console.error("Failed to fetch tasks:", error);
} finally {
setIsFetching(false);
} }
}, [isAuthenticated, isNoAuthMode, refetchSearch]); // Removed 'tasks' from dependencies to prevent infinite loop!
const addTask = useCallback((taskId: string) => { // Check for task status changes by comparing with previous tasks
// Immediately start aggressive polling for the new task tasks.forEach((currentTask) => {
let pollAttempts = 0; const previousTask = previousTasksRef.current.find(
const maxPollAttempts = 30; // Poll for up to 30 seconds (prev) => prev.task_id === currentTask.task_id,
);
const aggressivePoll = async () => { // Only show toasts if we have previous data and status has changed
try { if (
const response = await fetch("/api/tasks"); (previousTask && previousTask.status !== currentTask.status) ||
if (response.ok) { (!previousTask && previousTasksRef.current.length !== 0)
const data = await response.json(); ) {
const newTasks = data.tasks || []; // Process files from failed task and add them to files list
const foundTask = newTasks.find( if (currentTask.files && typeof currentTask.files === "object") {
(task: Task) => task.task_id === taskId, const taskFileEntries = Object.entries(currentTask.files);
); const now = new Date().toISOString();
if (foundTask) { taskFileEntries.forEach(([filePath, fileInfo]) => {
// Task found! Update the tasks state if (typeof fileInfo === "object" && fileInfo) {
setTasks((prevTasks) => { // Use the filename from backend if available, otherwise extract from path
// Check if task is already in the list const fileName =
const exists = prevTasks.some((t) => t.task_id === taskId); (fileInfo as any).filename ||
if (!exists) { filePath.split("/").pop() ||
return [...prevTasks, foundTask]; filePath;
const fileStatus = fileInfo.status as string;
// Map backend file status to our TaskFile status
let mappedStatus: TaskFile["status"];
switch (fileStatus) {
case "pending":
case "running":
mappedStatus = "processing";
break;
case "completed":
mappedStatus = "active";
break;
case "failed":
mappedStatus = "failed";
break;
default:
mappedStatus = "processing";
} }
// Update existing task
return prevTasks.map((t) => setFiles((prevFiles) => {
t.task_id === taskId ? foundTask : t, const existingFileIndex = prevFiles.findIndex(
); (f) =>
}); f.source_url === filePath &&
return; // Stop polling, we found it f.task_id === currentTask.task_id,
} );
// Detect connector type based on file path or other indicators
let connectorType = "local";
if (filePath.includes("/") && !filePath.startsWith("/")) {
// Likely S3 key format (bucket/path/file.ext)
connectorType = "s3";
}
const fileEntry: TaskFile = {
filename: fileName,
mimetype: "", // We don't have this info from the task
source_url: filePath,
size: 0, // We don't have this info from the task
connector_type: connectorType,
status: mappedStatus,
task_id: currentTask.task_id,
created_at:
typeof fileInfo.created_at === "string"
? fileInfo.created_at
: now,
updated_at:
typeof fileInfo.updated_at === "string"
? fileInfo.updated_at
: now,
};
if (existingFileIndex >= 0) {
// Update existing file
const updatedFiles = [...prevFiles];
updatedFiles[existingFileIndex] = fileEntry;
return updatedFiles;
} else {
// Add new file
return [...prevFiles, fileEntry];
}
});
}
});
} }
} catch (error) { if (
console.error("Aggressive polling failed:", error); previousTask &&
} previousTask.status !== "completed" &&
currentTask.status === "completed"
) {
// Task just completed - show success toast with file counts
const successfulFiles = currentTask.successful_files || 0;
const failedFiles = currentTask.failed_files || 0;
pollAttempts++; let description = "";
if (pollAttempts < maxPollAttempts) { if (failedFiles > 0) {
// Continue polling every 1 second for new tasks description = `${successfulFiles} file${
setTimeout(aggressivePoll, 1000); successfulFiles !== 1 ? "s" : ""
} } uploaded successfully, ${failedFiles} file${
}; failedFiles !== 1 ? "s" : ""
} failed`;
} else {
description = `${successfulFiles} file${
successfulFiles !== 1 ? "s" : ""
} uploaded successfully`;
}
// Start aggressive polling after a short delay to allow backend to process toast.success("Task completed", {
setTimeout(aggressivePoll, 500); description,
}, []); action: {
label: "View",
onClick: () => {
setIsMenuOpen(true);
setIsRecentTasksExpanded(true);
},
},
});
setTimeout(() => {
setFiles((prevFiles) =>
prevFiles.filter(
(file) =>
file.task_id !== currentTask.task_id ||
file.status === "failed",
),
);
refetchSearch();
}, 500);
} else if (
previousTask &&
previousTask.status !== "failed" &&
previousTask.status !== "error" &&
(currentTask.status === "failed" || currentTask.status === "error")
) {
// Task just failed - show error toast
toast.error("Task failed", {
description: `Task ${currentTask.task_id} failed: ${
currentTask.error || "Unknown error"
}`,
});
}
}
});
// Store current tasks as previous for next comparison
previousTasksRef.current = tasks;
}, [tasks, refetchSearch]);
const addTask = useCallback(
(_taskId: string) => {
// React Query will automatically handle polling when tasks are active
// Just trigger a refetch to get the latest data
setTimeout(() => {
refetchTasks();
}, 500);
},
[refetchTasks],
);
const refreshTasks = useCallback(async () => { const refreshTasks = useCallback(async () => {
await fetchTasks(); setFiles([]);
}, [fetchTasks]); await refetchTasks();
}, [refetchTasks]);
const removeTask = useCallback((taskId: string) => {
setTasks((prev) => prev.filter((task) => task.task_id !== taskId));
}, []);
const cancelTask = useCallback( const cancelTask = useCallback(
async (taskId: string) => { async (taskId: string) => {
try { cancelTaskMutation.mutate({ taskId });
const response = await fetch(`/api/tasks/${taskId}/cancel`, {
method: "POST",
});
if (response.ok) {
// Immediately refresh tasks to show the updated status
await fetchTasks();
toast.success("Task cancelled", {
description: `Task ${taskId.substring(0, 8)}... has been cancelled`,
});
} else {
const errorData = await response.json().catch(() => ({}));
throw new Error(errorData.error || "Failed to cancel task");
}
} catch (error) {
console.error("Failed to cancel task:", error);
toast.error("Failed to cancel task", {
description: error instanceof Error ? error.message : "Unknown error",
});
}
}, },
[fetchTasks], [cancelTaskMutation],
); );
const toggleMenu = useCallback(() => { const toggleMenu = useCallback(() => {
setIsMenuOpen((prev) => !prev); setIsMenuOpen((prev) => !prev);
}, []); }, []);
// Periodic polling for task updates // Determine if we're polling based on React Query's refetch interval
useEffect(() => { const isPolling =
if (!isAuthenticated && !isNoAuthMode) return; isFetching &&
tasks.some(
setIsPolling(true); (task) =>
task.status === "pending" ||
// Initial fetch task.status === "running" ||
fetchTasks(); task.status === "processing",
);
// Set up polling interval - every 3 seconds (more responsive for active tasks)
const interval = setInterval(fetchTasks, 3000);
return () => {
clearInterval(interval);
setIsPolling(false);
};
}, [isAuthenticated, isNoAuthMode, fetchTasks]);
const value: TaskContextType = { const value: TaskContextType = {
tasks, tasks,
files, files,
addTask, addTask,
addFiles, addFiles,
removeTask,
refreshTasks, refreshTasks,
cancelTask, cancelTask,
isPolling, isPolling,
isFetching, isFetching,
isMenuOpen, isMenuOpen,
toggleMenu, toggleMenu,
isRecentTasksExpanded,
setRecentTasksExpanded: setIsRecentTasksExpanded,
isLoading,
error,
}; };
return <TaskContext.Provider value={value}>{children}</TaskContext.Provider>; return <TaskContext.Provider value={value}>{children}</TaskContext.Provider>;

2
frontend/src/lib/constants.ts
View file

@ -12,7 +12,7 @@ export const DEFAULT_AGENT_SETTINGS = {
export const DEFAULT_KNOWLEDGE_SETTINGS = { export const DEFAULT_KNOWLEDGE_SETTINGS = {
chunk_size: 1000, chunk_size: 1000,
chunk_overlap: 200, chunk_overlap: 200,
table_structure: false, table_structure: true,
ocr: false, ocr: false,
picture_descriptions: false picture_descriptions: false
} as const; } as const;

2
pyproject.toml
View file

@ -1,6 +1,6 @@
[project] [project]
name = "openrag" name = "openrag"
version = "0.1.14.dev2" version = "0.1.14.dev3"
description = "Add your description here" description = "Add your description here"
readme = "README.md" readme = "README.md"
requires-python = ">=3.13" requires-python = ">=3.13"

86
src/api/documents.py
View file

@ -6,14 +6,13 @@ from config.settings import INDEX_NAME
logger = get_logger(__name__) logger = get_logger(__name__)
async def delete_documents_by_filename(request: Request, document_service, session_manager): async def check_filename_exists(request: Request, document_service, session_manager):
"""Delete all documents with a specific filename""" """Check if a document with a specific filename already exists"""
data = await request.json() filename = request.query_params.get("filename")
filename = data.get("filename")
if not filename: if not filename:
return JSONResponse({"error": "filename is required"}, status_code=400) return JSONResponse({"error": "filename parameter is required"}, status_code=400)
user = request.state.user user = request.state.user
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token) jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
@ -22,34 +21,79 @@ async def delete_documents_by_filename(request: Request, document_service, sessi
opensearch_client = session_manager.get_user_opensearch_client( opensearch_client = session_manager.get_user_opensearch_client(
user.user_id, jwt_token user.user_id, jwt_token
) )
# Search for any document with this exact filename
from utils.opensearch_queries import build_filename_search_body
search_body = build_filename_search_body(filename, size=1, source=["filename"])
logger.debug(f"Checking filename existence: {filename}")
response = await opensearch_client.search(
index=INDEX_NAME,
body=search_body
)
# Check if any hits were found
hits = response.get("hits", {}).get("hits", [])
exists = len(hits) > 0
logger.debug(f"Filename check result - exists: {exists}, hits: {len(hits)}")
return JSONResponse({
"exists": exists,
"filename": filename
}, status_code=200)
except Exception as e:
logger.error("Error checking filename existence", filename=filename, error=str(e))
error_str = str(e)
if "AuthenticationException" in error_str:
return JSONResponse({"error": "Access denied: insufficient permissions"}, status_code=403)
else:
return JSONResponse({"error": str(e)}, status_code=500)
async def delete_documents_by_filename(request: Request, document_service, session_manager):
"""Delete all documents with a specific filename"""
data = await request.json()
filename = data.get("filename")
if not filename:
return JSONResponse({"error": "filename is required"}, status_code=400)
user = request.state.user
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
try:
# Get user's OpenSearch client
opensearch_client = session_manager.get_user_opensearch_client(
user.user_id, jwt_token
)
# Delete by query to remove all chunks of this document # Delete by query to remove all chunks of this document
delete_query = { from utils.opensearch_queries import build_filename_delete_body
"query": {
"bool": { delete_query = build_filename_delete_body(filename)
"must": [
{"term": {"filename": filename}} logger.debug(f"Deleting documents with filename: {filename}")
]
}
}
}
result = await opensearch_client.delete_by_query( result = await opensearch_client.delete_by_query(
index=INDEX_NAME, index=INDEX_NAME,
body=delete_query, body=delete_query,
conflicts="proceed" conflicts="proceed"
) )
deleted_count = result.get("deleted", 0) deleted_count = result.get("deleted", 0)
logger.info(f"Deleted {deleted_count} chunks for filename {filename}", user_id=user.user_id) logger.info(f"Deleted {deleted_count} chunks for filename {filename}", user_id=user.user_id)
return JSONResponse({ return JSONResponse({
"success": True, "success": True,
"deleted_chunks": deleted_count, "deleted_chunks": deleted_count,
"filename": filename, "filename": filename,
"message": f"All documents with filename '{filename}' deleted successfully" "message": f"All documents with filename '{filename}' deleted successfully"
}, status_code=200) }, status_code=200)
except Exception as e: except Exception as e:
logger.error("Error deleting documents by filename", filename=filename, error=str(e)) logger.error("Error deleting documents by filename", filename=filename, error=str(e))
error_str = str(e) error_str = str(e)

15
src/api/langflow_files.py
View file

@ -189,19 +189,20 @@ async def upload_and_ingest_user_file(
# Create temporary file for task processing # Create temporary file for task processing
import tempfile import tempfile
import os import os
# Read file content # Read file content
content = await upload_file.read() content = await upload_file.read()
# Create temporary file # Create temporary file with the actual filename (not a temp prefix)
# Store in temp directory but use the real filename
temp_dir = tempfile.gettempdir()
safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_") safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_")
temp_fd, temp_path = tempfile.mkstemp( temp_path = os.path.join(temp_dir, safe_filename)
suffix=f"_{safe_filename}"
)
try: try:
# Write content to temp file # Write content to temp file
with os.fdopen(temp_fd, 'wb') as temp_file: with open(temp_path, 'wb') as temp_file:
temp_file.write(content) temp_file.write(content)
logger.debug("Created temporary file for task processing", temp_path=temp_path) logger.debug("Created temporary file for task processing", temp_path=temp_path)

106
src/api/router.py
View file

@ -13,27 +13,27 @@ logger = get_logger(__name__)
async def upload_ingest_router( async def upload_ingest_router(
request: Request, request: Request,
document_service=None, document_service=None,
langflow_file_service=None, langflow_file_service=None,
session_manager=None, session_manager=None,
task_service=None task_service=None,
): ):
""" """
Router endpoint that automatically routes upload requests based on configuration. Router endpoint that automatically routes upload requests based on configuration.
- If DISABLE_INGEST_WITH_LANGFLOW is True: uses traditional OpenRAG upload (/upload) - If DISABLE_INGEST_WITH_LANGFLOW is True: uses traditional OpenRAG upload (/upload)
- If DISABLE_INGEST_WITH_LANGFLOW is False (default): uses Langflow upload-ingest via task service - If DISABLE_INGEST_WITH_LANGFLOW is False (default): uses Langflow upload-ingest via task service
This provides a single endpoint that users can call regardless of backend configuration. This provides a single endpoint that users can call regardless of backend configuration.
All langflow uploads are processed as background tasks for better scalability. All langflow uploads are processed as background tasks for better scalability.
""" """
try: try:
logger.debug( logger.debug(
"Router upload_ingest endpoint called", "Router upload_ingest endpoint called",
disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,
) )
# Route based on configuration # Route based on configuration
if DISABLE_INGEST_WITH_LANGFLOW: if DISABLE_INGEST_WITH_LANGFLOW:
# Route to traditional OpenRAG upload # Route to traditional OpenRAG upload
@ -42,8 +42,10 @@ async def upload_ingest_router(
else: else:
# Route to Langflow upload and ingest using task service # Route to Langflow upload and ingest using task service
logger.debug("Routing to Langflow upload-ingest pipeline via task service") logger.debug("Routing to Langflow upload-ingest pipeline via task service")
return await langflow_upload_ingest_task(request, langflow_file_service, session_manager, task_service) return await langflow_upload_ingest_task(
request, langflow_file_service, session_manager, task_service
)
except Exception as e: except Exception as e:
logger.error("Error in upload_ingest_router", error=str(e)) logger.error("Error in upload_ingest_router", error=str(e))
error_msg = str(e) error_msg = str(e)
@ -57,17 +59,14 @@ async def upload_ingest_router(
async def langflow_upload_ingest_task( async def langflow_upload_ingest_task(
request: Request, request: Request, langflow_file_service, session_manager, task_service
langflow_file_service,
session_manager,
task_service
): ):
"""Task-based langflow upload and ingest for single/multiple files""" """Task-based langflow upload and ingest for single/multiple files"""
try: try:
logger.debug("Task-based langflow upload_ingest endpoint called") logger.debug("Task-based langflow upload_ingest endpoint called")
form = await request.form() form = await request.form()
upload_files = form.getlist("file") upload_files = form.getlist("file")
if not upload_files or len(upload_files) == 0: if not upload_files or len(upload_files) == 0:
logger.error("No files provided in task-based upload request") logger.error("No files provided in task-based upload request")
return JSONResponse({"error": "Missing files"}, status_code=400) return JSONResponse({"error": "Missing files"}, status_code=400)
@ -77,14 +76,16 @@ async def langflow_upload_ingest_task(
settings_json = form.get("settings") settings_json = form.get("settings")
tweaks_json = form.get("tweaks") tweaks_json = form.get("tweaks")
delete_after_ingest = form.get("delete_after_ingest", "true").lower() == "true" delete_after_ingest = form.get("delete_after_ingest", "true").lower() == "true"
replace_duplicates = form.get("replace_duplicates", "false").lower() == "true"
# Parse JSON fields if provided # Parse JSON fields if provided
settings = None settings = None
tweaks = None tweaks = None
if settings_json: if settings_json:
try: try:
import json import json
settings = json.loads(settings_json) settings = json.loads(settings_json)
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
logger.error("Invalid settings JSON", error=str(e)) logger.error("Invalid settings JSON", error=str(e))
@ -93,6 +94,7 @@ async def langflow_upload_ingest_task(
if tweaks_json: if tweaks_json:
try: try:
import json import json
tweaks = json.loads(tweaks_json) tweaks = json.loads(tweaks_json)
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
logger.error("Invalid tweaks JSON", error=str(e)) logger.error("Invalid tweaks JSON", error=str(e))
@ -106,28 +108,37 @@ async def langflow_upload_ingest_task(
jwt_token = getattr(request.state, "jwt_token", None) jwt_token = getattr(request.state, "jwt_token", None)
if not user_id: if not user_id:
return JSONResponse({"error": "User authentication required"}, status_code=401) return JSONResponse(
{"error": "User authentication required"}, status_code=401
)
# Create temporary files for task processing # Create temporary files for task processing
import tempfile import tempfile
import os import os
temp_file_paths = [] temp_file_paths = []
original_filenames = []
try: try:
# Create temp directory reference once
temp_dir = tempfile.gettempdir()
for upload_file in upload_files: for upload_file in upload_files:
# Read file content # Read file content
content = await upload_file.read() content = await upload_file.read()
# Create temporary file # Store ORIGINAL filename (not transformed)
original_filenames.append(upload_file.filename)
# Create temporary file with TRANSFORMED filename for filesystem safety
# Transform: spaces and / to underscore
safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_") safe_filename = upload_file.filename.replace(" ", "_").replace("/", "_")
temp_fd, temp_path = tempfile.mkstemp( temp_path = os.path.join(temp_dir, safe_filename)
suffix=f"_{safe_filename}"
)
# Write content to temp file # Write content to temp file
with os.fdopen(temp_fd, 'wb') as temp_file: with open(temp_path, "wb") as temp_file:
temp_file.write(content) temp_file.write(content)
temp_file_paths.append(temp_path) temp_file_paths.append(temp_path)
logger.debug( logger.debug(
@ -136,21 +147,22 @@ async def langflow_upload_ingest_task(
user_id=user_id, user_id=user_id,
has_settings=bool(settings), has_settings=bool(settings),
has_tweaks=bool(tweaks), has_tweaks=bool(tweaks),
delete_after_ingest=delete_after_ingest delete_after_ingest=delete_after_ingest,
) )
# Create langflow upload task # Create langflow upload task
print(f"tweaks: {tweaks}") logger.debug(
print(f"settings: {settings}") f"Preparing to create langflow upload task: tweaks={tweaks}, settings={settings}, jwt_token={jwt_token}, user_name={user_name}, user_email={user_email}, session_id={session_id}, delete_after_ingest={delete_after_ingest}, temp_file_paths={temp_file_paths}",
print(f"jwt_token: {jwt_token}") )
print(f"user_name: {user_name}") # Create a map between temp_file_paths and original_filenames
print(f"user_email: {user_email}") file_path_to_original_filename = dict(zip(temp_file_paths, original_filenames))
print(f"session_id: {session_id}") logger.debug(
print(f"delete_after_ingest: {delete_after_ingest}") f"File path to original filename map: {file_path_to_original_filename}",
print(f"temp_file_paths: {temp_file_paths}") )
task_id = await task_service.create_langflow_upload_task( task_id = await task_service.create_langflow_upload_task(
user_id=user_id, user_id=user_id,
file_paths=temp_file_paths, file_paths=temp_file_paths,
original_filenames=file_path_to_original_filename,
langflow_file_service=langflow_file_service, langflow_file_service=langflow_file_service,
session_manager=session_manager, session_manager=session_manager,
jwt_token=jwt_token, jwt_token=jwt_token,
@ -160,23 +172,28 @@ async def langflow_upload_ingest_task(
tweaks=tweaks, tweaks=tweaks,
settings=settings, settings=settings,
delete_after_ingest=delete_after_ingest, delete_after_ingest=delete_after_ingest,
replace_duplicates=replace_duplicates,
) )
logger.debug("Langflow upload task created successfully", task_id=task_id) logger.debug("Langflow upload task created successfully", task_id=task_id)
return JSONResponse({ return JSONResponse(
"task_id": task_id, {
"message": f"Langflow upload task created for {len(upload_files)} file(s)", "task_id": task_id,
"file_count": len(upload_files) "message": f"Langflow upload task created for {len(upload_files)} file(s)",
}, status_code=202) # 202 Accepted for async processing "file_count": len(upload_files),
},
status_code=202,
) # 202 Accepted for async processing
except Exception: except Exception:
# Clean up temp files on error # Clean up temp files on error
from utils.file_utils import safe_unlink from utils.file_utils import safe_unlink
for temp_path in temp_file_paths: for temp_path in temp_file_paths:
safe_unlink(temp_path) safe_unlink(temp_path)
raise raise
except Exception as e: except Exception as e:
logger.error( logger.error(
"Task-based langflow upload_ingest endpoint failed", "Task-based langflow upload_ingest endpoint failed",
@ -184,5 +201,6 @@ async def langflow_upload_ingest_task(
error=str(e), error=str(e),
) )
import traceback import traceback
logger.error("Full traceback", traceback=traceback.format_exc()) logger.error("Full traceback", traceback=traceback.format_exc())
return JSONResponse({"error": str(e)}, status_code=500) return JSONResponse({"error": str(e)}, status_code=500)

2
src/config/config_manager.py
View file

@ -27,7 +27,7 @@ class KnowledgeConfig:
embedding_model: str = "text-embedding-3-small" embedding_model: str = "text-embedding-3-small"
chunk_size: int = 1000 chunk_size: int = 1000
chunk_overlap: int = 200 chunk_overlap: int = 200
table_structure: bool = False table_structure: bool = True
ocr: bool = False ocr: bool = False
picture_descriptions: bool = False picture_descriptions: bool = False

1
src/config/settings.py
View file

@ -34,6 +34,7 @@ _legacy_flow_id = os.getenv("FLOW_ID")
LANGFLOW_CHAT_FLOW_ID = os.getenv("LANGFLOW_CHAT_FLOW_ID") or _legacy_flow_id LANGFLOW_CHAT_FLOW_ID = os.getenv("LANGFLOW_CHAT_FLOW_ID") or _legacy_flow_id
LANGFLOW_INGEST_FLOW_ID = os.getenv("LANGFLOW_INGEST_FLOW_ID") LANGFLOW_INGEST_FLOW_ID = os.getenv("LANGFLOW_INGEST_FLOW_ID")
LANGFLOW_URL_INGEST_FLOW_ID = os.getenv("LANGFLOW_URL_INGEST_FLOW_ID")
NUDGES_FLOW_ID = os.getenv("NUDGES_FLOW_ID") NUDGES_FLOW_ID = os.getenv("NUDGES_FLOW_ID")
if _legacy_flow_id and not os.getenv("LANGFLOW_CHAT_FLOW_ID"): if _legacy_flow_id and not os.getenv("LANGFLOW_CHAT_FLOW_ID"):

11
src/main.py
View file

@ -953,6 +953,17 @@ async def create_app():
methods=["POST", "GET"], methods=["POST", "GET"],
), ),
# Document endpoints # Document endpoints
Route(
"/documents/check-filename",
require_auth(services["session_manager"])(
partial(
documents.check_filename_exists,
document_service=services["document_service"],
session_manager=services["session_manager"],
)
),
methods=["GET"],
),
Route( Route(
"/documents/delete-by-filename", "/documents/delete-by-filename",
require_auth(services["session_manager"])( require_auth(services["session_manager"])(

131
src/models/processors.py
View file

@ -55,6 +55,96 @@ class TaskProcessor:
await asyncio.sleep(retry_delay) await asyncio.sleep(retry_delay)
retry_delay *= 2 # Exponential backoff retry_delay *= 2 # Exponential backoff
async def check_filename_exists(
self,
filename: str,
opensearch_client,
) -> bool:
"""
Check if a document with the given filename already exists in OpenSearch.
Returns True if any chunks with this filename exist.
"""
from config.settings import INDEX_NAME
from utils.opensearch_queries import build_filename_search_body
import asyncio
max_retries = 3
retry_delay = 1.0
for attempt in range(max_retries):
try:
# Search for any document with this exact filename
search_body = build_filename_search_body(filename, size=1, source=False)
response = await opensearch_client.search(
index=INDEX_NAME,
body=search_body
)
# Check if any hits were found
hits = response.get("hits", {}).get("hits", [])
return len(hits) > 0
except (asyncio.TimeoutError, Exception) as e:
if attempt == max_retries - 1:
logger.error(
"OpenSearch filename check failed after retries",
filename=filename,
error=str(e),
attempt=attempt + 1
)
# On final failure, assume document doesn't exist (safer to reprocess than skip)
logger.warning(
"Assuming filename doesn't exist due to connection issues",
filename=filename
)
return False
else:
logger.warning(
"OpenSearch filename check failed, retrying",
filename=filename,
error=str(e),
attempt=attempt + 1,
retry_in=retry_delay
)
await asyncio.sleep(retry_delay)
retry_delay *= 2 # Exponential backoff
async def delete_document_by_filename(
self,
filename: str,
opensearch_client,
) -> None:
"""
Delete all chunks of a document with the given filename from OpenSearch.
"""
from config.settings import INDEX_NAME
from utils.opensearch_queries import build_filename_delete_body
try:
# Delete all documents with this filename
delete_body = build_filename_delete_body(filename)
response = await opensearch_client.delete_by_query(
index=INDEX_NAME,
body=delete_body
)
deleted_count = response.get("deleted", 0)
logger.info(
"Deleted existing document chunks",
filename=filename,
deleted_count=deleted_count
)
except Exception as e:
logger.error(
"Failed to delete existing document",
filename=filename,
error=str(e)
)
raise
async def process_document_standard( async def process_document_standard(
self, self,
file_path: str, file_path: str,
@ -527,6 +617,7 @@ class LangflowFileProcessor(TaskProcessor):
tweaks: dict = None, tweaks: dict = None,
settings: dict = None, settings: dict = None,
delete_after_ingest: bool = True, delete_after_ingest: bool = True,
replace_duplicates: bool = False,
): ):
super().__init__() super().__init__()
self.langflow_file_service = langflow_file_service self.langflow_file_service = langflow_file_service
@ -539,6 +630,7 @@ class LangflowFileProcessor(TaskProcessor):
self.tweaks = tweaks or {} self.tweaks = tweaks or {}
self.settings = settings self.settings = settings
self.delete_after_ingest = delete_after_ingest self.delete_after_ingest = delete_after_ingest
self.replace_duplicates = replace_duplicates
async def process_item( async def process_item(
self, upload_task: UploadTask, item: str, file_task: FileTask self, upload_task: UploadTask, item: str, file_task: FileTask
@ -554,37 +646,40 @@ class LangflowFileProcessor(TaskProcessor):
file_task.updated_at = time.time() file_task.updated_at = time.time()
try: try:
# Compute hash and check if already exists # Use the ORIGINAL filename stored in file_task (not the transformed temp path)
from utils.hash_utils import hash_id # This ensures we check/store the original filename with spaces, etc.
file_hash = hash_id(item) original_filename = file_task.filename or os.path.basename(item)
# Check if document already exists # Check if document with same filename already exists
opensearch_client = self.session_manager.get_user_opensearch_client( opensearch_client = self.session_manager.get_user_opensearch_client(
self.owner_user_id, self.jwt_token self.owner_user_id, self.jwt_token
) )
if await self.check_document_exists(file_hash, opensearch_client):
file_task.status = TaskStatus.COMPLETED filename_exists = await self.check_filename_exists(original_filename, opensearch_client)
file_task.result = {"status": "unchanged", "id": file_hash}
if filename_exists and not self.replace_duplicates:
# Duplicate exists and user hasn't confirmed replacement
file_task.status = TaskStatus.FAILED
file_task.error = f"File with name '{original_filename}' already exists"
file_task.updated_at = time.time() file_task.updated_at = time.time()
upload_task.successful_files += 1 upload_task.failed_files += 1
return return
elif filename_exists and self.replace_duplicates:
# Delete existing document before uploading new one
logger.info(f"Replacing existing document: {original_filename}")
await self.delete_document_by_filename(original_filename, opensearch_client)
# Read file content for processing # Read file content for processing
with open(item, 'rb') as f: with open(item, 'rb') as f:
content = f.read() content = f.read()
# Create file tuple for upload # Create file tuple for upload using ORIGINAL filename
temp_filename = os.path.basename(item) # This ensures the document is indexed with the original name
# Extract original filename from temp file suffix (remove tmp prefix) content_type, _ = mimetypes.guess_type(original_filename)
if "_" in temp_filename:
filename = temp_filename.split("_", 1)[1] # Get everything after first _
else:
filename = temp_filename
content_type, _ = mimetypes.guess_type(filename)
if not content_type: if not content_type:
content_type = 'application/octet-stream' content_type = 'application/octet-stream'
file_tuple = (filename, content, content_type) file_tuple = (original_filename, content, content_type)
# Get JWT token using same logic as DocumentFileProcessor # Get JWT token using same logic as DocumentFileProcessor
# This will handle anonymous JWT creation if needed # This will handle anonymous JWT creation if needed

3
src/models/tasks.py
View file

@ -20,7 +20,8 @@ class FileTask:
retry_count: int = 0 retry_count: int = 0
created_at: float = field(default_factory=time.time) created_at: float = field(default_factory=time.time)
updated_at: float = field(default_factory=time.time) updated_at: float = field(default_factory=time.time)
filename: Optional[str] = None # Original filename for display
@property @property
def duration_seconds(self) -> float: def duration_seconds(self) -> float:
"""Duration in seconds from creation to last update""" """Duration in seconds from creation to last update"""

16
src/services/flows_service.py
View file

@ -1,5 +1,6 @@
from config.settings import ( from config.settings import (
DISABLE_INGEST_WITH_LANGFLOW, DISABLE_INGEST_WITH_LANGFLOW,
LANGFLOW_URL_INGEST_FLOW_ID,
NUDGES_FLOW_ID, NUDGES_FLOW_ID,
LANGFLOW_URL, LANGFLOW_URL,
LANGFLOW_CHAT_FLOW_ID, LANGFLOW_CHAT_FLOW_ID,
@ -116,9 +117,11 @@ class FlowsService:
flow_id = LANGFLOW_CHAT_FLOW_ID flow_id = LANGFLOW_CHAT_FLOW_ID
elif flow_type == "ingest": elif flow_type == "ingest":
flow_id = LANGFLOW_INGEST_FLOW_ID flow_id = LANGFLOW_INGEST_FLOW_ID
elif flow_type == "url_ingest":
flow_id = LANGFLOW_URL_INGEST_FLOW_ID
else: else:
raise ValueError( raise ValueError(
"flow_type must be either 'nudges', 'retrieval', or 'ingest'" "flow_type must be either 'nudges', 'retrieval', 'ingest', or 'url_ingest'"
) )
if not flow_id: if not flow_id:
@ -291,6 +294,13 @@ class FlowsService:
"llm_name": None, # Ingestion flow might not have LLM "llm_name": None, # Ingestion flow might not have LLM
"llm_text_name": None, "llm_text_name": None,
}, },
{
"name": "url_ingest",
"flow_id": LANGFLOW_URL_INGEST_FLOW_ID,
"embedding_name": OPENAI_EMBEDDING_COMPONENT_DISPLAY_NAME,
"llm_name": None,
"llm_text_name": None,
},
] ]
results = [] results = []
@ -716,6 +726,10 @@ class FlowsService:
"name": "ingest", "name": "ingest",
"flow_id": LANGFLOW_INGEST_FLOW_ID, "flow_id": LANGFLOW_INGEST_FLOW_ID,
}, },
{
"name": "url_ingest",
"flow_id": LANGFLOW_URL_INGEST_FLOW_ID,
},
] ]
# Determine target component IDs based on provider # Determine target component IDs based on provider

12
src/services/langflow_file_service.py
View file

@ -67,6 +67,7 @@ class LangflowFileService:
owner_name: Optional[str] = None, owner_name: Optional[str] = None,
owner_email: Optional[str] = None, owner_email: Optional[str] = None,
connector_type: Optional[str] = None, connector_type: Optional[str] = None,
file_tuples: Optional[list[tuple[str, str, str]]] = None,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
""" """
Trigger the ingestion flow with provided file paths. Trigger the ingestion flow with provided file paths.
@ -86,7 +87,9 @@ class LangflowFileService:
# Pass files via tweaks to File component (File-PSU37 from the flow) # Pass files via tweaks to File component (File-PSU37 from the flow)
if file_paths: if file_paths:
tweaks["File-PSU37"] = {"path": file_paths} tweaks["DoclingRemote-Dp3PX"] = {"path": file_paths}
# Pass JWT token via tweaks using the x-langflow-global-var- pattern # Pass JWT token via tweaks using the x-langflow-global-var- pattern
if jwt_token: if jwt_token:
@ -129,7 +132,8 @@ class LangflowFileService:
list(tweaks.keys()) if isinstance(tweaks, dict) else None, list(tweaks.keys()) if isinstance(tweaks, dict) else None,
bool(jwt_token), bool(jwt_token),
) )
# To compute the file size in bytes, use len() on the file content (which should be bytes)
file_size_bytes = len(file_tuples[0][1]) if file_tuples and len(file_tuples[0]) > 1 else 0
# Avoid logging full payload to prevent leaking sensitive data (e.g., JWT) # Avoid logging full payload to prevent leaking sensitive data (e.g., JWT)
headers={ headers={
"X-Langflow-Global-Var-JWT": str(jwt_token), "X-Langflow-Global-Var-JWT": str(jwt_token),
@ -137,6 +141,9 @@ class LangflowFileService:
"X-Langflow-Global-Var-OWNER_NAME": str(owner_name), "X-Langflow-Global-Var-OWNER_NAME": str(owner_name),
"X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email), "X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email),
"X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type), "X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),
"X-Langflow-Global-Var-FILENAME": str(file_tuples[0][0]),
"X-Langflow-Global-Var-MIMETYPE": str(file_tuples[0][2]),
"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),
} }
logger.info(f"[LF] Headers {headers}") logger.info(f"[LF] Headers {headers}")
logger.info(f"[LF] Payload {payload}") logger.info(f"[LF] Payload {payload}")
@ -271,6 +278,7 @@ class LangflowFileService:
owner_name=owner_name, owner_name=owner_name,
owner_email=owner_email, owner_email=owner_email,
connector_type=connector_type, connector_type=connector_type,
file_tuples=[file_tuple],
) )
logger.debug("[LF] Ingestion completed successfully") logger.debug("[LF] Ingestion completed successfully")
except Exception as e: except Exception as e:

28
src/services/task_service.py
View file

@ -1,6 +1,5 @@
import asyncio import asyncio
import random import random
from typing import Dict, Optional
import time import time
import uuid import uuid
@ -59,6 +58,7 @@ class TaskService:
file_paths: list, file_paths: list,
langflow_file_service, langflow_file_service,
session_manager, session_manager,
original_filenames: dict | None = None,
jwt_token: str = None, jwt_token: str = None,
owner_name: str = None, owner_name: str = None,
owner_email: str = None, owner_email: str = None,
@ -66,6 +66,7 @@ class TaskService:
tweaks: dict = None, tweaks: dict = None,
settings: dict = None, settings: dict = None,
delete_after_ingest: bool = True, delete_after_ingest: bool = True,
replace_duplicates: bool = False,
) -> str: ) -> str:
"""Create a new upload task for Langflow file processing with upload and ingest""" """Create a new upload task for Langflow file processing with upload and ingest"""
# Use LangflowFileProcessor with user context # Use LangflowFileProcessor with user context
@ -82,18 +83,35 @@ class TaskService:
tweaks=tweaks, tweaks=tweaks,
settings=settings, settings=settings,
delete_after_ingest=delete_after_ingest, delete_after_ingest=delete_after_ingest,
replace_duplicates=replace_duplicates,
) )
return await self.create_custom_task(user_id, file_paths, processor) return await self.create_custom_task(user_id, file_paths, processor, original_filenames)
async def create_custom_task(self, user_id: str, items: list, processor) -> str: async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: dict | None = None) -> str:
"""Create a new task with custom processor for any type of items""" """Create a new task with custom processor for any type of items"""
import os
# Store anonymous tasks under a stable key so they can be retrieved later # Store anonymous tasks under a stable key so they can be retrieved later
store_user_id = user_id or AnonymousUser().user_id store_user_id = user_id or AnonymousUser().user_id
task_id = str(uuid.uuid4()) task_id = str(uuid.uuid4())
# Create file tasks with original filenames if provided
normalized_originals = (
{str(k): v for k, v in original_filenames.items()} if original_filenames else {}
)
file_tasks = {
str(item): FileTask(
file_path=str(item),
filename=normalized_originals.get(
str(item), os.path.basename(str(item))
),
)
for item in items
}
upload_task = UploadTask( upload_task = UploadTask(
task_id=task_id, task_id=task_id,
total_files=len(items), total_files=len(items),
file_tasks={str(item): FileTask(file_path=str(item)) for item in items}, file_tasks=file_tasks,
) )
# Attach the custom processor to the task # Attach the custom processor to the task
@ -268,6 +286,7 @@ class TaskService:
"created_at": file_task.created_at, "created_at": file_task.created_at,
"updated_at": file_task.updated_at, "updated_at": file_task.updated_at,
"duration_seconds": file_task.duration_seconds, "duration_seconds": file_task.duration_seconds,
"filename": file_task.filename,
} }
# Count running and pending files # Count running and pending files
@ -322,6 +341,7 @@ class TaskService:
"created_at": file_task.created_at, "created_at": file_task.created_at,
"updated_at": file_task.updated_at, "updated_at": file_task.updated_at,
"duration_seconds": file_task.duration_seconds, "duration_seconds": file_task.duration_seconds,
"filename": file_task.filename,
} }
if file_task.status.value == "running": if file_task.status.value == "running":

14
src/tui/_assets/docker-compose-cpu.yml
View file

@ -55,6 +55,7 @@ services:
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
- LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID}
- LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID}
- LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID}
- DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false} - DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false}
- NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID}
- OPENSEARCH_PORT=9200 - OPENSEARCH_PORT=9200
@ -99,15 +100,22 @@ services:
- OPENAI_API_KEY=${OPENAI_API_KEY} - OPENAI_API_KEY=${OPENAI_API_KEY}
- LANGFLOW_LOAD_FLOWS_PATH=/app/flows - LANGFLOW_LOAD_FLOWS_PATH=/app/flows
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY} - LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
- JWT="dummy" - JWT=None
- OWNER=None
- OWNER_NAME=None
- OWNER_EMAIL=None
- CONNECTOR_TYPE=system
- OPENRAG-QUERY-FILTER="{}" - OPENRAG-QUERY-FILTER="{}"
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD - FILENAME=None
- MIMETYPE=None
- FILESIZE=0
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE
- LANGFLOW_LOG_LEVEL=DEBUG - LANGFLOW_LOG_LEVEL=DEBUG
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
- LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE} - LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE}
- LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI} - LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI}
- DEFAULT_FOLDER_NAME="OpenRAG" # - DEFAULT_FOLDER_NAME=OpenRAG
- HIDE_GETTING_STARTED_PROGRESS=true - HIDE_GETTING_STARTED_PROGRESS=true

14
src/tui/_assets/docker-compose.yml
View file

@ -54,6 +54,7 @@ services:
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
- LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID} - LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID}
- LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID} - LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID}
- LANGFLOW_URL_INGEST_FLOW_ID=${LANGFLOW_URL_INGEST_FLOW_ID}
- DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false} - DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false}
- NUDGES_FLOW_ID=${NUDGES_FLOW_ID} - NUDGES_FLOW_ID=${NUDGES_FLOW_ID}
- OPENSEARCH_PORT=9200 - OPENSEARCH_PORT=9200
@ -99,15 +100,22 @@ services:
- OPENAI_API_KEY=${OPENAI_API_KEY} - OPENAI_API_KEY=${OPENAI_API_KEY}
- LANGFLOW_LOAD_FLOWS_PATH=/app/flows - LANGFLOW_LOAD_FLOWS_PATH=/app/flows
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY} - LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
- JWT="dummy" - JWT=None
- OWNER=None
- OWNER_NAME=None
- OWNER_EMAIL=None
- CONNECTOR_TYPE=system
- OPENRAG-QUERY-FILTER="{}" - OPENRAG-QUERY-FILTER="{}"
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD - FILENAME=None
- MIMETYPE=None
- FILESIZE=0
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE
- LANGFLOW_LOG_LEVEL=DEBUG - LANGFLOW_LOG_LEVEL=DEBUG
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
- LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE} - LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE}
- LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI} - LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI}
- DEFAULT_FOLDER_NAME="OpenRAG" # - DEFAULT_FOLDER_NAME="OpenRAG"
- HIDE_GETTING_STARTED_PROGRESS=true - HIDE_GETTING_STARTED_PROGRESS=true

3
src/tui/managers/env_manager.py
View file

@ -33,6 +33,7 @@ class EnvConfig:
langflow_superuser_password: str = "" langflow_superuser_password: str = ""
langflow_chat_flow_id: str = "1098eea1-6649-4e1d-aed1-b77249fb8dd0" langflow_chat_flow_id: str = "1098eea1-6649-4e1d-aed1-b77249fb8dd0"
langflow_ingest_flow_id: str = "5488df7c-b93f-4f87-a446-b67028bc0813" langflow_ingest_flow_id: str = "5488df7c-b93f-4f87-a446-b67028bc0813"
langflow_url_ingest_flow_id: str = "72c3d17c-2dac-4a73-b48a-6518473d7830"
# OAuth settings # OAuth settings
google_oauth_client_id: str = "" google_oauth_client_id: str = ""
@ -114,6 +115,7 @@ class EnvManager:
"LANGFLOW_SUPERUSER_PASSWORD": "langflow_superuser_password", "LANGFLOW_SUPERUSER_PASSWORD": "langflow_superuser_password",
"LANGFLOW_CHAT_FLOW_ID": "langflow_chat_flow_id", "LANGFLOW_CHAT_FLOW_ID": "langflow_chat_flow_id",
"LANGFLOW_INGEST_FLOW_ID": "langflow_ingest_flow_id", "LANGFLOW_INGEST_FLOW_ID": "langflow_ingest_flow_id",
"LANGFLOW_URL_INGEST_FLOW_ID": "langflow_url_ingest_flow_id",
"NUDGES_FLOW_ID": "nudges_flow_id", "NUDGES_FLOW_ID": "nudges_flow_id",
"GOOGLE_OAUTH_CLIENT_ID": "google_oauth_client_id", "GOOGLE_OAUTH_CLIENT_ID": "google_oauth_client_id",
"GOOGLE_OAUTH_CLIENT_SECRET": "google_oauth_client_secret", "GOOGLE_OAUTH_CLIENT_SECRET": "google_oauth_client_secret",
@ -255,6 +257,7 @@ class EnvManager:
f.write( f.write(
f"LANGFLOW_INGEST_FLOW_ID={self._quote_env_value(self.config.langflow_ingest_flow_id)}\n" f"LANGFLOW_INGEST_FLOW_ID={self._quote_env_value(self.config.langflow_ingest_flow_id)}\n"
) )
f.write(f"LANGFLOW_URL_INGEST_FLOW_ID={self._quote_env_value(self.config.langflow_url_ingest_flow_id)}\n")
f.write(f"NUDGES_FLOW_ID={self._quote_env_value(self.config.nudges_flow_id)}\n") f.write(f"NUDGES_FLOW_ID={self._quote_env_value(self.config.nudges_flow_id)}\n")
f.write(f"OPENSEARCH_PASSWORD={self._quote_env_value(self.config.opensearch_password)}\n") f.write(f"OPENSEARCH_PASSWORD={self._quote_env_value(self.config.opensearch_password)}\n")
f.write(f"OPENAI_API_KEY={self._quote_env_value(self.config.openai_api_key)}\n") f.write(f"OPENAI_API_KEY={self._quote_env_value(self.config.openai_api_key)}\n")

55
src/utils/opensearch_queries.py Normal file
View file

@ -0,0 +1,55 @@
"""
Utility functions for constructing OpenSearch queries consistently.
"""
from typing import Union, List
def build_filename_query(filename: str) -> dict:
"""
Build a standardized query for finding documents by filename.
Args:
filename: The exact filename to search for
Returns:
A dict containing the OpenSearch query body
"""
return {
"term": {
"filename": filename
}
}
def build_filename_search_body(filename: str, size: int = 1, source: Union[bool, List[str]] = False) -> dict:
"""
Build a complete search body for checking if a filename exists.
Args:
filename: The exact filename to search for
size: Number of results to return (default: 1)
source: Whether to include source fields, or list of specific fields to include (default: False)
Returns:
A dict containing the complete OpenSearch search body
"""
return {
"query": build_filename_query(filename),
"size": size,
"_source": source
}
def build_filename_delete_body(filename: str) -> dict:
"""
Build a delete-by-query body for removing all documents with a filename.
Args:
filename: The exact filename to delete
Returns:
A dict containing the OpenSearch delete-by-query body
"""
return {
"query": build_filename_query(filename)
}

2
uv.lock generated
View file

@ -2282,7 +2282,7 @@ wheels = [
[[package]] [[package]]
name = "openrag" name = "openrag"
version = "0.1.14.dev2" version = "0.1.14.dev3"
source = { editable = "." } source = { editable = "." }
dependencies = [ dependencies = [
{ name = "agentd" }, { name = "agentd" },