Merge branch 'main' of https://github.com/langflow-ai/openrag into delete-knowledge
This commit is contained in:
commit
9b5a7be8b0
29 changed files with 1616 additions and 1305 deletions
|
|
@ -62,8 +62,7 @@ LANGFLOW_CHAT_FLOW_ID=your_chat_flow_id
|
||||||
LANGFLOW_INGEST_FLOW_ID=your_ingest_flow_id
|
LANGFLOW_INGEST_FLOW_ID=your_ingest_flow_id
|
||||||
NUDGES_FLOW_ID=your_nudges_flow_id
|
NUDGES_FLOW_ID=your_nudges_flow_id
|
||||||
```
|
```
|
||||||
ee extended configuration, including ingestion and optional variables: [docs/configuration.md](docs/
|
See extended configuration, including ingestion and optional variables: [docs/configuration.md](docs/configuration.md)
|
||||||
configuration.md)
|
|
||||||
### 3. Start OpenRAG
|
### 3. Start OpenRAG
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|
|
||||||
|
|
@ -91,7 +91,7 @@ services:
|
||||||
langflow:
|
langflow:
|
||||||
volumes:
|
volumes:
|
||||||
- ./flows:/app/flows:Z
|
- ./flows:/app/flows:Z
|
||||||
image: phact/langflow:${LANGFLOW_VERSION:-responses}
|
image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest}
|
||||||
container_name: langflow
|
container_name: langflow
|
||||||
ports:
|
ports:
|
||||||
- "7860:7860"
|
- "7860:7860"
|
||||||
|
|
|
||||||
|
|
@ -91,7 +91,7 @@ services:
|
||||||
langflow:
|
langflow:
|
||||||
volumes:
|
volumes:
|
||||||
- ./flows:/app/flows:Z
|
- ./flows:/app/flows:Z
|
||||||
image: phact/langflow:${LANGFLOW_VERSION:-responses}
|
image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest}
|
||||||
container_name: langflow
|
container_name: langflow
|
||||||
ports:
|
ports:
|
||||||
- "7860:7860"
|
- "7860:7860"
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { EllipsisVertical } from "lucide-react";
|
|
||||||
import { useState } from "react";
|
import { useState } from "react";
|
||||||
import {
|
import {
|
||||||
DropdownMenu,
|
DropdownMenu,
|
||||||
|
|
@ -8,6 +7,7 @@ import {
|
||||||
DropdownMenuItem,
|
DropdownMenuItem,
|
||||||
DropdownMenuTrigger,
|
DropdownMenuTrigger,
|
||||||
} from "@/components/ui/dropdown-menu";
|
} from "@/components/ui/dropdown-menu";
|
||||||
|
import { EllipsisVertical } from "lucide-react";
|
||||||
import { Button } from "./ui/button";
|
import { Button } from "./ui/button";
|
||||||
import { DeleteConfirmationDialog } from "./confirmation-dialog";
|
import { DeleteConfirmationDialog } from "./confirmation-dialog";
|
||||||
import { useDeleteDocument } from "@/app/api/mutations/useDeleteDocument";
|
import { useDeleteDocument } from "@/app/api/mutations/useDeleteDocument";
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,7 @@ export const MarkdownRenderer = ({ chatMessage }: MarkdownRendererProps) => {
|
||||||
<Markdown
|
<Markdown
|
||||||
remarkPlugins={[remarkGfm]}
|
remarkPlugins={[remarkGfm]}
|
||||||
rehypePlugins={[rehypeMathjax, rehypeRaw]}
|
rehypePlugins={[rehypeMathjax, rehypeRaw]}
|
||||||
linkTarget="_blank"
|
urlTransform={(url) => url}
|
||||||
components={{
|
components={{
|
||||||
p({ node, ...props }) {
|
p({ node, ...props }) {
|
||||||
return <p className="w-fit max-w-full">{props.children}</p>;
|
return <p className="w-fit max-w-full">{props.children}</p>;
|
||||||
|
|
@ -79,7 +79,7 @@ export const MarkdownRenderer = ({ chatMessage }: MarkdownRendererProps) => {
|
||||||
h3({ node, ...props }) {
|
h3({ node, ...props }) {
|
||||||
return <h3 className="mb-2 mt-4">{props.children}</h3>;
|
return <h3 className="mb-2 mt-4">{props.children}</h3>;
|
||||||
},
|
},
|
||||||
hr({ node, ...props }) {
|
hr() {
|
||||||
return <hr className="w-full mt-4 mb-8" />;
|
return <hr className="w-full mt-4 mb-8" />;
|
||||||
},
|
},
|
||||||
ul({ node, ...props }) {
|
ul({ node, ...props }) {
|
||||||
|
|
@ -97,8 +97,12 @@ export const MarkdownRenderer = ({ chatMessage }: MarkdownRendererProps) => {
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
a({ node, ...props }) {
|
||||||
|
return <a {...props} target="_blank" rel="noopener noreferrer">{props.children}</a>;
|
||||||
|
},
|
||||||
|
|
||||||
code: ({ node, className, inline, children, ...props }) => {
|
code(props) {
|
||||||
|
const { children, className, ...rest } = props;
|
||||||
let content = children as string;
|
let content = children as string;
|
||||||
if (
|
if (
|
||||||
Array.isArray(children) &&
|
Array.isArray(children) &&
|
||||||
|
|
@ -120,14 +124,15 @@ export const MarkdownRenderer = ({ chatMessage }: MarkdownRendererProps) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
const match = /language-(\w+)/.exec(className || "");
|
const match = /language-(\w+)/.exec(className || "");
|
||||||
|
const isInline = !className?.startsWith("language-");
|
||||||
|
|
||||||
return !inline ? (
|
return !isInline ? (
|
||||||
<CodeComponent
|
<CodeComponent
|
||||||
language={(match && match[1]) || ""}
|
language={(match && match[1]) || ""}
|
||||||
code={String(content).replace(/\n$/, "")}
|
code={String(content).replace(/\n$/, "")}
|
||||||
/>
|
/>
|
||||||
) : (
|
) : (
|
||||||
<code className={className} {...props}>
|
<code className={className} {...rest}>
|
||||||
{content}
|
{content}
|
||||||
</code>
|
</code>
|
||||||
);
|
);
|
||||||
|
|
|
||||||
2240
frontend/package-lock.json
generated
2240
frontend/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -38,11 +38,11 @@
|
||||||
"react": "^19.0.0",
|
"react": "^19.0.0",
|
||||||
"react-dom": "^19.0.0",
|
"react-dom": "^19.0.0",
|
||||||
"react-icons": "^5.5.0",
|
"react-icons": "^5.5.0",
|
||||||
"react-markdown": "^8.0.7",
|
"react-markdown": "^10.1.0",
|
||||||
"react-syntax-highlighter": "^15.6.1",
|
"react-syntax-highlighter": "^15.6.1",
|
||||||
"rehype-mathjax": "^4.0.3",
|
"rehype-mathjax": "^7.1.0",
|
||||||
"rehype-raw": "^6.1.1",
|
"rehype-raw": "^7.0.0",
|
||||||
"remark-gfm": "3.0.1",
|
"remark-gfm": "^4.0.1",
|
||||||
"sonner": "^2.0.6",
|
"sonner": "^2.0.6",
|
||||||
"tailwind-merge": "^3.3.1",
|
"tailwind-merge": "^3.3.1",
|
||||||
"tailwindcss-animate": "^1.0.7",
|
"tailwindcss-animate": "^1.0.7",
|
||||||
|
|
@ -53,6 +53,7 @@
|
||||||
"@types/node": "^20",
|
"@types/node": "^20",
|
||||||
"@types/react": "^19",
|
"@types/react": "^19",
|
||||||
"@types/react-dom": "^19",
|
"@types/react-dom": "^19",
|
||||||
|
"@types/react-syntax-highlighter": "^15.5.13",
|
||||||
"autoprefixer": "^10.4.21",
|
"autoprefixer": "^10.4.21",
|
||||||
"eslint": "^9",
|
"eslint": "^9",
|
||||||
"eslint-config-next": "15.3.5",
|
"eslint-config-next": "15.3.5",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[project]
|
[project]
|
||||||
name = "openrag"
|
name = "openrag"
|
||||||
version = "0.1.4"
|
version = "0.1.8"
|
||||||
description = "Add your description here"
|
description = "Add your description here"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
|
|
@ -37,6 +37,7 @@ openrag = "tui.main:run_tui"
|
||||||
[tool.uv]
|
[tool.uv]
|
||||||
package = true
|
package = true
|
||||||
|
|
||||||
|
|
||||||
[tool.uv.sources]
|
[tool.uv.sources]
|
||||||
torch = [
|
torch = [
|
||||||
{ index = "pytorch-cu128", marker = "sys_platform == 'linux' and platform_machine == 'x86_64'" },
|
{ index = "pytorch-cu128", marker = "sys_platform == 'linux' and platform_machine == 'x86_64'" },
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,7 @@ async def chat_endpoint(request: Request, chat_service, session_manager):
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
user_id = user.user_id
|
user_id = user.user_id
|
||||||
|
|
||||||
# Get JWT token from auth middleware
|
jwt_token = session_manager.get_effective_jwt_token(user_id, request.state.jwt_token)
|
||||||
jwt_token = request.state.jwt_token
|
|
||||||
|
|
||||||
if not prompt:
|
if not prompt:
|
||||||
return JSONResponse({"error": "Prompt is required"}, status_code=400)
|
return JSONResponse({"error": "Prompt is required"}, status_code=400)
|
||||||
|
|
@ -76,8 +75,7 @@ async def langflow_endpoint(request: Request, chat_service, session_manager):
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
user_id = user.user_id
|
user_id = user.user_id
|
||||||
|
|
||||||
# Get JWT token from auth middleware
|
jwt_token = session_manager.get_effective_jwt_token(user_id, request.state.jwt_token)
|
||||||
jwt_token = request.state.jwt_token
|
|
||||||
|
|
||||||
if not prompt:
|
if not prompt:
|
||||||
return JSONResponse({"error": "Prompt is required"}, status_code=400)
|
return JSONResponse({"error": "Prompt is required"}, status_code=400)
|
||||||
|
|
|
||||||
|
|
@ -13,8 +13,8 @@ async def list_connectors(request: Request, connector_service, session_manager):
|
||||||
)
|
)
|
||||||
return JSONResponse({"connectors": connector_types})
|
return JSONResponse({"connectors": connector_types})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("Error listing connectors", error=str(e))
|
logger.info("Error listing connectors", error=str(e))
|
||||||
return JSONResponse({"error": str(e)}, status_code=500)
|
return JSONResponse({"connectors": []})
|
||||||
|
|
||||||
|
|
||||||
async def connector_sync(request: Request, connector_service, session_manager):
|
async def connector_sync(request: Request, connector_service, session_manager):
|
||||||
|
|
@ -31,7 +31,7 @@ async def connector_sync(request: Request, connector_service, session_manager):
|
||||||
max_files=max_files,
|
max_files=max_files,
|
||||||
)
|
)
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
# Get all active connections for this connector type and user
|
# Get all active connections for this connector type and user
|
||||||
connections = await connector_service.connection_manager.list_connections(
|
connections = await connector_service.connection_manager.list_connections(
|
||||||
|
|
|
||||||
|
|
@ -26,7 +26,7 @@ async def create_knowledge_filter(
|
||||||
return JSONResponse({"error": "Query data is required"}, status_code=400)
|
return JSONResponse({"error": "Query data is required"}, status_code=400)
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
# Create knowledge filter document
|
# Create knowledge filter document
|
||||||
filter_id = str(uuid.uuid4())
|
filter_id = str(uuid.uuid4())
|
||||||
|
|
@ -70,7 +70,7 @@ async def search_knowledge_filters(
|
||||||
limit = payload.get("limit", 20)
|
limit = payload.get("limit", 20)
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
result = await knowledge_filter_service.search_knowledge_filters(
|
result = await knowledge_filter_service.search_knowledge_filters(
|
||||||
query, user_id=user.user_id, jwt_token=jwt_token, limit=limit
|
query, user_id=user.user_id, jwt_token=jwt_token, limit=limit
|
||||||
|
|
@ -101,7 +101,7 @@ async def get_knowledge_filter(
|
||||||
)
|
)
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
result = await knowledge_filter_service.get_knowledge_filter(
|
result = await knowledge_filter_service.get_knowledge_filter(
|
||||||
filter_id, user_id=user.user_id, jwt_token=jwt_token
|
filter_id, user_id=user.user_id, jwt_token=jwt_token
|
||||||
|
|
@ -136,7 +136,7 @@ async def update_knowledge_filter(
|
||||||
payload = await request.json()
|
payload = await request.json()
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
# First, get the existing knowledge filter
|
# First, get the existing knowledge filter
|
||||||
existing_result = await knowledge_filter_service.get_knowledge_filter(
|
existing_result = await knowledge_filter_service.get_knowledge_filter(
|
||||||
|
|
@ -205,7 +205,7 @@ async def delete_knowledge_filter(
|
||||||
)
|
)
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
result = await knowledge_filter_service.delete_knowledge_filter(
|
result = await knowledge_filter_service.delete_knowledge_filter(
|
||||||
filter_id, user_id=user.user_id, jwt_token=jwt_token
|
filter_id, user_id=user.user_id, jwt_token=jwt_token
|
||||||
|
|
@ -239,7 +239,7 @@ async def subscribe_to_knowledge_filter(
|
||||||
|
|
||||||
payload = await request.json()
|
payload = await request.json()
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
# Get the knowledge filter to validate it exists and get its details
|
# Get the knowledge filter to validate it exists and get its details
|
||||||
filter_result = await knowledge_filter_service.get_knowledge_filter(
|
filter_result = await knowledge_filter_service.get_knowledge_filter(
|
||||||
|
|
@ -309,7 +309,7 @@ async def list_knowledge_filter_subscriptions(
|
||||||
)
|
)
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
result = await knowledge_filter_service.get_filter_subscriptions(
|
result = await knowledge_filter_service.get_filter_subscriptions(
|
||||||
filter_id, user_id=user.user_id, jwt_token=jwt_token
|
filter_id, user_id=user.user_id, jwt_token=jwt_token
|
||||||
|
|
@ -341,7 +341,7 @@ async def cancel_knowledge_filter_subscription(
|
||||||
)
|
)
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
# Get subscription details to find the monitor ID
|
# Get subscription details to find the monitor ID
|
||||||
subscriptions_result = await knowledge_filter_service.get_filter_subscriptions(
|
subscriptions_result = await knowledge_filter_service.get_filter_subscriptions(
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ async def nudges_from_kb_endpoint(request: Request, chat_service, session_manage
|
||||||
"""Get nudges for a user"""
|
"""Get nudges for a user"""
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
user_id = user.user_id
|
user_id = user.user_id
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user_id, request.state.jwt_token)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = await chat_service.langflow_nudges_chat(
|
result = await chat_service.langflow_nudges_chat(
|
||||||
|
|
@ -28,7 +28,8 @@ async def nudges_from_chat_id_endpoint(request: Request, chat_service, session_m
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
user_id = user.user_id
|
user_id = user.user_id
|
||||||
chat_id = request.path_params["chat_id"]
|
chat_id = request.path_params["chat_id"]
|
||||||
jwt_token = request.state.jwt_token
|
|
||||||
|
jwt_token = session_manager.get_effective_jwt_token(user_id, request.state.jwt_token)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = await chat_service.langflow_nudges_chat(
|
result = await chat_service.langflow_nudges_chat(
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,7 @@ async def search(request: Request, search_service, session_manager):
|
||||||
) # Optional score threshold, defaults to 0
|
) # Optional score threshold, defaults to 0
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
# Extract JWT token from auth middleware
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
jwt_token = request.state.jwt_token
|
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Search API request",
|
"Search API request",
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ async def upload(request: Request, document_service, session_manager):
|
||||||
form = await request.form()
|
form = await request.form()
|
||||||
upload_file = form["file"]
|
upload_file = form["file"]
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
from config.settings import is_no_auth_mode
|
from config.settings import is_no_auth_mode
|
||||||
|
|
||||||
|
|
@ -60,7 +60,7 @@ async def upload_path(request: Request, task_service, session_manager):
|
||||||
return JSONResponse({"error": "No files found in directory"}, status_code=400)
|
return JSONResponse({"error": "No files found in directory"}, status_code=400)
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
from config.settings import is_no_auth_mode
|
from config.settings import is_no_auth_mode
|
||||||
|
|
||||||
|
|
@ -100,8 +100,7 @@ async def upload_context(
|
||||||
previous_response_id = form.get("previous_response_id")
|
previous_response_id = form.get("previous_response_id")
|
||||||
endpoint = form.get("endpoint", "langflow")
|
endpoint = form.get("endpoint", "langflow")
|
||||||
|
|
||||||
# Get JWT token from auth middleware
|
jwt_token = session_manager.get_effective_jwt_token(user_id, request.state.jwt_token)
|
||||||
jwt_token = request.state.jwt_token
|
|
||||||
|
|
||||||
# Get user info from request state (set by auth middleware)
|
# Get user info from request state (set by auth middleware)
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
|
|
@ -169,7 +168,7 @@ async def upload_bucket(request: Request, task_service, session_manager):
|
||||||
return JSONResponse({"error": "No files found in bucket"}, status_code=400)
|
return JSONResponse({"error": "No files found in bucket"}, status_code=400)
|
||||||
|
|
||||||
user = request.state.user
|
user = request.state.user
|
||||||
jwt_token = request.state.jwt_token
|
jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
|
||||||
|
|
||||||
from models.processors import S3FileProcessor
|
from models.processors import S3FileProcessor
|
||||||
from config.settings import is_no_auth_mode
|
from config.settings import is_no_auth_mode
|
||||||
|
|
|
||||||
|
|
@ -321,7 +321,7 @@ class ConnectionManager:
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get_available_connector_types(self) -> Dict[str, Dict[str, str]]:
|
def get_available_connector_types(self) -> Dict[str, Dict[str, Any]]:
|
||||||
"""Get available connector types with their metadata"""
|
"""Get available connector types with their metadata"""
|
||||||
return {
|
return {
|
||||||
"google_drive": {
|
"google_drive": {
|
||||||
|
|
|
||||||
|
|
@ -58,6 +58,7 @@ from config.settings import (
|
||||||
is_no_auth_mode,
|
is_no_auth_mode,
|
||||||
)
|
)
|
||||||
from services.auth_service import AuthService
|
from services.auth_service import AuthService
|
||||||
|
from services.langflow_mcp_service import LangflowMCPService
|
||||||
from services.chat_service import ChatService
|
from services.chat_service import ChatService
|
||||||
|
|
||||||
# Services
|
# Services
|
||||||
|
|
@ -438,7 +439,11 @@ async def initialize_services():
|
||||||
)
|
)
|
||||||
|
|
||||||
# Initialize auth service
|
# Initialize auth service
|
||||||
auth_service = AuthService(session_manager, connector_service)
|
auth_service = AuthService(
|
||||||
|
session_manager,
|
||||||
|
connector_service,
|
||||||
|
langflow_mcp_service=LangflowMCPService(),
|
||||||
|
)
|
||||||
|
|
||||||
# Load persisted connector connections at startup so webhooks and syncs
|
# Load persisted connector connections at startup so webhooks and syncs
|
||||||
# can resolve existing subscriptions immediately after server boot
|
# can resolve existing subscriptions immediately after server boot
|
||||||
|
|
|
||||||
|
|
@ -5,9 +5,11 @@ import httpx
|
||||||
import aiofiles
|
import aiofiles
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
import asyncio
|
||||||
|
|
||||||
from config.settings import WEBHOOK_BASE_URL, is_no_auth_mode
|
from config.settings import WEBHOOK_BASE_URL, is_no_auth_mode
|
||||||
from session_manager import SessionManager
|
from session_manager import SessionManager
|
||||||
|
from services.langflow_mcp_service import LangflowMCPService
|
||||||
from connectors.google_drive.oauth import GoogleDriveOAuth
|
from connectors.google_drive.oauth import GoogleDriveOAuth
|
||||||
from connectors.onedrive.oauth import OneDriveOAuth
|
from connectors.onedrive.oauth import OneDriveOAuth
|
||||||
from connectors.sharepoint.oauth import SharePointOAuth
|
from connectors.sharepoint.oauth import SharePointOAuth
|
||||||
|
|
@ -17,10 +19,12 @@ from connectors.sharepoint import SharePointConnector
|
||||||
|
|
||||||
|
|
||||||
class AuthService:
|
class AuthService:
|
||||||
def __init__(self, session_manager: SessionManager, connector_service=None):
|
def __init__(self, session_manager: SessionManager, connector_service=None, langflow_mcp_service: LangflowMCPService | None = None):
|
||||||
self.session_manager = session_manager
|
self.session_manager = session_manager
|
||||||
self.connector_service = connector_service
|
self.connector_service = connector_service
|
||||||
self.used_auth_codes = set() # Track used authorization codes
|
self.used_auth_codes = set() # Track used authorization codes
|
||||||
|
self.langflow_mcp_service = langflow_mcp_service
|
||||||
|
self._background_tasks = set()
|
||||||
|
|
||||||
async def init_oauth(
|
async def init_oauth(
|
||||||
self,
|
self,
|
||||||
|
|
@ -287,6 +291,20 @@ class AuthService:
|
||||||
user_info = await self.session_manager.get_user_info_from_token(
|
user_info = await self.session_manager.get_user_info_from_token(
|
||||||
token_data["access_token"]
|
token_data["access_token"]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Best-effort: update Langflow MCP servers to include user's JWT header
|
||||||
|
try:
|
||||||
|
if self.langflow_mcp_service and isinstance(jwt_token, str) and jwt_token.strip():
|
||||||
|
# Run in background to avoid delaying login flow
|
||||||
|
task = asyncio.create_task(
|
||||||
|
self.langflow_mcp_service.update_mcp_servers_with_jwt(jwt_token)
|
||||||
|
)
|
||||||
|
# Keep reference until done to avoid premature GC
|
||||||
|
self._background_tasks.add(task)
|
||||||
|
task.add_done_callback(self._background_tasks.discard)
|
||||||
|
except Exception:
|
||||||
|
# Do not block login on MCP update issues
|
||||||
|
pass
|
||||||
|
|
||||||
response_data = {
|
response_data = {
|
||||||
"status": "authenticated",
|
"status": "authenticated",
|
||||||
|
|
|
||||||
147
src/services/langflow_mcp_service.py
Normal file
147
src/services/langflow_mcp_service.py
Normal file
|
|
@ -0,0 +1,147 @@
|
||||||
|
from typing import List, Dict, Any
|
||||||
|
|
||||||
|
from config.settings import clients
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
|
||||||
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class LangflowMCPService:
|
||||||
|
async def list_mcp_servers(self) -> List[Dict[str, Any]]:
|
||||||
|
"""Fetch list of MCP servers from Langflow (v2 API)."""
|
||||||
|
try:
|
||||||
|
response = await clients.langflow_request(
|
||||||
|
method="GET",
|
||||||
|
endpoint="/api/v2/mcp/servers",
|
||||||
|
params={"action_count": "false"},
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
if isinstance(data, list):
|
||||||
|
return data
|
||||||
|
logger.warning("Unexpected response format for MCP servers list", data_type=type(data).__name__)
|
||||||
|
return []
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Failed to list MCP servers", error=str(e))
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def get_mcp_server(self, server_name: str) -> Dict[str, Any]:
|
||||||
|
"""Get MCP server configuration by name."""
|
||||||
|
response = await clients.langflow_request(
|
||||||
|
method="GET",
|
||||||
|
endpoint=f"/api/v2/mcp/servers/{server_name}",
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
|
|
||||||
|
def _upsert_jwt_header_in_args(self, args: List[str], jwt_token: str) -> List[str]:
|
||||||
|
"""Ensure args contains a header triplet for X-Langflow-Global-Var-JWT with the provided JWT.
|
||||||
|
|
||||||
|
Args are expected in the pattern: [..., "--headers", key, value, ...].
|
||||||
|
If the header exists, update its value; otherwise append the triplet at the end.
|
||||||
|
"""
|
||||||
|
if not isinstance(args, list):
|
||||||
|
return [
|
||||||
|
"mcp-proxy",
|
||||||
|
"--headers",
|
||||||
|
"X-Langflow-Global-Var-JWT",
|
||||||
|
jwt_token,
|
||||||
|
]
|
||||||
|
|
||||||
|
updated_args = list(args)
|
||||||
|
i = 0
|
||||||
|
found_index = -1
|
||||||
|
while i < len(updated_args):
|
||||||
|
token = updated_args[i]
|
||||||
|
if token == "--headers" and i + 2 < len(updated_args):
|
||||||
|
header_key = updated_args[i + 1]
|
||||||
|
if isinstance(header_key, str) and header_key.lower() == "x-langflow-global-var-jwt".lower():
|
||||||
|
found_index = i
|
||||||
|
break
|
||||||
|
i += 3
|
||||||
|
continue
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
if found_index >= 0:
|
||||||
|
# Replace existing value at found_index + 2
|
||||||
|
if found_index + 2 < len(updated_args):
|
||||||
|
updated_args[found_index + 2] = jwt_token
|
||||||
|
else:
|
||||||
|
# Malformed existing header triplet; make sure to append a value
|
||||||
|
updated_args.append(jwt_token)
|
||||||
|
else:
|
||||||
|
updated_args.extend([
|
||||||
|
"--headers",
|
||||||
|
"X-Langflow-Global-Var-JWT",
|
||||||
|
jwt_token,
|
||||||
|
])
|
||||||
|
|
||||||
|
return updated_args
|
||||||
|
|
||||||
|
async def patch_mcp_server_args_with_jwt(self, server_name: str, jwt_token: str) -> bool:
|
||||||
|
"""Patch a single MCP server to include/update the JWT header in args."""
|
||||||
|
try:
|
||||||
|
current = await self.get_mcp_server(server_name)
|
||||||
|
command = current.get("command")
|
||||||
|
args = current.get("args", [])
|
||||||
|
updated_args = self._upsert_jwt_header_in_args(args, jwt_token)
|
||||||
|
|
||||||
|
payload = {"command": command, "args": updated_args}
|
||||||
|
response = await clients.langflow_request(
|
||||||
|
method="PATCH",
|
||||||
|
endpoint=f"/api/v2/mcp/servers/{server_name}",
|
||||||
|
json=payload,
|
||||||
|
)
|
||||||
|
if response.status_code in (200, 201):
|
||||||
|
logger.info(
|
||||||
|
"Patched MCP server with JWT header",
|
||||||
|
server_name=server_name,
|
||||||
|
args_len=len(updated_args),
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to patch MCP server",
|
||||||
|
server_name=server_name,
|
||||||
|
status_code=response.status_code,
|
||||||
|
body=response.text,
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(
|
||||||
|
"Exception while patching MCP server",
|
||||||
|
server_name=server_name,
|
||||||
|
error=str(e),
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def update_mcp_servers_with_jwt(self, jwt_token: str) -> Dict[str, Any]:
|
||||||
|
"""Fetch all MCP servers and ensure each includes the JWT header in args.
|
||||||
|
|
||||||
|
Returns a summary dict with counts.
|
||||||
|
"""
|
||||||
|
servers = await self.list_mcp_servers()
|
||||||
|
if not servers:
|
||||||
|
return {"updated": 0, "failed": 0, "total": 0}
|
||||||
|
|
||||||
|
updated = 0
|
||||||
|
failed = 0
|
||||||
|
for server in servers:
|
||||||
|
name = server.get("name") or server.get("server") or server.get("id")
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
ok = await self.patch_mcp_server_args_with_jwt(name, jwt_token)
|
||||||
|
if ok:
|
||||||
|
updated += 1
|
||||||
|
else:
|
||||||
|
failed += 1
|
||||||
|
|
||||||
|
summary = {"updated": updated, "failed": failed, "total": len(servers)}
|
||||||
|
if failed == 0:
|
||||||
|
logger.info("MCP servers updated with JWT header", **summary)
|
||||||
|
else:
|
||||||
|
logger.warning("MCP servers update had failures", **summary)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -85,6 +85,8 @@ class TaskService:
|
||||||
|
|
||||||
async def create_custom_task(self, user_id: str, items: list, processor) -> str:
|
async def create_custom_task(self, user_id: str, items: list, processor) -> str:
|
||||||
"""Create a new task with custom processor for any type of items"""
|
"""Create a new task with custom processor for any type of items"""
|
||||||
|
# Store anonymous tasks under a stable key so they can be retrieved later
|
||||||
|
store_user_id = user_id or AnonymousUser().user_id
|
||||||
task_id = str(uuid.uuid4())
|
task_id = str(uuid.uuid4())
|
||||||
upload_task = UploadTask(
|
upload_task = UploadTask(
|
||||||
task_id=task_id,
|
task_id=task_id,
|
||||||
|
|
@ -95,12 +97,14 @@ class TaskService:
|
||||||
# Attach the custom processor to the task
|
# Attach the custom processor to the task
|
||||||
upload_task.processor = processor
|
upload_task.processor = processor
|
||||||
|
|
||||||
if user_id not in self.task_store:
|
if store_user_id not in self.task_store:
|
||||||
self.task_store[user_id] = {}
|
self.task_store[store_user_id] = {}
|
||||||
self.task_store[user_id][task_id] = upload_task
|
self.task_store[store_user_id][task_id] = upload_task
|
||||||
|
|
||||||
# Start background processing
|
# Start background processing
|
||||||
background_task = asyncio.create_task(self.background_custom_processor(user_id, task_id, items))
|
background_task = asyncio.create_task(
|
||||||
|
self.background_custom_processor(store_user_id, task_id, items)
|
||||||
|
)
|
||||||
self.background_tasks.add(background_task)
|
self.background_tasks.add(background_task)
|
||||||
background_task.add_done_callback(self.background_tasks.discard)
|
background_task.add_done_callback(self.background_tasks.discard)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -191,26 +191,8 @@ class SessionManager:
|
||||||
|
|
||||||
def get_user_opensearch_client(self, user_id: str, jwt_token: str):
|
def get_user_opensearch_client(self, user_id: str, jwt_token: str):
|
||||||
"""Get or create OpenSearch client for user with their JWT"""
|
"""Get or create OpenSearch client for user with their JWT"""
|
||||||
from config.settings import is_no_auth_mode
|
# Get the effective JWT token (handles anonymous JWT creation)
|
||||||
|
jwt_token = self.get_effective_jwt_token(user_id, jwt_token)
|
||||||
logger.debug(
|
|
||||||
"get_user_opensearch_client",
|
|
||||||
user_id=user_id,
|
|
||||||
jwt_token_present=(jwt_token is not None),
|
|
||||||
no_auth_mode=is_no_auth_mode(),
|
|
||||||
)
|
|
||||||
|
|
||||||
# In no-auth mode, create anonymous JWT for OpenSearch DLS
|
|
||||||
if jwt_token is None and (is_no_auth_mode() or user_id in (None, AnonymousUser().user_id)):
|
|
||||||
if not hasattr(self, "_anonymous_jwt"):
|
|
||||||
# Create anonymous JWT token for OpenSearch OIDC
|
|
||||||
logger.debug("Creating anonymous JWT")
|
|
||||||
self._anonymous_jwt = self._create_anonymous_jwt()
|
|
||||||
logger.debug(
|
|
||||||
"Anonymous JWT created", jwt_prefix=self._anonymous_jwt[:50]
|
|
||||||
)
|
|
||||||
jwt_token = self._anonymous_jwt
|
|
||||||
logger.debug("Using anonymous JWT for OpenSearch")
|
|
||||||
|
|
||||||
# Check if we have a cached client for this user
|
# Check if we have a cached client for this user
|
||||||
if user_id not in self.user_opensearch_clients:
|
if user_id not in self.user_opensearch_clients:
|
||||||
|
|
@ -222,6 +204,31 @@ class SessionManager:
|
||||||
|
|
||||||
return self.user_opensearch_clients[user_id]
|
return self.user_opensearch_clients[user_id]
|
||||||
|
|
||||||
|
def get_effective_jwt_token(self, user_id: str, jwt_token: str) -> str:
|
||||||
|
"""Get the effective JWT token, creating anonymous JWT if needed in no-auth mode"""
|
||||||
|
from config.settings import is_no_auth_mode
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"get_effective_jwt_token",
|
||||||
|
user_id=user_id,
|
||||||
|
jwt_token_present=(jwt_token is not None),
|
||||||
|
no_auth_mode=is_no_auth_mode(),
|
||||||
|
)
|
||||||
|
|
||||||
|
# In no-auth mode, create anonymous JWT if needed
|
||||||
|
if jwt_token is None and (is_no_auth_mode() or user_id in (None, AnonymousUser().user_id)):
|
||||||
|
if not hasattr(self, "_anonymous_jwt"):
|
||||||
|
# Create anonymous JWT token for OpenSearch OIDC
|
||||||
|
logger.debug("Creating anonymous JWT")
|
||||||
|
self._anonymous_jwt = self._create_anonymous_jwt()
|
||||||
|
logger.debug(
|
||||||
|
"Anonymous JWT created", jwt_prefix=self._anonymous_jwt[:50]
|
||||||
|
)
|
||||||
|
jwt_token = self._anonymous_jwt
|
||||||
|
logger.debug("Using anonymous JWT")
|
||||||
|
|
||||||
|
return jwt_token
|
||||||
|
|
||||||
def _create_anonymous_jwt(self) -> str:
|
def _create_anonymous_jwt(self) -> str:
|
||||||
"""Create JWT token for anonymous user in no-auth mode"""
|
"""Create JWT token for anonymous user in no-auth mode"""
|
||||||
anonymous_user = AnonymousUser()
|
anonymous_user = AnonymousUser()
|
||||||
|
|
|
||||||
111
src/tui/_assets/docker-compose-cpu.yml
Normal file
111
src/tui/_assets/docker-compose-cpu.yml
Normal file
|
|
@ -0,0 +1,111 @@
|
||||||
|
services:
|
||||||
|
opensearch:
|
||||||
|
image: phact/openrag-opensearch:${OPENRAG_VERSION:-latest}
|
||||||
|
#build:
|
||||||
|
# context: .
|
||||||
|
# dockerfile: Dockerfile
|
||||||
|
container_name: os
|
||||||
|
depends_on:
|
||||||
|
- openrag-backend
|
||||||
|
environment:
|
||||||
|
- discovery.type=single-node
|
||||||
|
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}
|
||||||
|
# Run security setup in background after OpenSearch starts
|
||||||
|
command: >
|
||||||
|
bash -c "
|
||||||
|
# Start OpenSearch in background
|
||||||
|
/usr/share/opensearch/opensearch-docker-entrypoint.sh opensearch &
|
||||||
|
|
||||||
|
# Wait a bit for OpenSearch to start, then apply security config
|
||||||
|
sleep 10 && /usr/share/opensearch/setup-security.sh &
|
||||||
|
|
||||||
|
# Wait for background processes
|
||||||
|
wait
|
||||||
|
"
|
||||||
|
ports:
|
||||||
|
- "9200:9200"
|
||||||
|
- "9600:9600"
|
||||||
|
|
||||||
|
dashboards:
|
||||||
|
image: opensearchproject/opensearch-dashboards:3.0.0
|
||||||
|
container_name: osdash
|
||||||
|
depends_on:
|
||||||
|
- opensearch
|
||||||
|
environment:
|
||||||
|
OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
|
||||||
|
OPENSEARCH_USERNAME: "admin"
|
||||||
|
OPENSEARCH_PASSWORD: ${OPENSEARCH_PASSWORD}
|
||||||
|
ports:
|
||||||
|
- "5601:5601"
|
||||||
|
|
||||||
|
openrag-backend:
|
||||||
|
image: phact/openrag-backend:${OPENRAG_VERSION:-latest}
|
||||||
|
#build:
|
||||||
|
#context: .
|
||||||
|
#dockerfile: Dockerfile.backend
|
||||||
|
container_name: openrag-backend
|
||||||
|
depends_on:
|
||||||
|
- langflow
|
||||||
|
environment:
|
||||||
|
- OPENSEARCH_HOST=opensearch
|
||||||
|
- LANGFLOW_URL=http://langflow:7860
|
||||||
|
- LANGFLOW_PUBLIC_URL=${LANGFLOW_PUBLIC_URL}
|
||||||
|
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
|
||||||
|
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
|
||||||
|
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
|
||||||
|
- LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID}
|
||||||
|
- LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID}
|
||||||
|
- DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false}
|
||||||
|
- NUDGES_FLOW_ID=${NUDGES_FLOW_ID}
|
||||||
|
- OPENSEARCH_PORT=9200
|
||||||
|
- OPENSEARCH_USERNAME=admin
|
||||||
|
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
|
||||||
|
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||||
|
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||||
|
- NVIDIA_VISIBLE_DEVICES=all
|
||||||
|
- GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID}
|
||||||
|
- GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET}
|
||||||
|
- MICROSOFT_GRAPH_OAUTH_CLIENT_ID=${MICROSOFT_GRAPH_OAUTH_CLIENT_ID}
|
||||||
|
- MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET=${MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET}
|
||||||
|
- WEBHOOK_BASE_URL=${WEBHOOK_BASE_URL}
|
||||||
|
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
|
||||||
|
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
|
||||||
|
volumes:
|
||||||
|
- ./documents:/app/documents:Z
|
||||||
|
- ./keys:/app/keys:Z
|
||||||
|
- ./flows:/app/flows:Z
|
||||||
|
|
||||||
|
openrag-frontend:
|
||||||
|
image: phact/openrag-frontend:${OPENRAG_VERSION:-latest}
|
||||||
|
#build:
|
||||||
|
#context: .
|
||||||
|
#dockerfile: Dockerfile.frontend
|
||||||
|
container_name: openrag-frontend
|
||||||
|
depends_on:
|
||||||
|
- openrag-backend
|
||||||
|
environment:
|
||||||
|
- OPENRAG_BACKEND_HOST=openrag-backend
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
|
||||||
|
langflow:
|
||||||
|
volumes:
|
||||||
|
- ./flows:/app/flows:Z
|
||||||
|
image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest}
|
||||||
|
container_name: langflow
|
||||||
|
ports:
|
||||||
|
- "7860:7860"
|
||||||
|
environment:
|
||||||
|
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||||
|
- LANGFLOW_LOAD_FLOWS_PATH=/app/flows
|
||||||
|
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
|
||||||
|
- JWT="dummy"
|
||||||
|
- OPENRAG-QUERY-FILTER="{}"
|
||||||
|
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
|
||||||
|
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD
|
||||||
|
- LANGFLOW_LOG_LEVEL=DEBUG
|
||||||
|
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
|
||||||
|
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
|
||||||
|
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
|
||||||
|
- LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE}
|
||||||
|
- LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI}
|
||||||
111
src/tui/_assets/docker-compose.yml
Normal file
111
src/tui/_assets/docker-compose.yml
Normal file
|
|
@ -0,0 +1,111 @@
|
||||||
|
services:
|
||||||
|
opensearch:
|
||||||
|
image: phact/openrag-opensearch:${OPENRAG_VERSION:-latest}
|
||||||
|
#build:
|
||||||
|
#context: .
|
||||||
|
#dockerfile: Dockerfile
|
||||||
|
container_name: os
|
||||||
|
depends_on:
|
||||||
|
- openrag-backend
|
||||||
|
environment:
|
||||||
|
- discovery.type=single-node
|
||||||
|
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}
|
||||||
|
# Run security setup in background after OpenSearch starts
|
||||||
|
command: >
|
||||||
|
bash -c "
|
||||||
|
# Start OpenSearch in background
|
||||||
|
/usr/share/opensearch/opensearch-docker-entrypoint.sh opensearch &
|
||||||
|
|
||||||
|
# Wait a bit for OpenSearch to start, then apply security config
|
||||||
|
sleep 10 && /usr/share/opensearch/setup-security.sh &
|
||||||
|
|
||||||
|
# Wait for background processes
|
||||||
|
wait
|
||||||
|
"
|
||||||
|
ports:
|
||||||
|
- "9200:9200"
|
||||||
|
- "9600:9600"
|
||||||
|
|
||||||
|
dashboards:
|
||||||
|
image: opensearchproject/opensearch-dashboards:3.0.0
|
||||||
|
container_name: osdash
|
||||||
|
depends_on:
|
||||||
|
- opensearch
|
||||||
|
environment:
|
||||||
|
OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
|
||||||
|
OPENSEARCH_USERNAME: "admin"
|
||||||
|
OPENSEARCH_PASSWORD: ${OPENSEARCH_PASSWORD}
|
||||||
|
ports:
|
||||||
|
- "5601:5601"
|
||||||
|
|
||||||
|
openrag-backend:
|
||||||
|
image: phact/openrag-backend:${OPENRAG_VERSION:-latest}
|
||||||
|
#build:
|
||||||
|
#context: .
|
||||||
|
#dockerfile: Dockerfile.backend
|
||||||
|
container_name: openrag-backend
|
||||||
|
depends_on:
|
||||||
|
- langflow
|
||||||
|
environment:
|
||||||
|
- OPENSEARCH_HOST=opensearch
|
||||||
|
- LANGFLOW_URL=http://langflow:7860
|
||||||
|
- LANGFLOW_PUBLIC_URL=${LANGFLOW_PUBLIC_URL}
|
||||||
|
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
|
||||||
|
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
|
||||||
|
- LANGFLOW_CHAT_FLOW_ID=${LANGFLOW_CHAT_FLOW_ID}
|
||||||
|
- LANGFLOW_INGEST_FLOW_ID=${LANGFLOW_INGEST_FLOW_ID}
|
||||||
|
- DISABLE_INGEST_WITH_LANGFLOW=${DISABLE_INGEST_WITH_LANGFLOW:-false}
|
||||||
|
- NUDGES_FLOW_ID=${NUDGES_FLOW_ID}
|
||||||
|
- OPENSEARCH_PORT=9200
|
||||||
|
- OPENSEARCH_USERNAME=admin
|
||||||
|
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
|
||||||
|
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||||
|
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||||
|
- NVIDIA_VISIBLE_DEVICES=all
|
||||||
|
- GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID}
|
||||||
|
- GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET}
|
||||||
|
- MICROSOFT_GRAPH_OAUTH_CLIENT_ID=${MICROSOFT_GRAPH_OAUTH_CLIENT_ID}
|
||||||
|
- MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET=${MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET}
|
||||||
|
- WEBHOOK_BASE_URL=${WEBHOOK_BASE_URL}
|
||||||
|
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
|
||||||
|
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
|
||||||
|
volumes:
|
||||||
|
- ./documents:/app/documents:Z
|
||||||
|
- ./keys:/app/keys:Z
|
||||||
|
- ./flows:/app/flows:Z
|
||||||
|
gpus: all
|
||||||
|
|
||||||
|
openrag-frontend:
|
||||||
|
image: phact/openrag-frontend:${OPENRAG_VERSION:-latest}
|
||||||
|
#build:
|
||||||
|
#context: .
|
||||||
|
#dockerfile: Dockerfile.frontend
|
||||||
|
container_name: openrag-frontend
|
||||||
|
depends_on:
|
||||||
|
- openrag-backend
|
||||||
|
environment:
|
||||||
|
- OPENRAG_BACKEND_HOST=openrag-backend
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
|
||||||
|
langflow:
|
||||||
|
volumes:
|
||||||
|
- ./flows:/app/flows:Z
|
||||||
|
image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest}
|
||||||
|
container_name: langflow
|
||||||
|
ports:
|
||||||
|
- "7860:7860"
|
||||||
|
environment:
|
||||||
|
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||||
|
- LANGFLOW_LOAD_FLOWS_PATH=/app/flows
|
||||||
|
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
|
||||||
|
- JWT="dummy"
|
||||||
|
- OPENRAG-QUERY-FILTER="{}"
|
||||||
|
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
|
||||||
|
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD
|
||||||
|
- LANGFLOW_LOG_LEVEL=DEBUG
|
||||||
|
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
|
||||||
|
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
|
||||||
|
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
|
||||||
|
- LANGFLOW_NEW_USER_IS_ACTIVE=${LANGFLOW_NEW_USER_IS_ACTIVE}
|
||||||
|
- LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI}
|
||||||
BIN
src/tui/_assets/documents/2506.08231v1.pdf
Normal file
BIN
src/tui/_assets/documents/2506.08231v1.pdf
Normal file
Binary file not shown.
BIN
src/tui/_assets/documents/ai-human-resources.pdf
Normal file
BIN
src/tui/_assets/documents/ai-human-resources.pdf
Normal file
Binary file not shown.
BIN
src/tui/_assets/documents/warmup_ocr.pdf
Normal file
BIN
src/tui/_assets/documents/warmup_ocr.pdf
Normal file
Binary file not shown.
|
|
@ -4,6 +4,10 @@ import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from textual.app import App, ComposeResult
|
from textual.app import App, ComposeResult
|
||||||
from utils.logging_config import get_logger
|
from utils.logging_config import get_logger
|
||||||
|
try:
|
||||||
|
from importlib.resources import files
|
||||||
|
except ImportError:
|
||||||
|
from importlib_resources import files
|
||||||
|
|
||||||
logger = get_logger(__name__)
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
|
@ -301,10 +305,42 @@ class OpenRAGTUI(App):
|
||||||
return True, "Runtime requirements satisfied"
|
return True, "Runtime requirements satisfied"
|
||||||
|
|
||||||
|
|
||||||
|
def copy_sample_documents():
|
||||||
|
"""Copy sample documents from package to current directory if they don't exist."""
|
||||||
|
documents_dir = Path("documents")
|
||||||
|
|
||||||
|
# Check if documents directory already exists and has files
|
||||||
|
if documents_dir.exists() and any(documents_dir.glob("*.pdf")):
|
||||||
|
return # Documents already exist, don't overwrite
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Get sample documents from package assets
|
||||||
|
assets_files = files("tui._assets.documents")
|
||||||
|
|
||||||
|
# Create documents directory if it doesn't exist
|
||||||
|
documents_dir.mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
# Copy each sample document
|
||||||
|
for resource in assets_files.iterdir():
|
||||||
|
if resource.is_file() and resource.name.endswith('.pdf'):
|
||||||
|
dest_path = documents_dir / resource.name
|
||||||
|
if not dest_path.exists():
|
||||||
|
content = resource.read_bytes()
|
||||||
|
dest_path.write_bytes(content)
|
||||||
|
logger.info(f"Copied sample document: {resource.name}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"Could not copy sample documents: {e}")
|
||||||
|
# This is not a critical error - the app can work without sample documents
|
||||||
|
|
||||||
|
|
||||||
def run_tui():
|
def run_tui():
|
||||||
"""Run the OpenRAG TUI application."""
|
"""Run the OpenRAG TUI application."""
|
||||||
app = None
|
app = None
|
||||||
try:
|
try:
|
||||||
|
# Copy sample documents on first run
|
||||||
|
copy_sample_documents()
|
||||||
|
|
||||||
app = OpenRAGTUI()
|
app = OpenRAGTUI()
|
||||||
app.run()
|
app.run()
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,10 @@ from enum import Enum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, List, Optional, AsyncIterator
|
from typing import Dict, List, Optional, AsyncIterator
|
||||||
from utils.logging_config import get_logger
|
from utils.logging_config import get_logger
|
||||||
|
try:
|
||||||
|
from importlib.resources import files
|
||||||
|
except ImportError:
|
||||||
|
from importlib_resources import files
|
||||||
|
|
||||||
logger = get_logger(__name__)
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
|
@ -51,8 +55,8 @@ class ContainerManager:
|
||||||
def __init__(self, compose_file: Optional[Path] = None):
|
def __init__(self, compose_file: Optional[Path] = None):
|
||||||
self.platform_detector = PlatformDetector()
|
self.platform_detector = PlatformDetector()
|
||||||
self.runtime_info = self.platform_detector.detect_runtime()
|
self.runtime_info = self.platform_detector.detect_runtime()
|
||||||
self.compose_file = compose_file or Path("docker-compose.yml")
|
self.compose_file = compose_file or self._find_compose_file("docker-compose.yml")
|
||||||
self.cpu_compose_file = Path("docker-compose-cpu.yml")
|
self.cpu_compose_file = self._find_compose_file("docker-compose-cpu.yml")
|
||||||
self.services_cache: Dict[str, ServiceInfo] = {}
|
self.services_cache: Dict[str, ServiceInfo] = {}
|
||||||
self.last_status_update = 0
|
self.last_status_update = 0
|
||||||
# Auto-select CPU compose if no GPU available
|
# Auto-select CPU compose if no GPU available
|
||||||
|
|
@ -80,6 +84,42 @@ class ContainerManager:
|
||||||
"langflow": "langflow",
|
"langflow": "langflow",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def _find_compose_file(self, filename: str) -> Path:
|
||||||
|
"""Find compose file in current directory or package resources."""
|
||||||
|
# First check current working directory
|
||||||
|
cwd_path = Path(filename)
|
||||||
|
self._compose_search_log = f"Searching for {filename}:\n"
|
||||||
|
self._compose_search_log += f" 1. Current directory: {cwd_path.absolute()}"
|
||||||
|
|
||||||
|
if cwd_path.exists():
|
||||||
|
self._compose_search_log += " ✓ FOUND"
|
||||||
|
return cwd_path
|
||||||
|
else:
|
||||||
|
self._compose_search_log += " ✗ NOT FOUND"
|
||||||
|
|
||||||
|
# Then check package resources
|
||||||
|
self._compose_search_log += f"\n 2. Package resources: "
|
||||||
|
try:
|
||||||
|
pkg_files = files("tui._assets")
|
||||||
|
self._compose_search_log += f"{pkg_files}"
|
||||||
|
compose_resource = pkg_files / filename
|
||||||
|
|
||||||
|
if compose_resource.is_file():
|
||||||
|
self._compose_search_log += f" ✓ FOUND, copying to current directory"
|
||||||
|
# Copy to cwd for compose command to work
|
||||||
|
content = compose_resource.read_text()
|
||||||
|
cwd_path.write_text(content)
|
||||||
|
return cwd_path
|
||||||
|
else:
|
||||||
|
self._compose_search_log += f" ✗ NOT FOUND"
|
||||||
|
except Exception as e:
|
||||||
|
self._compose_search_log += f" ✗ SKIPPED ({e})"
|
||||||
|
# Don't log this as an error since it's expected when running from source
|
||||||
|
|
||||||
|
# Fall back to original path (will fail later if not found)
|
||||||
|
self._compose_search_log += f"\n 3. Falling back to: {cwd_path.absolute()}"
|
||||||
|
return Path(filename)
|
||||||
|
|
||||||
def is_available(self) -> bool:
|
def is_available(self) -> bool:
|
||||||
"""Check if container runtime is available."""
|
"""Check if container runtime is available."""
|
||||||
return self.runtime_info.runtime_type != RuntimeType.NONE
|
return self.runtime_info.runtime_type != RuntimeType.NONE
|
||||||
|
|
@ -469,6 +509,20 @@ class ContainerManager:
|
||||||
yield False, "No container runtime available"
|
yield False, "No container runtime available"
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Diagnostic info about compose files
|
||||||
|
compose_file = self.cpu_compose_file if (cpu_mode if cpu_mode is not None else self.use_cpu_compose) else self.compose_file
|
||||||
|
|
||||||
|
# Show the search process for debugging
|
||||||
|
if hasattr(self, '_compose_search_log'):
|
||||||
|
for line in self._compose_search_log.split('\n'):
|
||||||
|
if line.strip():
|
||||||
|
yield False, line
|
||||||
|
|
||||||
|
yield False, f"Final compose file: {compose_file.absolute()}"
|
||||||
|
if not compose_file.exists():
|
||||||
|
yield False, f"ERROR: Compose file not found at {compose_file.absolute()}"
|
||||||
|
return
|
||||||
|
|
||||||
yield False, "Starting OpenRAG services..."
|
yield False, "Starting OpenRAG services..."
|
||||||
|
|
||||||
missing_images: List[str] = []
|
missing_images: List[str] = []
|
||||||
|
|
|
||||||
|
|
@ -79,6 +79,15 @@ class EnvManager:
|
||||||
"""Generate a secure secret key for Langflow."""
|
"""Generate a secure secret key for Langflow."""
|
||||||
return secrets.token_urlsafe(32)
|
return secrets.token_urlsafe(32)
|
||||||
|
|
||||||
|
def _quote_env_value(self, value: str) -> str:
|
||||||
|
"""Single quote all environment variable values for consistency."""
|
||||||
|
if not value:
|
||||||
|
return "''"
|
||||||
|
|
||||||
|
# Escape any existing single quotes by replacing ' with '\''
|
||||||
|
escaped_value = value.replace("'", "'\\''")
|
||||||
|
return f"'{escaped_value}'"
|
||||||
|
|
||||||
def load_existing_env(self) -> bool:
|
def load_existing_env(self) -> bool:
|
||||||
"""Load existing .env file if it exists."""
|
"""Load existing .env file if it exists."""
|
||||||
if not self.env_file.exists():
|
if not self.env_file.exists():
|
||||||
|
|
@ -237,36 +246,36 @@ class EnvManager:
|
||||||
|
|
||||||
# Core settings
|
# Core settings
|
||||||
f.write("# Core settings\n")
|
f.write("# Core settings\n")
|
||||||
f.write(f"LANGFLOW_SECRET_KEY={self.config.langflow_secret_key}\n")
|
f.write(f"LANGFLOW_SECRET_KEY={self._quote_env_value(self.config.langflow_secret_key)}\n")
|
||||||
f.write(f"LANGFLOW_SUPERUSER={self.config.langflow_superuser}\n")
|
f.write(f"LANGFLOW_SUPERUSER={self._quote_env_value(self.config.langflow_superuser)}\n")
|
||||||
f.write(
|
f.write(
|
||||||
f"LANGFLOW_SUPERUSER_PASSWORD={self.config.langflow_superuser_password}\n"
|
f"LANGFLOW_SUPERUSER_PASSWORD={self._quote_env_value(self.config.langflow_superuser_password)}\n"
|
||||||
)
|
)
|
||||||
f.write(f"LANGFLOW_CHAT_FLOW_ID={self.config.langflow_chat_flow_id}\n")
|
f.write(f"LANGFLOW_CHAT_FLOW_ID={self._quote_env_value(self.config.langflow_chat_flow_id)}\n")
|
||||||
f.write(
|
f.write(
|
||||||
f"LANGFLOW_INGEST_FLOW_ID={self.config.langflow_ingest_flow_id}\n"
|
f"LANGFLOW_INGEST_FLOW_ID={self._quote_env_value(self.config.langflow_ingest_flow_id)}\n"
|
||||||
)
|
)
|
||||||
f.write(f"NUDGES_FLOW_ID={self.config.nudges_flow_id}\n")
|
f.write(f"NUDGES_FLOW_ID={self._quote_env_value(self.config.nudges_flow_id)}\n")
|
||||||
f.write(f"OPENSEARCH_PASSWORD={self.config.opensearch_password}\n")
|
f.write(f"OPENSEARCH_PASSWORD={self._quote_env_value(self.config.opensearch_password)}\n")
|
||||||
f.write(f"OPENAI_API_KEY={self.config.openai_api_key}\n")
|
f.write(f"OPENAI_API_KEY={self._quote_env_value(self.config.openai_api_key)}\n")
|
||||||
f.write(
|
f.write(
|
||||||
f"OPENRAG_DOCUMENTS_PATHS={self.config.openrag_documents_paths}\n"
|
f"OPENRAG_DOCUMENTS_PATHS={self._quote_env_value(self.config.openrag_documents_paths)}\n"
|
||||||
)
|
)
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
||||||
# Ingestion settings
|
# Ingestion settings
|
||||||
f.write("# Ingestion settings\n")
|
f.write("# Ingestion settings\n")
|
||||||
f.write(f"DISABLE_INGEST_WITH_LANGFLOW={self.config.disable_ingest_with_langflow}\n")
|
f.write(f"DISABLE_INGEST_WITH_LANGFLOW={self._quote_env_value(self.config.disable_ingest_with_langflow)}\n")
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
||||||
# Langflow auth settings
|
# Langflow auth settings
|
||||||
f.write("# Langflow auth settings\n")
|
f.write("# Langflow auth settings\n")
|
||||||
f.write(f"LANGFLOW_AUTO_LOGIN={self.config.langflow_auto_login}\n")
|
f.write(f"LANGFLOW_AUTO_LOGIN={self._quote_env_value(self.config.langflow_auto_login)}\n")
|
||||||
f.write(
|
f.write(
|
||||||
f"LANGFLOW_NEW_USER_IS_ACTIVE={self.config.langflow_new_user_is_active}\n"
|
f"LANGFLOW_NEW_USER_IS_ACTIVE={self._quote_env_value(self.config.langflow_new_user_is_active)}\n"
|
||||||
)
|
)
|
||||||
f.write(
|
f.write(
|
||||||
f"LANGFLOW_ENABLE_SUPERUSER_CLI={self.config.langflow_enable_superuser_cli}\n"
|
f"LANGFLOW_ENABLE_SUPERUSER_CLI={self._quote_env_value(self.config.langflow_enable_superuser_cli)}\n"
|
||||||
)
|
)
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
||||||
|
|
@ -277,10 +286,10 @@ class EnvManager:
|
||||||
):
|
):
|
||||||
f.write("# Google OAuth settings\n")
|
f.write("# Google OAuth settings\n")
|
||||||
f.write(
|
f.write(
|
||||||
f"GOOGLE_OAUTH_CLIENT_ID={self.config.google_oauth_client_id}\n"
|
f"GOOGLE_OAUTH_CLIENT_ID={self._quote_env_value(self.config.google_oauth_client_id)}\n"
|
||||||
)
|
)
|
||||||
f.write(
|
f.write(
|
||||||
f"GOOGLE_OAUTH_CLIENT_SECRET={self.config.google_oauth_client_secret}\n"
|
f"GOOGLE_OAUTH_CLIENT_SECRET={self._quote_env_value(self.config.google_oauth_client_secret)}\n"
|
||||||
)
|
)
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
||||||
|
|
@ -290,10 +299,10 @@ class EnvManager:
|
||||||
):
|
):
|
||||||
f.write("# Microsoft Graph OAuth settings\n")
|
f.write("# Microsoft Graph OAuth settings\n")
|
||||||
f.write(
|
f.write(
|
||||||
f"MICROSOFT_GRAPH_OAUTH_CLIENT_ID={self.config.microsoft_graph_oauth_client_id}\n"
|
f"MICROSOFT_GRAPH_OAUTH_CLIENT_ID={self._quote_env_value(self.config.microsoft_graph_oauth_client_id)}\n"
|
||||||
)
|
)
|
||||||
f.write(
|
f.write(
|
||||||
f"MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET={self.config.microsoft_graph_oauth_client_secret}\n"
|
f"MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET={self._quote_env_value(self.config.microsoft_graph_oauth_client_secret)}\n"
|
||||||
)
|
)
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
||||||
|
|
@ -311,7 +320,7 @@ class EnvManager:
|
||||||
if not optional_written:
|
if not optional_written:
|
||||||
f.write("# Optional settings\n")
|
f.write("# Optional settings\n")
|
||||||
optional_written = True
|
optional_written = True
|
||||||
f.write(f"{var_name}={var_value}\n")
|
f.write(f"{var_name}={self._quote_env_value(var_value)}\n")
|
||||||
|
|
||||||
if optional_written:
|
if optional_written:
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
|
||||||
4
uv.lock
generated
4
uv.lock
generated
|
|
@ -1,5 +1,5 @@
|
||||||
version = 1
|
version = 1
|
||||||
revision = 3
|
revision = 2
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
resolution-markers = [
|
resolution-markers = [
|
||||||
"sys_platform == 'darwin'",
|
"sys_platform == 'darwin'",
|
||||||
|
|
@ -2282,7 +2282,7 @@ wheels = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "openrag"
|
name = "openrag"
|
||||||
version = "0.1.3"
|
version = "0.1.8"
|
||||||
source = { editable = "." }
|
source = { editable = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "agentd" },
|
{ name = "agentd" },
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue