refactor to openrag and knowledge filters

This commit is contained in:
phact 2025-08-13 16:31:35 -04:00
parent 125a6f0cbe
commit 346b938d98
25 changed files with 390 additions and 2629 deletions

View file

@ -1,6 +1,6 @@
### gendb
### OpenRAG
[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/phact/gendb)
[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/phact/openrag)
docker compose build

View file

@ -5,7 +5,7 @@ services:
dockerfile: Dockerfile
container_name: os
depends_on:
- gendb-backend
- openrag-backend
environment:
- discovery.type=single-node
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}
@ -37,12 +37,12 @@ services:
ports:
- "5601:5601"
gendb-backend:
image: phact/gendb-backend:latest
openrag-backend:
image: phact/openrag-backend:latest
#build:
#context: .
#dockerfile: Dockerfile.backend
container_name: gendb-backend
container_name: openrag-backend
depends_on:
- langflow
environment:
@ -67,16 +67,16 @@ services:
gpus: all
platform: linux/amd64
gendb-frontend:
image: phact/gendb-frontend:latest
openrag-frontend:
image: phact/openrag-frontend:latest
#build:
#context: .
#dockerfile: Dockerfile.frontend
container_name: gendb-frontend
container_name: openrag-frontend
depends_on:
- gendb-backend
- openrag-backend
environment:
- GENDB_BACKEND_HOST=gendb-backend
- OPENRAG_BACKEND_HOST=openrag-backend
ports:
- "3000:3000"
volumes:
@ -94,6 +94,6 @@ services:
- LANGFLOW_LOAD_FLOWS_PATH=/app/flows
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
- JWT="dummy"
- GENDB-QUERY-FILTER="{}"
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT
- OPENRAG-QUERY-FILTER="{}"
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER
- LANGFLOW_LOG_LEVEL=DEBUG

File diff suppressed because one or more lines are too long

View file

@ -17,7 +17,7 @@ export function NavigationLayout({ children }: NavigationLayoutProps) {
<div className="container flex h-14 max-w-screen-2xl items-center">
<div className="mr-4 hidden md:flex">
<h1 className="text-lg font-semibold tracking-tight">
GenDB
OpenRAG
</h1>
</div>
<div className="flex flex-1 items-center justify-between space-x-2 md:justify-end">

View file

@ -28,10 +28,10 @@ export function Navigation() {
active: pathname === "/chat",
},
{
label: "Contexts",
label: "Knowledge Filters",
icon: BookOpenCheck,
href: "/contexts",
active: pathname.startsWith("/contexts"),
href: "/knowledge-filters",
active: pathname.startsWith("/knowledge-filters"),
},
{
label: "Connectors",

View file

@ -39,7 +39,7 @@ async function proxyRequest(
request: NextRequest,
params: { path: string[] }
) {
const backendHost = process.env.GENDB_BACKEND_HOST || 'localhost';
const backendHost = process.env.OPENRAG_BACKEND_HOST || 'localhost';
const path = params.path.join('/');
const searchParams = request.nextUrl.searchParams.toString();
const backendUrl = `http://${backendHost}:8000/${path}${searchParams ? `?${searchParams}` : ''}`;

View file

@ -135,7 +135,7 @@ function AuthCallbackContent() {
return isAppAuth ? "Signing you in..." : "Connecting..."
}
if (status === "success") {
return isAppAuth ? "Welcome to GenDB!" : "Connection Successful!"
return isAppAuth ? "Welcome to OpenRAG!" : "Connection Successful!"
}
if (status === "error") {
return isAppAuth ? "Sign In Failed" : "Connection Failed"

View file

@ -89,17 +89,17 @@ function ChatPage() {
const [scoreThreshold, setScoreThreshold] = useState(0)
const [loadedContextName, setLoadedContextName] = useState<string | null>(null)
// Load context if contextId is provided in URL
// Load knowledge filter if filterId is provided in URL
useEffect(() => {
const contextId = searchParams.get('contextId')
if (contextId) {
loadContext(contextId)
const filterId = searchParams.get('filterId')
if (filterId) {
loadKnowledgeFilter(filterId)
}
}, [searchParams])
const loadContext = async (contextId: string) => {
const loadKnowledgeFilter = async (filterId: string) => {
try {
const response = await fetch(`/api/contexts/${contextId}`, {
const response = await fetch(`/api/knowledge-filter/${filterId}`, {
method: "GET",
headers: {
"Content-Type": "application/json",
@ -108,19 +108,19 @@ function ChatPage() {
const result = await response.json()
if (response.ok && result.success) {
const context = result.context
const parsedQueryData = JSON.parse(context.query_data)
const filter = result.filter
const parsedQueryData = JSON.parse(filter.query_data)
// Load the context data into state
setSelectedFilters(parsedQueryData.filters)
setResultLimit(parsedQueryData.limit)
setScoreThreshold(parsedQueryData.scoreThreshold)
setLoadedContextName(context.name)
setLoadedContextName(filter.name)
} else {
console.error("Failed to load context:", result.error)
console.error("Failed to load knowledge filter:", result.error)
}
} catch (error) {
console.error("Error loading context:", error)
console.error("Error loading knowledge filter:", error)
}
}
@ -283,10 +283,14 @@ function ChatPage() {
const apiEndpoint = endpoint === "chat" ? "/api/chat" : "/api/langflow"
try {
const hasFilters = selectedFilters.data_sources.length > 0 ||
selectedFilters.document_types.length > 0 ||
selectedFilters.owners.length > 0
const requestBody: RequestBody = {
prompt: userMessage.content,
stream: true,
filters: selectedFilters,
...(hasFilters && { filters: selectedFilters }),
limit: resultLimit,
scoreThreshold: scoreThreshold
}
@ -744,9 +748,13 @@ function ChatPage() {
try {
const apiEndpoint = endpoint === "chat" ? "/api/chat" : "/api/langflow"
const hasFilters = selectedFilters.data_sources.length > 0 ||
selectedFilters.document_types.length > 0 ||
selectedFilters.owners.length > 0
const requestBody: RequestBody = {
prompt: userMessage.content,
filters: selectedFilters,
...(hasFilters && { filters: selectedFilters }),
limit: resultLimit,
scoreThreshold: scoreThreshold
}

View file

@ -9,7 +9,7 @@ import { Label } from "@/components/ui/label"
import { Search, Loader2, BookOpenCheck, Settings, Calendar, MessageCircle } from "lucide-react"
import { ProtectedRoute } from "@/components/protected-route"
interface Context {
interface KnowledgeFilter {
id: string
name: string
description: string
@ -30,18 +30,18 @@ interface ParsedQueryData {
scoreThreshold: number
}
function ContextsPage() {
function KnowledgeFiltersPage() {
const router = useRouter()
const [contexts, setContexts] = useState<Context[]>([])
const [filters, setFilters] = useState<KnowledgeFilter[]>([])
const [loading, setLoading] = useState(true)
const [searchQuery, setSearchQuery] = useState("")
const [selectedContext, setSelectedContext] = useState<Context | null>(null)
const [selectedFilter, setSelectedFilter] = useState<KnowledgeFilter | null>(null)
const [parsedQueryData, setParsedQueryData] = useState<ParsedQueryData | null>(null)
const loadContexts = async (query = "") => {
const loadFilters = async (query = "") => {
setLoading(true)
try {
const response = await fetch("/api/contexts/search", {
const response = await fetch("/api/knowledge-filter/search", {
method: "POST",
headers: {
"Content-Type": "application/json",
@ -54,32 +54,32 @@ function ContextsPage() {
const result = await response.json()
if (response.ok && result.success) {
setContexts(result.contexts)
setFilters(result.filters)
} else {
console.error("Failed to load contexts:", result.error)
setContexts([])
console.error("Failed to load knowledge filters:", result.error)
setFilters([])
}
} catch (error) {
console.error("Error loading contexts:", error)
setContexts([])
console.error("Error loading knowledge filters:", error)
setFilters([])
} finally {
setLoading(false)
}
}
useEffect(() => {
loadContexts()
loadFilters()
}, [])
const handleSearch = async (e: React.FormEvent) => {
e.preventDefault()
await loadContexts(searchQuery)
await loadFilters(searchQuery)
}
const handleContextClick = (context: Context) => {
setSelectedContext(context)
const handleFilterClick = (filter: KnowledgeFilter) => {
setSelectedFilter(filter)
try {
const parsed = JSON.parse(context.query_data) as ParsedQueryData
const parsed = JSON.parse(filter.query_data) as ParsedQueryData
setParsedQueryData(parsed)
} catch (error) {
console.error("Error parsing query data:", error)
@ -97,16 +97,16 @@ function ContextsPage() {
})
}
const handleSearchWithContext = () => {
if (!selectedContext) return
const handleSearchWithFilter = () => {
if (!selectedFilter) return
router.push(`/?contextId=${selectedContext.id}`)
router.push(`/?filterId=${selectedFilter.id}`)
}
const handleChatWithContext = () => {
if (!selectedContext) return
const handleChatWithFilter = () => {
if (!selectedFilter) return
router.push(`/chat?contextId=${selectedContext.id}`)
router.push(`/chat?filterId=${selectedFilter.id}`)
}
return (
@ -115,14 +115,14 @@ function ContextsPage() {
<div className="space-y-4">
<div className="mb-4">
<h1 className="text-4xl font-bold tracking-tight text-white">
Contexts
Knowledge Filters
</h1>
</div>
<p className="text-xl text-muted-foreground">
Manage your saved search contexts
Manage your saved knowledge filters
</p>
<p className="text-sm text-muted-foreground max-w-2xl">
View and manage your saved search queries, filters, and configurations for quick access to your most important searches.
View and manage your saved search configurations that help you focus on specific subsets of your knowledge base.
</p>
</div>
@ -131,10 +131,10 @@ function ContextsPage() {
<CardHeader>
<CardTitle className="flex items-center gap-2">
<BookOpenCheck className="h-5 w-5" />
Search Contexts
Search Knowledge Filters
</CardTitle>
<CardDescription>
Search through your saved search contexts by name or description
Search through your saved knowledge filters by name or description
</CardDescription>
</CardHeader>
<CardContent className="space-y-6">
@ -142,7 +142,7 @@ function ContextsPage() {
<div className="flex gap-2">
<Input
type="text"
placeholder="Search contexts..."
placeholder="Search knowledge filters..."
value={searchQuery}
onChange={(e) => setSearchQuery(e.target.value)}
className="h-12 bg-background/50 border-border/50 focus:border-blue-400/50 focus:ring-blue-400/20 flex-1"
@ -170,7 +170,7 @@ function ContextsPage() {
<div className="flex gap-6">
{/* Context List */}
<div className="flex-1 space-y-4">
{contexts.length === 0 ? (
{filters.length === 0 ? (
<Card className="bg-muted/20 border-dashed border-muted-foreground/30">
<CardContent className="pt-8 pb-8">
<div className="text-center space-y-3">
@ -178,40 +178,40 @@ function ContextsPage() {
<BookOpenCheck className="h-8 w-8 text-muted-foreground/50" />
</div>
<p className="text-lg font-medium text-muted-foreground">
No contexts found
No knowledge filters found
</p>
<p className="text-sm text-muted-foreground/70 max-w-md mx-auto">
Create your first context by saving a search configuration from the search page.
Create your first knowledge filter by saving a search configuration from the search page.
</p>
</div>
</CardContent>
</Card>
) : (
<div className="space-y-3">
{contexts.map((context) => (
{filters.map((filter) => (
<Card
key={context.id}
key={filter.id}
className={`bg-card/50 backdrop-blur-sm border-border/50 hover:bg-card/70 transition-all duration-200 hover:shadow-lg hover:shadow-blue-500/10 cursor-pointer ${
selectedContext?.id === context.id ? 'ring-2 ring-blue-500/50 bg-card/70' : ''
selectedFilter?.id === filter.id ? 'ring-2 ring-blue-500/50 bg-card/70' : ''
}`}
onClick={() => handleContextClick(context)}
onClick={() => handleFilterClick(filter)}
>
<CardContent className="p-4">
<div className="flex items-start justify-between">
<div className="flex-1 space-y-2">
<h3 className="font-semibold text-lg">{context.name}</h3>
{context.description && (
<p className="text-sm text-muted-foreground">{context.description}</p>
<h3 className="font-semibold text-lg">{filter.name}</h3>
{filter.description && (
<p className="text-sm text-muted-foreground">{filter.description}</p>
)}
<div className="flex items-center gap-4 text-xs text-muted-foreground">
<div className="flex items-center gap-1">
<Calendar className="h-3 w-3" />
<span>Created {formatDate(context.created_at)}</span>
<span>Created {formatDate(filter.created_at)}</span>
</div>
{context.updated_at !== context.created_at && (
{filter.updated_at !== filter.created_at && (
<div className="flex items-center gap-1">
<Calendar className="h-3 w-3" />
<span>Updated {formatDate(context.updated_at)}</span>
<span>Updated {formatDate(filter.updated_at)}</span>
</div>
)}
</div>
@ -225,12 +225,12 @@ function ContextsPage() {
</div>
{/* Context Detail Panel */}
{selectedContext && parsedQueryData && (
{selectedFilter && parsedQueryData && (
<div className="w-64 space-y-6 flex-shrink-0">
<div className="flex items-center justify-between">
<h2 className="text-lg font-semibold flex items-center gap-2">
<Settings className="h-5 w-5" />
Context Details
Knowledge Filter Details
</h2>
</div>
@ -311,21 +311,21 @@ function ContextsPage() {
{/* Action Buttons */}
<div className="space-y-2 pt-4 border-t border-border/50">
<Button
onClick={handleSearchWithContext}
onClick={handleSearchWithFilter}
className="w-full flex items-center gap-2"
variant="default"
>
<Search className="h-4 w-4" />
Search with Context
Search with Filter
</Button>
<Button
onClick={handleChatWithContext}
onClick={handleChatWithFilter}
className="w-full flex items-center gap-2"
variant="outline"
>
<MessageCircle className="h-4 w-4" />
Chat with Context
Chat with Filter
</Button>
</div>
</div>
@ -338,10 +338,10 @@ function ContextsPage() {
)
}
export default function ProtectedContextsPage() {
export default function ProtectedKnowledgeFiltersPage() {
return (
<ProtectedRoute>
<ContextsPage />
<KnowledgeFiltersPage />
</ProtectedRoute>
)
}

View file

@ -18,8 +18,8 @@ const geistMono = Geist_Mono({
});
export const metadata: Metadata = {
title: "GenDB",
description: "Document search and management system",
title: "OpenRAG",
description: "Open source RAG (Retrieval Augmented Generation) system",
};
export default function RootLayout({

View file

@ -43,7 +43,7 @@ function LoginPageContent() {
<Lock className="h-8 w-8 text-primary" />
</div>
<div>
<CardTitle className="text-2xl">Welcome to GenDB</CardTitle>
<CardTitle className="text-2xl">Welcome to OpenRAG</CardTitle>
<CardDescription className="mt-2">
Sign in to access your documents and AI chat
</CardDescription>

View file

@ -88,48 +88,7 @@ function SearchPage() {
const [savingContext, setSavingContext] = useState(false)
const [loadedContextName, setLoadedContextName] = useState<string | null>(null)
const loadContext = useCallback(async (contextId: string) => {
try {
const response = await fetch(`/api/contexts/${contextId}`, {
method: "GET",
headers: {
"Content-Type": "application/json",
},
})
const result = await response.json()
if (response.ok && result.success) {
const context = result.context
const parsedQueryData = JSON.parse(context.query_data)
// Load the context data into state
setQuery(parsedQueryData.query)
setSelectedFilters(parsedQueryData.filters)
setResultLimit(parsedQueryData.limit)
setScoreThreshold(parsedQueryData.scoreThreshold)
setLoadedContextName(context.name)
// Automatically perform the search
setTimeout(() => {
handleSearch()
}, 100)
} else {
console.error("Failed to load context:", result.error)
}
} catch (err) {
console.error("Error loading context:", err)
}
}, [])
// Load context if contextId is provided in URL
useEffect(() => {
const contextId = searchParams.get('contextId')
if (contextId) {
loadContext(contextId)
}
}, [searchParams, loadContext])
const handleSearch = async (e?: React.FormEvent) => {
const handleSearch = useCallback(async (e?: React.FormEvent) => {
if (e) e.preventDefault()
if (!query.trim()) return
@ -202,7 +161,48 @@ function SearchPage() {
} finally {
setLoading(false)
}
}
}, [query, resultLimit, scoreThreshold, searchPerformed, selectedFilters])
const loadKnowledgeFilter = useCallback(async (filterId: string) => {
try {
const response = await fetch(`/api/knowledge-filter/${filterId}`, {
method: "GET",
headers: {
"Content-Type": "application/json",
},
})
const result = await response.json()
if (response.ok && result.success) {
const filter = result.filter
const parsedQueryData = JSON.parse(filter.query_data)
// Load the context data into state
setQuery(parsedQueryData.query)
setSelectedFilters(parsedQueryData.filters)
setResultLimit(parsedQueryData.limit)
setScoreThreshold(parsedQueryData.scoreThreshold)
setLoadedContextName(filter.name)
// Automatically perform the search
setTimeout(() => {
handleSearch()
}, 100)
} else {
console.error("Failed to load knowledge filter:", result.error)
}
} catch (err) {
console.error("Error loading knowledge filter:", err)
}
}, [handleSearch])
// Load knowledge filter if filterId is provided in URL
useEffect(() => {
const filterId = searchParams.get('filterId')
if (filterId) {
loadKnowledgeFilter(filterId)
}
}, [searchParams, loadKnowledgeFilter])
const handleFilterChange = async (facetType: keyof SelectedFilters, value: string, checked: boolean) => {
const newFilters = {
@ -277,16 +277,16 @@ function SearchPage() {
selectedFilters.owners.length
}
const handleSaveContext = async () => {
const contextId = searchParams.get('contextId')
const handleSaveKnowledgeFilter = async () => {
const filterId = searchParams.get('filterId')
// If no contextId present and no title, we need the modal
if (!contextId && !contextTitle.trim()) return
// If no filterId present and no title, we need the modal
if (!filterId && !contextTitle.trim()) return
setSavingContext(true)
try {
const contextData = {
const filterData = {
query,
filters: selectedFilters,
limit: resultLimit,
@ -295,20 +295,20 @@ function SearchPage() {
let response;
if (contextId) {
// Update existing context (upsert)
response = await fetch(`/api/contexts/${contextId}`, {
if (filterId) {
// Update existing knowledge filter (upsert)
response = await fetch(`/api/knowledge-filter/${filterId}`, {
method: "PUT",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
queryData: JSON.stringify(contextData)
queryData: JSON.stringify(filterData)
}),
})
} else {
// Create new context
response = await fetch("/api/contexts", {
// Create new knowledge filter
response = await fetch("/api/knowledge-filter", {
method: "POST",
headers: {
"Content-Type": "application/json",
@ -316,7 +316,7 @@ function SearchPage() {
body: JSON.stringify({
name: contextTitle,
description: contextDescription,
queryData: JSON.stringify(contextData)
queryData: JSON.stringify(filterData)
}),
})
}
@ -324,18 +324,18 @@ function SearchPage() {
const result = await response.json()
if (response.ok && result.success) {
if (!contextId) {
// Reset modal state only if we were creating a new context
if (!filterId) {
// Reset modal state only if we were creating a new knowledge filter
setShowSaveModal(false)
setContextTitle("")
setContextDescription("")
}
toast.success(contextId ? "Context updated successfully" : "Context saved successfully")
toast.success(filterId ? "Knowledge filter updated successfully" : "Knowledge filter saved successfully")
} else {
toast.error(contextId ? "Failed to update context" : "Failed to save context")
toast.error(filterId ? "Failed to update knowledge filter" : "Failed to save knowledge filter")
}
} catch {
toast.error(contextId ? "Error updating context" : "Error saving context")
toast.error(filterId ? "Error updating knowledge filter" : "Error saving knowledge filter")
} finally {
setSavingContext(false)
}
@ -843,13 +843,13 @@ function SearchPage() {
</div>
</div>
{/* Save Context Button */}
{/* Save Knowledge Filter Button */}
<div className="pt-4 border-t border-border/50">
<Button
onClick={() => {
const contextId = searchParams.get('contextId')
if (contextId) {
handleSaveContext()
const filterId = searchParams.get('filterId')
if (filterId) {
handleSaveKnowledgeFilter()
} else {
setShowSaveModal(true)
}
@ -861,12 +861,12 @@ function SearchPage() {
{savingContext ? (
<>
<Loader2 className="h-4 w-4 animate-spin" />
{searchParams.get('contextId') ? 'Updating...' : 'Saving...'}
{searchParams.get('filterId') ? 'Updating...' : 'Saving...'}
</>
) : (
<>
<Save className="h-4 w-4" />
{searchParams.get('contextId') ? 'Update Context' : 'Save Context'}
{searchParams.get('filterId') ? 'Update Knowledge Filter' : 'Save Knowledge Filter'}
</>
)}
</Button>
@ -938,7 +938,7 @@ function SearchPage() {
Cancel
</Button>
<Button
onClick={handleSaveContext}
onClick={handleSaveKnowledgeFilter}
disabled={!contextTitle.trim() || savingContext}
className="flex items-center gap-2"
>

View file

@ -39,7 +39,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
<div className="flex h-14 items-center px-4">
<div className="flex items-center">
<h1 className="text-lg font-semibold tracking-tight text-white">
GenDB
OpenRAG
</h1>
</div>
<div className="flex flex-1 items-center justify-end space-x-2">

View file

@ -1,5 +1,5 @@
[project]
name = "gendb"
name = "openrag"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"

View file

@ -12,7 +12,7 @@ config:
type: openid
challenge: false
config:
openid_connect_url: "http://gendb-backend:8000/.well-known/openid-configuration"
openid_connect_url: "http://openrag-backend:8000/.well-known/openid-configuration"
subject_key: "sub"
jwt_header: "Authorization" # expects Bearer token
roles_key: "roles"

View file

@ -2,13 +2,13 @@ _meta:
type: "roles"
config_version: 2
gendb_user_role:
openrag_user_role:
description: "DLS: user can read/write docs they own or are allowed on"
cluster_permissions:
- "indices:data/write/bulk"
- "indices:data/write/index"
index_permissions:
- index_patterns: ["documents", "documents*", "search_contexts", "search_contexts*"]
- index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*"]
allowed_actions:
- crud
- create_index

View file

@ -2,11 +2,11 @@ _meta:
type: "rolesmapping"
config_version: 2
gendb_user_role:
openrag_user_role:
users: []
hosts: []
backend_roles:
- "gendb_user"
- "openrag_user"
all_access:
users:

View file

@ -1,114 +0,0 @@
from starlette.requests import Request
from starlette.responses import JSONResponse
import uuid
from datetime import datetime
async def create_context(request: Request, contexts_service, session_manager):
"""Create a new search context"""
payload = await request.json()
name = payload.get("name")
if not name:
return JSONResponse({"error": "Context name is required"}, status_code=400)
description = payload.get("description", "")
query_data = payload.get("queryData")
if not query_data:
return JSONResponse({"error": "Query data is required"}, status_code=400)
user = request.state.user
jwt_token = request.cookies.get("auth_token")
# Create context document
context_id = str(uuid.uuid4())
context_doc = {
"id": context_id,
"name": name,
"description": description,
"query_data": query_data, # Store the full search query JSON
"owner": user.user_id,
"allowed_users": payload.get("allowedUsers", []), # ACL field for future use
"allowed_groups": payload.get("allowedGroups", []), # ACL field for future use
"created_at": datetime.utcnow().isoformat(),
"updated_at": datetime.utcnow().isoformat()
}
result = await contexts_service.create_context(context_doc, user_id=user.user_id, jwt_token=jwt_token)
return JSONResponse(result)
async def search_contexts(request: Request, contexts_service, session_manager):
"""Search for contexts by name, description, or query content"""
payload = await request.json()
query = payload.get("query", "")
limit = payload.get("limit", 20)
user = request.state.user
jwt_token = request.cookies.get("auth_token")
result = await contexts_service.search_contexts(query, user_id=user.user_id, jwt_token=jwt_token, limit=limit)
return JSONResponse(result)
async def get_context(request: Request, contexts_service, session_manager):
"""Get a specific context by ID"""
context_id = request.path_params.get("context_id")
if not context_id:
return JSONResponse({"error": "Context ID is required"}, status_code=400)
user = request.state.user
jwt_token = request.cookies.get("auth_token")
result = await contexts_service.get_context(context_id, user_id=user.user_id, jwt_token=jwt_token)
return JSONResponse(result)
async def update_context(request: Request, contexts_service, session_manager):
"""Update an existing context by delete + recreate (due to DLS limitations)"""
context_id = request.path_params.get("context_id")
if not context_id:
return JSONResponse({"error": "Context ID is required"}, status_code=400)
payload = await request.json()
user = request.state.user
jwt_token = request.cookies.get("auth_token")
# First, get the existing context
existing_result = await contexts_service.get_context(context_id, user_id=user.user_id, jwt_token=jwt_token)
if not existing_result.get("success"):
return JSONResponse({"error": "Context not found or access denied"}, status_code=404)
existing_context = existing_result["context"]
# Delete the existing context
delete_result = await contexts_service.delete_context(context_id, user_id=user.user_id, jwt_token=jwt_token)
if not delete_result.get("success"):
return JSONResponse({"error": "Failed to delete existing context"}, status_code=500)
# Create updated context document with same ID
updated_context = {
"id": context_id,
"name": payload.get("name", existing_context["name"]),
"description": payload.get("description", existing_context["description"]),
"query_data": payload.get("queryData", existing_context["query_data"]),
"owner": existing_context["owner"],
"allowed_users": payload.get("allowedUsers", existing_context.get("allowed_users", [])),
"allowed_groups": payload.get("allowedGroups", existing_context.get("allowed_groups", [])),
"created_at": existing_context["created_at"], # Preserve original creation time
"updated_at": datetime.utcnow().isoformat()
}
# Recreate the context
result = await contexts_service.create_context(updated_context, user_id=user.user_id, jwt_token=jwt_token)
return JSONResponse(result)
async def delete_context(request: Request, contexts_service, session_manager):
"""Delete a context"""
context_id = request.path_params.get("context_id")
if not context_id:
return JSONResponse({"error": "Context ID is required"}, status_code=400)
user = request.state.user
jwt_token = request.cookies.get("auth_token")
result = await contexts_service.delete_context(context_id, user_id=user.user_id, jwt_token=jwt_token)
return JSONResponse(result)

114
src/api/knowledge_filter.py Normal file
View file

@ -0,0 +1,114 @@
from starlette.requests import Request
from starlette.responses import JSONResponse
import uuid
from datetime import datetime
async def create_knowledge_filter(request: Request, knowledge_filter_service, session_manager):
"""Create a new knowledge filter"""
payload = await request.json()
name = payload.get("name")
if not name:
return JSONResponse({"error": "Knowledge filter name is required"}, status_code=400)
description = payload.get("description", "")
query_data = payload.get("queryData")
if not query_data:
return JSONResponse({"error": "Query data is required"}, status_code=400)
user = request.state.user
jwt_token = request.cookies.get("auth_token")
# Create knowledge filter document
filter_id = str(uuid.uuid4())
filter_doc = {
"id": filter_id,
"name": name,
"description": description,
"query_data": query_data, # Store the full search query JSON
"owner": user.user_id,
"allowed_users": payload.get("allowedUsers", []), # ACL field for future use
"allowed_groups": payload.get("allowedGroups", []), # ACL field for future use
"created_at": datetime.utcnow().isoformat(),
"updated_at": datetime.utcnow().isoformat()
}
result = await knowledge_filter_service.create_knowledge_filter(filter_doc, user_id=user.user_id, jwt_token=jwt_token)
return JSONResponse(result)
async def search_knowledge_filters(request: Request, knowledge_filter_service, session_manager):
"""Search for knowledge filters by name, description, or query content"""
payload = await request.json()
query = payload.get("query", "")
limit = payload.get("limit", 20)
user = request.state.user
jwt_token = request.cookies.get("auth_token")
result = await knowledge_filter_service.search_knowledge_filters(query, user_id=user.user_id, jwt_token=jwt_token, limit=limit)
return JSONResponse(result)
async def get_knowledge_filter(request: Request, knowledge_filter_service, session_manager):
"""Get a specific knowledge filter by ID"""
filter_id = request.path_params.get("filter_id")
if not filter_id:
return JSONResponse({"error": "Knowledge filter ID is required"}, status_code=400)
user = request.state.user
jwt_token = request.cookies.get("auth_token")
result = await knowledge_filter_service.get_knowledge_filter(filter_id, user_id=user.user_id, jwt_token=jwt_token)
return JSONResponse(result)
async def update_knowledge_filter(request: Request, knowledge_filter_service, session_manager):
"""Update an existing knowledge filter by delete + recreate (due to DLS limitations)"""
filter_id = request.path_params.get("filter_id")
if not filter_id:
return JSONResponse({"error": "Knowledge filter ID is required"}, status_code=400)
payload = await request.json()
user = request.state.user
jwt_token = request.cookies.get("auth_token")
# First, get the existing knowledge filter
existing_result = await knowledge_filter_service.get_knowledge_filter(filter_id, user_id=user.user_id, jwt_token=jwt_token)
if not existing_result.get("success"):
return JSONResponse({"error": "Knowledge filter not found or access denied"}, status_code=404)
existing_filter = existing_result["filter"]
# Delete the existing knowledge filter
delete_result = await knowledge_filter_service.delete_knowledge_filter(filter_id, user_id=user.user_id, jwt_token=jwt_token)
if not delete_result.get("success"):
return JSONResponse({"error": "Failed to delete existing knowledge filter"}, status_code=500)
# Create updated knowledge filter document with same ID
updated_filter = {
"id": filter_id,
"name": payload.get("name", existing_filter["name"]),
"description": payload.get("description", existing_filter["description"]),
"query_data": payload.get("queryData", existing_filter["query_data"]),
"owner": existing_filter["owner"],
"allowed_users": payload.get("allowedUsers", existing_filter.get("allowed_users", [])),
"allowed_groups": payload.get("allowedGroups", existing_filter.get("allowed_groups", [])),
"created_at": existing_filter["created_at"], # Preserve original creation time
"updated_at": datetime.utcnow().isoformat()
}
# Recreate the knowledge filter
result = await knowledge_filter_service.create_knowledge_filter(updated_filter, user_id=user.user_id, jwt_token=jwt_token)
return JSONResponse(result)
async def delete_knowledge_filter(request: Request, knowledge_filter_service, session_manager):
"""Delete a knowledge filter"""
filter_id = request.path_params.get("filter_id")
if not filter_id:
return JSONResponse({"error": "Knowledge filter ID is required"}, status_code=400)
user = request.state.user
jwt_token = request.cookies.get("auth_token")
result = await knowledge_filter_service.delete_knowledge_filter(filter_id, user_id=user.user_id, jwt_token=jwt_token)
return JSONResponse(result)

View file

@ -51,7 +51,7 @@ async def jwks_endpoint(request: Request, session_manager):
"kty": "RSA",
"use": "sig",
"alg": "RS256",
"kid": "gendb-key-1",
"kid": "openrag-key-1",
"n": int_to_base64url(public_numbers.n),
"e": int_to_base64url(public_numbers.e)
}

View file

@ -23,7 +23,7 @@ from services.search_service import SearchService
from services.task_service import TaskService
from services.auth_service import AuthService
from services.chat_service import ChatService
from services.contexts_service import ContextsService
from services.knowledge_filter_service import KnowledgeFilterService
# Existing services
from connectors.service import ConnectorService
@ -31,7 +31,7 @@ from session_manager import SessionManager
from auth_middleware import require_auth, optional_auth
# API endpoints
from api import upload, search, chat, auth, connectors, tasks, oidc, contexts
from api import upload, search, chat, auth, connectors, tasks, oidc, knowledge_filter
print("CUDA available:", torch.cuda.is_available())
print("CUDA version PyTorch was built with:", torch.version.cuda)
@ -64,9 +64,9 @@ async def init_index():
else:
print(f"Index '{INDEX_NAME}' already exists, skipping creation.")
# Create contexts index
contexts_index_name = "search_contexts"
contexts_index_body = {
# Create knowledge filters index
knowledge_filter_index_name = "knowledge_filters"
knowledge_filter_index_body = {
"mappings": {
"properties": {
"id": {"type": "keyword"},
@ -82,11 +82,11 @@ async def init_index():
}
}
if not await clients.opensearch.indices.exists(index=contexts_index_name):
await clients.opensearch.indices.create(index=contexts_index_name, body=contexts_index_body)
print(f"Created index '{contexts_index_name}'")
if not await clients.opensearch.indices.exists(index=knowledge_filter_index_name):
await clients.opensearch.indices.create(index=knowledge_filter_index_name, body=knowledge_filter_index_body)
print(f"Created index '{knowledge_filter_index_name}'")
else:
print(f"Index '{contexts_index_name}' already exists, skipping creation.")
print(f"Index '{knowledge_filter_index_name}' already exists, skipping creation.")
async def init_index_when_ready():
"""Initialize OpenSearch index when it becomes available"""
@ -111,7 +111,7 @@ def initialize_services():
search_service = SearchService(session_manager)
task_service = TaskService(document_service, process_pool)
chat_service = ChatService()
contexts_service = ContextsService(session_manager)
knowledge_filter_service = KnowledgeFilterService(session_manager)
# Set process pool for document service
document_service.process_pool = process_pool
@ -136,7 +136,7 @@ def initialize_services():
'chat_service': chat_service,
'auth_service': auth_service,
'connector_service': connector_service,
'contexts_service': contexts_service,
'knowledge_filter_service': knowledge_filter_service,
'session_manager': session_manager
}
@ -198,39 +198,39 @@ def create_app():
session_manager=services['session_manager'])
), methods=["POST"]),
# Contexts endpoints
Route("/contexts",
# Knowledge Filter endpoints
Route("/knowledge-filter",
require_auth(services['session_manager'])(
partial(contexts.create_context,
contexts_service=services['contexts_service'],
partial(knowledge_filter.create_knowledge_filter,
knowledge_filter_service=services['knowledge_filter_service'],
session_manager=services['session_manager'])
), methods=["POST"]),
Route("/contexts/search",
Route("/knowledge-filter/search",
require_auth(services['session_manager'])(
partial(contexts.search_contexts,
contexts_service=services['contexts_service'],
partial(knowledge_filter.search_knowledge_filters,
knowledge_filter_service=services['knowledge_filter_service'],
session_manager=services['session_manager'])
), methods=["POST"]),
Route("/contexts/{context_id}",
Route("/knowledge-filter/{filter_id}",
require_auth(services['session_manager'])(
partial(contexts.get_context,
contexts_service=services['contexts_service'],
partial(knowledge_filter.get_knowledge_filter,
knowledge_filter_service=services['knowledge_filter_service'],
session_manager=services['session_manager'])
), methods=["GET"]),
Route("/contexts/{context_id}",
Route("/knowledge-filter/{filter_id}",
require_auth(services['session_manager'])(
partial(contexts.update_context,
contexts_service=services['contexts_service'],
partial(knowledge_filter.update_knowledge_filter,
knowledge_filter_service=services['knowledge_filter_service'],
session_manager=services['session_manager'])
), methods=["PUT"]),
Route("/contexts/{context_id}",
Route("/knowledge-filter/{filter_id}",
require_auth(services['session_manager'])(
partial(contexts.delete_context,
contexts_service=services['contexts_service'],
partial(knowledge_filter.delete_knowledge_filter,
knowledge_filter_service=services['knowledge_filter_service'],
session_manager=services['session_manager'])
), methods=["DELETE"]),

View file

@ -77,8 +77,8 @@ class ChatService:
# Pass the complete filter expression as a single header to Langflow (only if we have something to send)
if filter_expression:
print(f"Sending GenDB query filter to Langflow: {json.dumps(filter_expression, indent=2)}")
extra_headers['X-LANGFLOW-GLOBAL-VAR-GENDB-QUERY-FILTER'] = json.dumps(filter_expression)
print(f"Sending OpenRAG query filter to Langflow: {json.dumps(filter_expression, indent=2)}")
extra_headers['X-LANGFLOW-GLOBAL-VAR-OPENRAG-QUERY-FILTER'] = json.dumps(filter_expression)
if stream:
return async_langflow_stream(clients.langflow_client, FLOW_ID, prompt, extra_headers=extra_headers, previous_response_id=previous_response_id)

View file

@ -1,34 +1,34 @@
from typing import Any, Dict, Optional
CONTEXTS_INDEX_NAME = "search_contexts"
KNOWLEDGE_FILTERS_INDEX_NAME = "knowledge_filters"
class ContextsService:
class KnowledgeFilterService:
def __init__(self, session_manager=None):
self.session_manager = session_manager
async def create_context(self, context_doc: Dict[str, Any], user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
"""Create a new search context"""
async def create_knowledge_filter(self, filter_doc: Dict[str, Any], user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
"""Create a new knowledge filter"""
try:
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
# Index the context document
# Index the knowledge filter document
result = await opensearch_client.index(
index=CONTEXTS_INDEX_NAME,
id=context_doc["id"],
body=context_doc
index=KNOWLEDGE_FILTERS_INDEX_NAME,
id=filter_doc["id"],
body=filter_doc
)
if result.get("result") == "created":
return {"success": True, "id": context_doc["id"], "context": context_doc}
return {"success": True, "id": filter_doc["id"], "filter": filter_doc}
else:
return {"success": False, "error": "Failed to create context"}
return {"success": False, "error": "Failed to create knowledge filter"}
except Exception as e:
return {"success": False, "error": str(e)}
async def search_contexts(self, query: str, user_id: str = None, jwt_token: str = None, limit: int = 20) -> Dict[str, Any]:
"""Search for contexts by name, description, or query content"""
async def search_knowledge_filters(self, query: str, user_id: str = None, jwt_token: str = None, limit: int = 20) -> Dict[str, Any]:
"""Search for knowledge filters by name, description, or query content"""
try:
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
@ -52,7 +52,7 @@ class ContextsService:
"size": limit
}
else:
# No query - return all contexts sorted by most recent
# No query - return all knowledge filters sorted by most recent
search_body = {
"query": {"match_all": {}},
"sort": [{"updated_at": {"order": "desc"}}],
@ -60,72 +60,72 @@ class ContextsService:
"size": limit
}
result = await opensearch_client.search(index=CONTEXTS_INDEX_NAME, body=search_body)
result = await opensearch_client.search(index=KNOWLEDGE_FILTERS_INDEX_NAME, body=search_body)
# Transform results
contexts = []
filters = []
for hit in result["hits"]["hits"]:
context = hit["_source"]
context["score"] = hit.get("_score")
contexts.append(context)
knowledge_filter = hit["_source"]
knowledge_filter["score"] = hit.get("_score")
filters.append(knowledge_filter)
return {"success": True, "contexts": contexts}
return {"success": True, "filters": filters}
except Exception as e:
return {"success": False, "error": str(e), "contexts": []}
return {"success": False, "error": str(e), "filters": []}
async def get_context(self, context_id: str, user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
"""Get a specific context by ID"""
async def get_knowledge_filter(self, filter_id: str, user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
"""Get a specific knowledge filter by ID"""
try:
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
result = await opensearch_client.get(index=CONTEXTS_INDEX_NAME, id=context_id)
result = await opensearch_client.get(index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id)
if result.get("found"):
context = result["_source"]
return {"success": True, "context": context}
knowledge_filter = result["_source"]
return {"success": True, "filter": knowledge_filter}
else:
return {"success": False, "error": "Context not found"}
return {"success": False, "error": "Knowledge filter not found"}
except Exception as e:
return {"success": False, "error": str(e)}
async def update_context(self, context_id: str, updates: Dict[str, Any], user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
"""Update an existing context"""
async def update_knowledge_filter(self, filter_id: str, updates: Dict[str, Any], user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
"""Update an existing knowledge filter"""
try:
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
# Update the document
result = await opensearch_client.update(
index=CONTEXTS_INDEX_NAME,
id=context_id,
index=KNOWLEDGE_FILTERS_INDEX_NAME,
id=filter_id,
body={"doc": updates}
)
if result.get("result") in ["updated", "noop"]:
# Get the updated document
updated_doc = await opensearch_client.get(index=CONTEXTS_INDEX_NAME, id=context_id)
return {"success": True, "context": updated_doc["_source"]}
updated_doc = await opensearch_client.get(index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id)
return {"success": True, "filter": updated_doc["_source"]}
else:
return {"success": False, "error": "Failed to update context"}
return {"success": False, "error": "Failed to update knowledge filter"}
except Exception as e:
return {"success": False, "error": str(e)}
async def delete_context(self, context_id: str, user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
"""Delete a context"""
async def delete_knowledge_filter(self, filter_id: str, user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
"""Delete a knowledge filter"""
try:
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
result = await opensearch_client.delete(index=CONTEXTS_INDEX_NAME, id=context_id)
result = await opensearch_client.delete(index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id)
if result.get("result") == "deleted":
return {"success": True, "message": "Context deleted successfully"}
return {"success": True, "message": "Knowledge filter deleted successfully"}
else:
return {"success": False, "error": "Failed to delete context"}
return {"success": False, "error": "Failed to delete knowledge filter"}
except Exception as e:
return {"success": False, "error": str(e)}

View file

@ -109,7 +109,7 @@ class SessionManager:
# OIDC standard claims
"iss": issuer, # Issuer from request
"sub": user_id, # Subject (user ID)
"aud": ["opensearch", "gendb"], # Audience
"aud": ["opensearch", "openrag"], # Audience
"exp": now + timedelta(days=7), # Expiration
"iat": now, # Issued at
"auth_time": int(now.timestamp()), # Authentication time
@ -120,7 +120,7 @@ class SessionManager:
"name": user.name,
"preferred_username": user.email,
"email_verified": True,
"roles": ["gendb_user"] # Backend role for OpenSearch
"roles": ["openrag_user"] # Backend role for OpenSearch
}
token = jwt.encode(token_payload, self.private_key, algorithm="RS256")
@ -133,7 +133,7 @@ class SessionManager:
token,
self.public_key,
algorithms=["RS256"],
audience=["opensearch", "gendb"]
audience=["opensearch", "openrag"]
)
return payload
except jwt.ExpiredSignatureError:

78
uv.lock generated
View file

@ -483,45 +483,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/bb/61/78c7b3851add1481b048b5fdc29067397a1784e2910592bc81bb3f608635/fsspec-2025.5.1-py3-none-any.whl", hash = "sha256:24d3a2e663d5fc735ab256263c4075f374a174c3410c0b25e5bd1970bceaa462", size = 199052, upload-time = "2025-05-24T12:03:21.66Z" },
]
[[package]]
name = "gendb"
version = "0.1.0"
source = { virtual = "." }
dependencies = [
{ name = "agentd" },
{ name = "aiofiles" },
{ name = "cryptography" },
{ name = "docling" },
{ name = "google-api-python-client" },
{ name = "google-auth-httplib2" },
{ name = "google-auth-oauthlib" },
{ name = "httpx" },
{ name = "opensearch-py", extra = ["async"] },
{ name = "pyjwt" },
{ name = "python-multipart" },
{ name = "starlette" },
{ name = "torch" },
{ name = "uvicorn" },
]
[package.metadata]
requires-dist = [
{ name = "agentd", specifier = ">=0.2.2" },
{ name = "aiofiles", specifier = ">=24.1.0" },
{ name = "cryptography", specifier = ">=45.0.6" },
{ name = "docling", specifier = ">=2.41.0" },
{ name = "google-api-python-client", specifier = ">=2.143.0" },
{ name = "google-auth-httplib2", specifier = ">=0.2.0" },
{ name = "google-auth-oauthlib", specifier = ">=1.2.0" },
{ name = "httpx", specifier = ">=0.27.0" },
{ name = "opensearch-py", extras = ["async"], specifier = ">=3.0.0" },
{ name = "pyjwt", specifier = ">=2.8.0" },
{ name = "python-multipart", specifier = ">=0.0.20" },
{ name = "starlette", specifier = ">=0.47.1" },
{ name = "torch", specifier = ">=2.7.1", index = "https://download.pytorch.org/whl/cu128" },
{ name = "uvicorn", specifier = ">=0.35.0" },
]
[[package]]
name = "google-api-core"
version = "2.25.1"
@ -1354,6 +1315,45 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" },
]
[[package]]
name = "openrag"
version = "0.1.0"
source = { virtual = "." }
dependencies = [
{ name = "agentd" },
{ name = "aiofiles" },
{ name = "cryptography" },
{ name = "docling" },
{ name = "google-api-python-client" },
{ name = "google-auth-httplib2" },
{ name = "google-auth-oauthlib" },
{ name = "httpx" },
{ name = "opensearch-py", extra = ["async"] },
{ name = "pyjwt" },
{ name = "python-multipart" },
{ name = "starlette" },
{ name = "torch" },
{ name = "uvicorn" },
]
[package.metadata]
requires-dist = [
{ name = "agentd", specifier = ">=0.2.2" },
{ name = "aiofiles", specifier = ">=24.1.0" },
{ name = "cryptography", specifier = ">=45.0.6" },
{ name = "docling", specifier = ">=2.41.0" },
{ name = "google-api-python-client", specifier = ">=2.143.0" },
{ name = "google-auth-httplib2", specifier = ">=0.2.0" },
{ name = "google-auth-oauthlib", specifier = ">=1.2.0" },
{ name = "httpx", specifier = ">=0.27.0" },
{ name = "opensearch-py", extras = ["async"], specifier = ">=3.0.0" },
{ name = "pyjwt", specifier = ">=2.8.0" },
{ name = "python-multipart", specifier = ">=0.0.20" },
{ name = "starlette", specifier = ">=0.47.1" },
{ name = "torch", specifier = ">=2.7.1", index = "https://download.pytorch.org/whl/cu128" },
{ name = "uvicorn", specifier = ">=0.35.0" },
]
[[package]]
name = "opensearch-py"
version = "3.0.0"