refactor to openrag and knowledge filters
This commit is contained in:
parent
125a6f0cbe
commit
346b938d98
25 changed files with 390 additions and 2629 deletions
|
|
@ -1,6 +1,6 @@
|
|||
### gendb
|
||||
### OpenRAG
|
||||
|
||||
[](https://deepwiki.com/phact/gendb)
|
||||
[](https://deepwiki.com/phact/openrag)
|
||||
|
||||
docker compose build
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ services:
|
|||
dockerfile: Dockerfile
|
||||
container_name: os
|
||||
depends_on:
|
||||
- gendb-backend
|
||||
- openrag-backend
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}
|
||||
|
|
@ -37,12 +37,12 @@ services:
|
|||
ports:
|
||||
- "5601:5601"
|
||||
|
||||
gendb-backend:
|
||||
image: phact/gendb-backend:latest
|
||||
openrag-backend:
|
||||
image: phact/openrag-backend:latest
|
||||
#build:
|
||||
#context: .
|
||||
#dockerfile: Dockerfile.backend
|
||||
container_name: gendb-backend
|
||||
container_name: openrag-backend
|
||||
depends_on:
|
||||
- langflow
|
||||
environment:
|
||||
|
|
@ -67,16 +67,16 @@ services:
|
|||
gpus: all
|
||||
platform: linux/amd64
|
||||
|
||||
gendb-frontend:
|
||||
image: phact/gendb-frontend:latest
|
||||
openrag-frontend:
|
||||
image: phact/openrag-frontend:latest
|
||||
#build:
|
||||
#context: .
|
||||
#dockerfile: Dockerfile.frontend
|
||||
container_name: gendb-frontend
|
||||
container_name: openrag-frontend
|
||||
depends_on:
|
||||
- gendb-backend
|
||||
- openrag-backend
|
||||
environment:
|
||||
- GENDB_BACKEND_HOST=gendb-backend
|
||||
- OPENRAG_BACKEND_HOST=openrag-backend
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
|
|
@ -94,6 +94,6 @@ services:
|
|||
- LANGFLOW_LOAD_FLOWS_PATH=/app/flows
|
||||
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
|
||||
- JWT="dummy"
|
||||
- GENDB-QUERY-FILTER="{}"
|
||||
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT
|
||||
- OPENRAG-QUERY-FILTER="{}"
|
||||
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER
|
||||
- LANGFLOW_LOG_LEVEL=DEBUG
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -17,7 +17,7 @@ export function NavigationLayout({ children }: NavigationLayoutProps) {
|
|||
<div className="container flex h-14 max-w-screen-2xl items-center">
|
||||
<div className="mr-4 hidden md:flex">
|
||||
<h1 className="text-lg font-semibold tracking-tight">
|
||||
GenDB
|
||||
OpenRAG
|
||||
</h1>
|
||||
</div>
|
||||
<div className="flex flex-1 items-center justify-between space-x-2 md:justify-end">
|
||||
|
|
|
|||
|
|
@ -28,10 +28,10 @@ export function Navigation() {
|
|||
active: pathname === "/chat",
|
||||
},
|
||||
{
|
||||
label: "Contexts",
|
||||
label: "Knowledge Filters",
|
||||
icon: BookOpenCheck,
|
||||
href: "/contexts",
|
||||
active: pathname.startsWith("/contexts"),
|
||||
href: "/knowledge-filters",
|
||||
active: pathname.startsWith("/knowledge-filters"),
|
||||
},
|
||||
{
|
||||
label: "Connectors",
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ async function proxyRequest(
|
|||
request: NextRequest,
|
||||
params: { path: string[] }
|
||||
) {
|
||||
const backendHost = process.env.GENDB_BACKEND_HOST || 'localhost';
|
||||
const backendHost = process.env.OPENRAG_BACKEND_HOST || 'localhost';
|
||||
const path = params.path.join('/');
|
||||
const searchParams = request.nextUrl.searchParams.toString();
|
||||
const backendUrl = `http://${backendHost}:8000/${path}${searchParams ? `?${searchParams}` : ''}`;
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ function AuthCallbackContent() {
|
|||
return isAppAuth ? "Signing you in..." : "Connecting..."
|
||||
}
|
||||
if (status === "success") {
|
||||
return isAppAuth ? "Welcome to GenDB!" : "Connection Successful!"
|
||||
return isAppAuth ? "Welcome to OpenRAG!" : "Connection Successful!"
|
||||
}
|
||||
if (status === "error") {
|
||||
return isAppAuth ? "Sign In Failed" : "Connection Failed"
|
||||
|
|
|
|||
|
|
@ -89,17 +89,17 @@ function ChatPage() {
|
|||
const [scoreThreshold, setScoreThreshold] = useState(0)
|
||||
const [loadedContextName, setLoadedContextName] = useState<string | null>(null)
|
||||
|
||||
// Load context if contextId is provided in URL
|
||||
// Load knowledge filter if filterId is provided in URL
|
||||
useEffect(() => {
|
||||
const contextId = searchParams.get('contextId')
|
||||
if (contextId) {
|
||||
loadContext(contextId)
|
||||
const filterId = searchParams.get('filterId')
|
||||
if (filterId) {
|
||||
loadKnowledgeFilter(filterId)
|
||||
}
|
||||
}, [searchParams])
|
||||
|
||||
const loadContext = async (contextId: string) => {
|
||||
const loadKnowledgeFilter = async (filterId: string) => {
|
||||
try {
|
||||
const response = await fetch(`/api/contexts/${contextId}`, {
|
||||
const response = await fetch(`/api/knowledge-filter/${filterId}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
|
|
@ -108,19 +108,19 @@ function ChatPage() {
|
|||
|
||||
const result = await response.json()
|
||||
if (response.ok && result.success) {
|
||||
const context = result.context
|
||||
const parsedQueryData = JSON.parse(context.query_data)
|
||||
const filter = result.filter
|
||||
const parsedQueryData = JSON.parse(filter.query_data)
|
||||
|
||||
// Load the context data into state
|
||||
setSelectedFilters(parsedQueryData.filters)
|
||||
setResultLimit(parsedQueryData.limit)
|
||||
setScoreThreshold(parsedQueryData.scoreThreshold)
|
||||
setLoadedContextName(context.name)
|
||||
setLoadedContextName(filter.name)
|
||||
} else {
|
||||
console.error("Failed to load context:", result.error)
|
||||
console.error("Failed to load knowledge filter:", result.error)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error loading context:", error)
|
||||
console.error("Error loading knowledge filter:", error)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -283,10 +283,14 @@ function ChatPage() {
|
|||
const apiEndpoint = endpoint === "chat" ? "/api/chat" : "/api/langflow"
|
||||
|
||||
try {
|
||||
const hasFilters = selectedFilters.data_sources.length > 0 ||
|
||||
selectedFilters.document_types.length > 0 ||
|
||||
selectedFilters.owners.length > 0
|
||||
|
||||
const requestBody: RequestBody = {
|
||||
prompt: userMessage.content,
|
||||
stream: true,
|
||||
filters: selectedFilters,
|
||||
...(hasFilters && { filters: selectedFilters }),
|
||||
limit: resultLimit,
|
||||
scoreThreshold: scoreThreshold
|
||||
}
|
||||
|
|
@ -744,9 +748,13 @@ function ChatPage() {
|
|||
try {
|
||||
const apiEndpoint = endpoint === "chat" ? "/api/chat" : "/api/langflow"
|
||||
|
||||
const hasFilters = selectedFilters.data_sources.length > 0 ||
|
||||
selectedFilters.document_types.length > 0 ||
|
||||
selectedFilters.owners.length > 0
|
||||
|
||||
const requestBody: RequestBody = {
|
||||
prompt: userMessage.content,
|
||||
filters: selectedFilters,
|
||||
...(hasFilters && { filters: selectedFilters }),
|
||||
limit: resultLimit,
|
||||
scoreThreshold: scoreThreshold
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { Label } from "@/components/ui/label"
|
|||
import { Search, Loader2, BookOpenCheck, Settings, Calendar, MessageCircle } from "lucide-react"
|
||||
import { ProtectedRoute } from "@/components/protected-route"
|
||||
|
||||
interface Context {
|
||||
interface KnowledgeFilter {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
|
|
@ -30,18 +30,18 @@ interface ParsedQueryData {
|
|||
scoreThreshold: number
|
||||
}
|
||||
|
||||
function ContextsPage() {
|
||||
function KnowledgeFiltersPage() {
|
||||
const router = useRouter()
|
||||
const [contexts, setContexts] = useState<Context[]>([])
|
||||
const [filters, setFilters] = useState<KnowledgeFilter[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [searchQuery, setSearchQuery] = useState("")
|
||||
const [selectedContext, setSelectedContext] = useState<Context | null>(null)
|
||||
const [selectedFilter, setSelectedFilter] = useState<KnowledgeFilter | null>(null)
|
||||
const [parsedQueryData, setParsedQueryData] = useState<ParsedQueryData | null>(null)
|
||||
|
||||
const loadContexts = async (query = "") => {
|
||||
const loadFilters = async (query = "") => {
|
||||
setLoading(true)
|
||||
try {
|
||||
const response = await fetch("/api/contexts/search", {
|
||||
const response = await fetch("/api/knowledge-filter/search", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
|
|
@ -54,32 +54,32 @@ function ContextsPage() {
|
|||
|
||||
const result = await response.json()
|
||||
if (response.ok && result.success) {
|
||||
setContexts(result.contexts)
|
||||
setFilters(result.filters)
|
||||
} else {
|
||||
console.error("Failed to load contexts:", result.error)
|
||||
setContexts([])
|
||||
console.error("Failed to load knowledge filters:", result.error)
|
||||
setFilters([])
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error loading contexts:", error)
|
||||
setContexts([])
|
||||
console.error("Error loading knowledge filters:", error)
|
||||
setFilters([])
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
loadContexts()
|
||||
loadFilters()
|
||||
}, [])
|
||||
|
||||
const handleSearch = async (e: React.FormEvent) => {
|
||||
e.preventDefault()
|
||||
await loadContexts(searchQuery)
|
||||
await loadFilters(searchQuery)
|
||||
}
|
||||
|
||||
const handleContextClick = (context: Context) => {
|
||||
setSelectedContext(context)
|
||||
const handleFilterClick = (filter: KnowledgeFilter) => {
|
||||
setSelectedFilter(filter)
|
||||
try {
|
||||
const parsed = JSON.parse(context.query_data) as ParsedQueryData
|
||||
const parsed = JSON.parse(filter.query_data) as ParsedQueryData
|
||||
setParsedQueryData(parsed)
|
||||
} catch (error) {
|
||||
console.error("Error parsing query data:", error)
|
||||
|
|
@ -97,16 +97,16 @@ function ContextsPage() {
|
|||
})
|
||||
}
|
||||
|
||||
const handleSearchWithContext = () => {
|
||||
if (!selectedContext) return
|
||||
const handleSearchWithFilter = () => {
|
||||
if (!selectedFilter) return
|
||||
|
||||
router.push(`/?contextId=${selectedContext.id}`)
|
||||
router.push(`/?filterId=${selectedFilter.id}`)
|
||||
}
|
||||
|
||||
const handleChatWithContext = () => {
|
||||
if (!selectedContext) return
|
||||
const handleChatWithFilter = () => {
|
||||
if (!selectedFilter) return
|
||||
|
||||
router.push(`/chat?contextId=${selectedContext.id}`)
|
||||
router.push(`/chat?filterId=${selectedFilter.id}`)
|
||||
}
|
||||
|
||||
return (
|
||||
|
|
@ -115,14 +115,14 @@ function ContextsPage() {
|
|||
<div className="space-y-4">
|
||||
<div className="mb-4">
|
||||
<h1 className="text-4xl font-bold tracking-tight text-white">
|
||||
Contexts
|
||||
Knowledge Filters
|
||||
</h1>
|
||||
</div>
|
||||
<p className="text-xl text-muted-foreground">
|
||||
Manage your saved search contexts
|
||||
Manage your saved knowledge filters
|
||||
</p>
|
||||
<p className="text-sm text-muted-foreground max-w-2xl">
|
||||
View and manage your saved search queries, filters, and configurations for quick access to your most important searches.
|
||||
View and manage your saved search configurations that help you focus on specific subsets of your knowledge base.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
|
@ -131,10 +131,10 @@ function ContextsPage() {
|
|||
<CardHeader>
|
||||
<CardTitle className="flex items-center gap-2">
|
||||
<BookOpenCheck className="h-5 w-5" />
|
||||
Search Contexts
|
||||
Search Knowledge Filters
|
||||
</CardTitle>
|
||||
<CardDescription>
|
||||
Search through your saved search contexts by name or description
|
||||
Search through your saved knowledge filters by name or description
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-6">
|
||||
|
|
@ -142,7 +142,7 @@ function ContextsPage() {
|
|||
<div className="flex gap-2">
|
||||
<Input
|
||||
type="text"
|
||||
placeholder="Search contexts..."
|
||||
placeholder="Search knowledge filters..."
|
||||
value={searchQuery}
|
||||
onChange={(e) => setSearchQuery(e.target.value)}
|
||||
className="h-12 bg-background/50 border-border/50 focus:border-blue-400/50 focus:ring-blue-400/20 flex-1"
|
||||
|
|
@ -170,7 +170,7 @@ function ContextsPage() {
|
|||
<div className="flex gap-6">
|
||||
{/* Context List */}
|
||||
<div className="flex-1 space-y-4">
|
||||
{contexts.length === 0 ? (
|
||||
{filters.length === 0 ? (
|
||||
<Card className="bg-muted/20 border-dashed border-muted-foreground/30">
|
||||
<CardContent className="pt-8 pb-8">
|
||||
<div className="text-center space-y-3">
|
||||
|
|
@ -178,40 +178,40 @@ function ContextsPage() {
|
|||
<BookOpenCheck className="h-8 w-8 text-muted-foreground/50" />
|
||||
</div>
|
||||
<p className="text-lg font-medium text-muted-foreground">
|
||||
No contexts found
|
||||
No knowledge filters found
|
||||
</p>
|
||||
<p className="text-sm text-muted-foreground/70 max-w-md mx-auto">
|
||||
Create your first context by saving a search configuration from the search page.
|
||||
Create your first knowledge filter by saving a search configuration from the search page.
|
||||
</p>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
) : (
|
||||
<div className="space-y-3">
|
||||
{contexts.map((context) => (
|
||||
{filters.map((filter) => (
|
||||
<Card
|
||||
key={context.id}
|
||||
key={filter.id}
|
||||
className={`bg-card/50 backdrop-blur-sm border-border/50 hover:bg-card/70 transition-all duration-200 hover:shadow-lg hover:shadow-blue-500/10 cursor-pointer ${
|
||||
selectedContext?.id === context.id ? 'ring-2 ring-blue-500/50 bg-card/70' : ''
|
||||
selectedFilter?.id === filter.id ? 'ring-2 ring-blue-500/50 bg-card/70' : ''
|
||||
}`}
|
||||
onClick={() => handleContextClick(context)}
|
||||
onClick={() => handleFilterClick(filter)}
|
||||
>
|
||||
<CardContent className="p-4">
|
||||
<div className="flex items-start justify-between">
|
||||
<div className="flex-1 space-y-2">
|
||||
<h3 className="font-semibold text-lg">{context.name}</h3>
|
||||
{context.description && (
|
||||
<p className="text-sm text-muted-foreground">{context.description}</p>
|
||||
<h3 className="font-semibold text-lg">{filter.name}</h3>
|
||||
{filter.description && (
|
||||
<p className="text-sm text-muted-foreground">{filter.description}</p>
|
||||
)}
|
||||
<div className="flex items-center gap-4 text-xs text-muted-foreground">
|
||||
<div className="flex items-center gap-1">
|
||||
<Calendar className="h-3 w-3" />
|
||||
<span>Created {formatDate(context.created_at)}</span>
|
||||
<span>Created {formatDate(filter.created_at)}</span>
|
||||
</div>
|
||||
{context.updated_at !== context.created_at && (
|
||||
{filter.updated_at !== filter.created_at && (
|
||||
<div className="flex items-center gap-1">
|
||||
<Calendar className="h-3 w-3" />
|
||||
<span>Updated {formatDate(context.updated_at)}</span>
|
||||
<span>Updated {formatDate(filter.updated_at)}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -225,12 +225,12 @@ function ContextsPage() {
|
|||
</div>
|
||||
|
||||
{/* Context Detail Panel */}
|
||||
{selectedContext && parsedQueryData && (
|
||||
{selectedFilter && parsedQueryData && (
|
||||
<div className="w-64 space-y-6 flex-shrink-0">
|
||||
<div className="flex items-center justify-between">
|
||||
<h2 className="text-lg font-semibold flex items-center gap-2">
|
||||
<Settings className="h-5 w-5" />
|
||||
Context Details
|
||||
Knowledge Filter Details
|
||||
</h2>
|
||||
</div>
|
||||
|
||||
|
|
@ -311,21 +311,21 @@ function ContextsPage() {
|
|||
{/* Action Buttons */}
|
||||
<div className="space-y-2 pt-4 border-t border-border/50">
|
||||
<Button
|
||||
onClick={handleSearchWithContext}
|
||||
onClick={handleSearchWithFilter}
|
||||
className="w-full flex items-center gap-2"
|
||||
variant="default"
|
||||
>
|
||||
<Search className="h-4 w-4" />
|
||||
Search with Context
|
||||
Search with Filter
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
onClick={handleChatWithContext}
|
||||
onClick={handleChatWithFilter}
|
||||
className="w-full flex items-center gap-2"
|
||||
variant="outline"
|
||||
>
|
||||
<MessageCircle className="h-4 w-4" />
|
||||
Chat with Context
|
||||
Chat with Filter
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -338,10 +338,10 @@ function ContextsPage() {
|
|||
)
|
||||
}
|
||||
|
||||
export default function ProtectedContextsPage() {
|
||||
export default function ProtectedKnowledgeFiltersPage() {
|
||||
return (
|
||||
<ProtectedRoute>
|
||||
<ContextsPage />
|
||||
<KnowledgeFiltersPage />
|
||||
</ProtectedRoute>
|
||||
)
|
||||
}
|
||||
|
|
@ -18,8 +18,8 @@ const geistMono = Geist_Mono({
|
|||
});
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "GenDB",
|
||||
description: "Document search and management system",
|
||||
title: "OpenRAG",
|
||||
description: "Open source RAG (Retrieval Augmented Generation) system",
|
||||
};
|
||||
|
||||
export default function RootLayout({
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ function LoginPageContent() {
|
|||
<Lock className="h-8 w-8 text-primary" />
|
||||
</div>
|
||||
<div>
|
||||
<CardTitle className="text-2xl">Welcome to GenDB</CardTitle>
|
||||
<CardTitle className="text-2xl">Welcome to OpenRAG</CardTitle>
|
||||
<CardDescription className="mt-2">
|
||||
Sign in to access your documents and AI chat
|
||||
</CardDescription>
|
||||
|
|
|
|||
|
|
@ -88,48 +88,7 @@ function SearchPage() {
|
|||
const [savingContext, setSavingContext] = useState(false)
|
||||
const [loadedContextName, setLoadedContextName] = useState<string | null>(null)
|
||||
|
||||
const loadContext = useCallback(async (contextId: string) => {
|
||||
try {
|
||||
const response = await fetch(`/api/contexts/${contextId}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
})
|
||||
|
||||
const result = await response.json()
|
||||
if (response.ok && result.success) {
|
||||
const context = result.context
|
||||
const parsedQueryData = JSON.parse(context.query_data)
|
||||
|
||||
// Load the context data into state
|
||||
setQuery(parsedQueryData.query)
|
||||
setSelectedFilters(parsedQueryData.filters)
|
||||
setResultLimit(parsedQueryData.limit)
|
||||
setScoreThreshold(parsedQueryData.scoreThreshold)
|
||||
setLoadedContextName(context.name)
|
||||
|
||||
// Automatically perform the search
|
||||
setTimeout(() => {
|
||||
handleSearch()
|
||||
}, 100)
|
||||
} else {
|
||||
console.error("Failed to load context:", result.error)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error loading context:", err)
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Load context if contextId is provided in URL
|
||||
useEffect(() => {
|
||||
const contextId = searchParams.get('contextId')
|
||||
if (contextId) {
|
||||
loadContext(contextId)
|
||||
}
|
||||
}, [searchParams, loadContext])
|
||||
|
||||
const handleSearch = async (e?: React.FormEvent) => {
|
||||
const handleSearch = useCallback(async (e?: React.FormEvent) => {
|
||||
if (e) e.preventDefault()
|
||||
if (!query.trim()) return
|
||||
|
||||
|
|
@ -202,7 +161,48 @@ function SearchPage() {
|
|||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
}, [query, resultLimit, scoreThreshold, searchPerformed, selectedFilters])
|
||||
|
||||
const loadKnowledgeFilter = useCallback(async (filterId: string) => {
|
||||
try {
|
||||
const response = await fetch(`/api/knowledge-filter/${filterId}`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
})
|
||||
|
||||
const result = await response.json()
|
||||
if (response.ok && result.success) {
|
||||
const filter = result.filter
|
||||
const parsedQueryData = JSON.parse(filter.query_data)
|
||||
|
||||
// Load the context data into state
|
||||
setQuery(parsedQueryData.query)
|
||||
setSelectedFilters(parsedQueryData.filters)
|
||||
setResultLimit(parsedQueryData.limit)
|
||||
setScoreThreshold(parsedQueryData.scoreThreshold)
|
||||
setLoadedContextName(filter.name)
|
||||
|
||||
// Automatically perform the search
|
||||
setTimeout(() => {
|
||||
handleSearch()
|
||||
}, 100)
|
||||
} else {
|
||||
console.error("Failed to load knowledge filter:", result.error)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error loading knowledge filter:", err)
|
||||
}
|
||||
}, [handleSearch])
|
||||
|
||||
// Load knowledge filter if filterId is provided in URL
|
||||
useEffect(() => {
|
||||
const filterId = searchParams.get('filterId')
|
||||
if (filterId) {
|
||||
loadKnowledgeFilter(filterId)
|
||||
}
|
||||
}, [searchParams, loadKnowledgeFilter])
|
||||
|
||||
const handleFilterChange = async (facetType: keyof SelectedFilters, value: string, checked: boolean) => {
|
||||
const newFilters = {
|
||||
|
|
@ -277,16 +277,16 @@ function SearchPage() {
|
|||
selectedFilters.owners.length
|
||||
}
|
||||
|
||||
const handleSaveContext = async () => {
|
||||
const contextId = searchParams.get('contextId')
|
||||
const handleSaveKnowledgeFilter = async () => {
|
||||
const filterId = searchParams.get('filterId')
|
||||
|
||||
// If no contextId present and no title, we need the modal
|
||||
if (!contextId && !contextTitle.trim()) return
|
||||
// If no filterId present and no title, we need the modal
|
||||
if (!filterId && !contextTitle.trim()) return
|
||||
|
||||
setSavingContext(true)
|
||||
|
||||
try {
|
||||
const contextData = {
|
||||
const filterData = {
|
||||
query,
|
||||
filters: selectedFilters,
|
||||
limit: resultLimit,
|
||||
|
|
@ -295,20 +295,20 @@ function SearchPage() {
|
|||
|
||||
let response;
|
||||
|
||||
if (contextId) {
|
||||
// Update existing context (upsert)
|
||||
response = await fetch(`/api/contexts/${contextId}`, {
|
||||
if (filterId) {
|
||||
// Update existing knowledge filter (upsert)
|
||||
response = await fetch(`/api/knowledge-filter/${filterId}`, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
queryData: JSON.stringify(contextData)
|
||||
queryData: JSON.stringify(filterData)
|
||||
}),
|
||||
})
|
||||
} else {
|
||||
// Create new context
|
||||
response = await fetch("/api/contexts", {
|
||||
// Create new knowledge filter
|
||||
response = await fetch("/api/knowledge-filter", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
|
|
@ -316,7 +316,7 @@ function SearchPage() {
|
|||
body: JSON.stringify({
|
||||
name: contextTitle,
|
||||
description: contextDescription,
|
||||
queryData: JSON.stringify(contextData)
|
||||
queryData: JSON.stringify(filterData)
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
|
@ -324,18 +324,18 @@ function SearchPage() {
|
|||
const result = await response.json()
|
||||
|
||||
if (response.ok && result.success) {
|
||||
if (!contextId) {
|
||||
// Reset modal state only if we were creating a new context
|
||||
if (!filterId) {
|
||||
// Reset modal state only if we were creating a new knowledge filter
|
||||
setShowSaveModal(false)
|
||||
setContextTitle("")
|
||||
setContextDescription("")
|
||||
}
|
||||
toast.success(contextId ? "Context updated successfully" : "Context saved successfully")
|
||||
toast.success(filterId ? "Knowledge filter updated successfully" : "Knowledge filter saved successfully")
|
||||
} else {
|
||||
toast.error(contextId ? "Failed to update context" : "Failed to save context")
|
||||
toast.error(filterId ? "Failed to update knowledge filter" : "Failed to save knowledge filter")
|
||||
}
|
||||
} catch {
|
||||
toast.error(contextId ? "Error updating context" : "Error saving context")
|
||||
toast.error(filterId ? "Error updating knowledge filter" : "Error saving knowledge filter")
|
||||
} finally {
|
||||
setSavingContext(false)
|
||||
}
|
||||
|
|
@ -843,13 +843,13 @@ function SearchPage() {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{/* Save Context Button */}
|
||||
{/* Save Knowledge Filter Button */}
|
||||
<div className="pt-4 border-t border-border/50">
|
||||
<Button
|
||||
onClick={() => {
|
||||
const contextId = searchParams.get('contextId')
|
||||
if (contextId) {
|
||||
handleSaveContext()
|
||||
const filterId = searchParams.get('filterId')
|
||||
if (filterId) {
|
||||
handleSaveKnowledgeFilter()
|
||||
} else {
|
||||
setShowSaveModal(true)
|
||||
}
|
||||
|
|
@ -861,12 +861,12 @@ function SearchPage() {
|
|||
{savingContext ? (
|
||||
<>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
{searchParams.get('contextId') ? 'Updating...' : 'Saving...'}
|
||||
{searchParams.get('filterId') ? 'Updating...' : 'Saving...'}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Save className="h-4 w-4" />
|
||||
{searchParams.get('contextId') ? 'Update Context' : 'Save Context'}
|
||||
{searchParams.get('filterId') ? 'Update Knowledge Filter' : 'Save Knowledge Filter'}
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
|
|
@ -938,7 +938,7 @@ function SearchPage() {
|
|||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
onClick={handleSaveContext}
|
||||
onClick={handleSaveKnowledgeFilter}
|
||||
disabled={!contextTitle.trim() || savingContext}
|
||||
className="flex items-center gap-2"
|
||||
>
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {
|
|||
<div className="flex h-14 items-center px-4">
|
||||
<div className="flex items-center">
|
||||
<h1 className="text-lg font-semibold tracking-tight text-white">
|
||||
GenDB
|
||||
OpenRAG
|
||||
</h1>
|
||||
</div>
|
||||
<div className="flex flex-1 items-center justify-end space-x-2">
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[project]
|
||||
name = "gendb"
|
||||
name = "openrag"
|
||||
version = "0.1.0"
|
||||
description = "Add your description here"
|
||||
readme = "README.md"
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ config:
|
|||
type: openid
|
||||
challenge: false
|
||||
config:
|
||||
openid_connect_url: "http://gendb-backend:8000/.well-known/openid-configuration"
|
||||
openid_connect_url: "http://openrag-backend:8000/.well-known/openid-configuration"
|
||||
subject_key: "sub"
|
||||
jwt_header: "Authorization" # expects Bearer token
|
||||
roles_key: "roles"
|
||||
|
|
|
|||
|
|
@ -2,13 +2,13 @@ _meta:
|
|||
type: "roles"
|
||||
config_version: 2
|
||||
|
||||
gendb_user_role:
|
||||
openrag_user_role:
|
||||
description: "DLS: user can read/write docs they own or are allowed on"
|
||||
cluster_permissions:
|
||||
- "indices:data/write/bulk"
|
||||
- "indices:data/write/index"
|
||||
index_permissions:
|
||||
- index_patterns: ["documents", "documents*", "search_contexts", "search_contexts*"]
|
||||
- index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*"]
|
||||
allowed_actions:
|
||||
- crud
|
||||
- create_index
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ _meta:
|
|||
type: "rolesmapping"
|
||||
config_version: 2
|
||||
|
||||
gendb_user_role:
|
||||
openrag_user_role:
|
||||
users: []
|
||||
hosts: []
|
||||
backend_roles:
|
||||
- "gendb_user"
|
||||
- "openrag_user"
|
||||
|
||||
all_access:
|
||||
users:
|
||||
|
|
|
|||
|
|
@ -1,114 +0,0 @@
|
|||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
async def create_context(request: Request, contexts_service, session_manager):
|
||||
"""Create a new search context"""
|
||||
payload = await request.json()
|
||||
|
||||
name = payload.get("name")
|
||||
if not name:
|
||||
return JSONResponse({"error": "Context name is required"}, status_code=400)
|
||||
|
||||
description = payload.get("description", "")
|
||||
query_data = payload.get("queryData")
|
||||
if not query_data:
|
||||
return JSONResponse({"error": "Query data is required"}, status_code=400)
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
# Create context document
|
||||
context_id = str(uuid.uuid4())
|
||||
context_doc = {
|
||||
"id": context_id,
|
||||
"name": name,
|
||||
"description": description,
|
||||
"query_data": query_data, # Store the full search query JSON
|
||||
"owner": user.user_id,
|
||||
"allowed_users": payload.get("allowedUsers", []), # ACL field for future use
|
||||
"allowed_groups": payload.get("allowedGroups", []), # ACL field for future use
|
||||
"created_at": datetime.utcnow().isoformat(),
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
result = await contexts_service.create_context(context_doc, user_id=user.user_id, jwt_token=jwt_token)
|
||||
return JSONResponse(result)
|
||||
|
||||
async def search_contexts(request: Request, contexts_service, session_manager):
|
||||
"""Search for contexts by name, description, or query content"""
|
||||
payload = await request.json()
|
||||
|
||||
query = payload.get("query", "")
|
||||
limit = payload.get("limit", 20)
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
result = await contexts_service.search_contexts(query, user_id=user.user_id, jwt_token=jwt_token, limit=limit)
|
||||
return JSONResponse(result)
|
||||
|
||||
async def get_context(request: Request, contexts_service, session_manager):
|
||||
"""Get a specific context by ID"""
|
||||
context_id = request.path_params.get("context_id")
|
||||
if not context_id:
|
||||
return JSONResponse({"error": "Context ID is required"}, status_code=400)
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
result = await contexts_service.get_context(context_id, user_id=user.user_id, jwt_token=jwt_token)
|
||||
return JSONResponse(result)
|
||||
|
||||
async def update_context(request: Request, contexts_service, session_manager):
|
||||
"""Update an existing context by delete + recreate (due to DLS limitations)"""
|
||||
context_id = request.path_params.get("context_id")
|
||||
if not context_id:
|
||||
return JSONResponse({"error": "Context ID is required"}, status_code=400)
|
||||
|
||||
payload = await request.json()
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
# First, get the existing context
|
||||
existing_result = await contexts_service.get_context(context_id, user_id=user.user_id, jwt_token=jwt_token)
|
||||
if not existing_result.get("success"):
|
||||
return JSONResponse({"error": "Context not found or access denied"}, status_code=404)
|
||||
|
||||
existing_context = existing_result["context"]
|
||||
|
||||
# Delete the existing context
|
||||
delete_result = await contexts_service.delete_context(context_id, user_id=user.user_id, jwt_token=jwt_token)
|
||||
if not delete_result.get("success"):
|
||||
return JSONResponse({"error": "Failed to delete existing context"}, status_code=500)
|
||||
|
||||
# Create updated context document with same ID
|
||||
updated_context = {
|
||||
"id": context_id,
|
||||
"name": payload.get("name", existing_context["name"]),
|
||||
"description": payload.get("description", existing_context["description"]),
|
||||
"query_data": payload.get("queryData", existing_context["query_data"]),
|
||||
"owner": existing_context["owner"],
|
||||
"allowed_users": payload.get("allowedUsers", existing_context.get("allowed_users", [])),
|
||||
"allowed_groups": payload.get("allowedGroups", existing_context.get("allowed_groups", [])),
|
||||
"created_at": existing_context["created_at"], # Preserve original creation time
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Recreate the context
|
||||
result = await contexts_service.create_context(updated_context, user_id=user.user_id, jwt_token=jwt_token)
|
||||
return JSONResponse(result)
|
||||
|
||||
async def delete_context(request: Request, contexts_service, session_manager):
|
||||
"""Delete a context"""
|
||||
context_id = request.path_params.get("context_id")
|
||||
if not context_id:
|
||||
return JSONResponse({"error": "Context ID is required"}, status_code=400)
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
result = await contexts_service.delete_context(context_id, user_id=user.user_id, jwt_token=jwt_token)
|
||||
return JSONResponse(result)
|
||||
114
src/api/knowledge_filter.py
Normal file
114
src/api/knowledge_filter.py
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
async def create_knowledge_filter(request: Request, knowledge_filter_service, session_manager):
|
||||
"""Create a new knowledge filter"""
|
||||
payload = await request.json()
|
||||
|
||||
name = payload.get("name")
|
||||
if not name:
|
||||
return JSONResponse({"error": "Knowledge filter name is required"}, status_code=400)
|
||||
|
||||
description = payload.get("description", "")
|
||||
query_data = payload.get("queryData")
|
||||
if not query_data:
|
||||
return JSONResponse({"error": "Query data is required"}, status_code=400)
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
# Create knowledge filter document
|
||||
filter_id = str(uuid.uuid4())
|
||||
filter_doc = {
|
||||
"id": filter_id,
|
||||
"name": name,
|
||||
"description": description,
|
||||
"query_data": query_data, # Store the full search query JSON
|
||||
"owner": user.user_id,
|
||||
"allowed_users": payload.get("allowedUsers", []), # ACL field for future use
|
||||
"allowed_groups": payload.get("allowedGroups", []), # ACL field for future use
|
||||
"created_at": datetime.utcnow().isoformat(),
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
result = await knowledge_filter_service.create_knowledge_filter(filter_doc, user_id=user.user_id, jwt_token=jwt_token)
|
||||
return JSONResponse(result)
|
||||
|
||||
async def search_knowledge_filters(request: Request, knowledge_filter_service, session_manager):
|
||||
"""Search for knowledge filters by name, description, or query content"""
|
||||
payload = await request.json()
|
||||
|
||||
query = payload.get("query", "")
|
||||
limit = payload.get("limit", 20)
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
result = await knowledge_filter_service.search_knowledge_filters(query, user_id=user.user_id, jwt_token=jwt_token, limit=limit)
|
||||
return JSONResponse(result)
|
||||
|
||||
async def get_knowledge_filter(request: Request, knowledge_filter_service, session_manager):
|
||||
"""Get a specific knowledge filter by ID"""
|
||||
filter_id = request.path_params.get("filter_id")
|
||||
if not filter_id:
|
||||
return JSONResponse({"error": "Knowledge filter ID is required"}, status_code=400)
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
result = await knowledge_filter_service.get_knowledge_filter(filter_id, user_id=user.user_id, jwt_token=jwt_token)
|
||||
return JSONResponse(result)
|
||||
|
||||
async def update_knowledge_filter(request: Request, knowledge_filter_service, session_manager):
|
||||
"""Update an existing knowledge filter by delete + recreate (due to DLS limitations)"""
|
||||
filter_id = request.path_params.get("filter_id")
|
||||
if not filter_id:
|
||||
return JSONResponse({"error": "Knowledge filter ID is required"}, status_code=400)
|
||||
|
||||
payload = await request.json()
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
# First, get the existing knowledge filter
|
||||
existing_result = await knowledge_filter_service.get_knowledge_filter(filter_id, user_id=user.user_id, jwt_token=jwt_token)
|
||||
if not existing_result.get("success"):
|
||||
return JSONResponse({"error": "Knowledge filter not found or access denied"}, status_code=404)
|
||||
|
||||
existing_filter = existing_result["filter"]
|
||||
|
||||
# Delete the existing knowledge filter
|
||||
delete_result = await knowledge_filter_service.delete_knowledge_filter(filter_id, user_id=user.user_id, jwt_token=jwt_token)
|
||||
if not delete_result.get("success"):
|
||||
return JSONResponse({"error": "Failed to delete existing knowledge filter"}, status_code=500)
|
||||
|
||||
# Create updated knowledge filter document with same ID
|
||||
updated_filter = {
|
||||
"id": filter_id,
|
||||
"name": payload.get("name", existing_filter["name"]),
|
||||
"description": payload.get("description", existing_filter["description"]),
|
||||
"query_data": payload.get("queryData", existing_filter["query_data"]),
|
||||
"owner": existing_filter["owner"],
|
||||
"allowed_users": payload.get("allowedUsers", existing_filter.get("allowed_users", [])),
|
||||
"allowed_groups": payload.get("allowedGroups", existing_filter.get("allowed_groups", [])),
|
||||
"created_at": existing_filter["created_at"], # Preserve original creation time
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Recreate the knowledge filter
|
||||
result = await knowledge_filter_service.create_knowledge_filter(updated_filter, user_id=user.user_id, jwt_token=jwt_token)
|
||||
return JSONResponse(result)
|
||||
|
||||
async def delete_knowledge_filter(request: Request, knowledge_filter_service, session_manager):
|
||||
"""Delete a knowledge filter"""
|
||||
filter_id = request.path_params.get("filter_id")
|
||||
if not filter_id:
|
||||
return JSONResponse({"error": "Knowledge filter ID is required"}, status_code=400)
|
||||
|
||||
user = request.state.user
|
||||
jwt_token = request.cookies.get("auth_token")
|
||||
|
||||
result = await knowledge_filter_service.delete_knowledge_filter(filter_id, user_id=user.user_id, jwt_token=jwt_token)
|
||||
return JSONResponse(result)
|
||||
|
|
@ -51,7 +51,7 @@ async def jwks_endpoint(request: Request, session_manager):
|
|||
"kty": "RSA",
|
||||
"use": "sig",
|
||||
"alg": "RS256",
|
||||
"kid": "gendb-key-1",
|
||||
"kid": "openrag-key-1",
|
||||
"n": int_to_base64url(public_numbers.n),
|
||||
"e": int_to_base64url(public_numbers.e)
|
||||
}
|
||||
|
|
|
|||
54
src/main.py
54
src/main.py
|
|
@ -23,7 +23,7 @@ from services.search_service import SearchService
|
|||
from services.task_service import TaskService
|
||||
from services.auth_service import AuthService
|
||||
from services.chat_service import ChatService
|
||||
from services.contexts_service import ContextsService
|
||||
from services.knowledge_filter_service import KnowledgeFilterService
|
||||
|
||||
# Existing services
|
||||
from connectors.service import ConnectorService
|
||||
|
|
@ -31,7 +31,7 @@ from session_manager import SessionManager
|
|||
from auth_middleware import require_auth, optional_auth
|
||||
|
||||
# API endpoints
|
||||
from api import upload, search, chat, auth, connectors, tasks, oidc, contexts
|
||||
from api import upload, search, chat, auth, connectors, tasks, oidc, knowledge_filter
|
||||
|
||||
print("CUDA available:", torch.cuda.is_available())
|
||||
print("CUDA version PyTorch was built with:", torch.version.cuda)
|
||||
|
|
@ -64,9 +64,9 @@ async def init_index():
|
|||
else:
|
||||
print(f"Index '{INDEX_NAME}' already exists, skipping creation.")
|
||||
|
||||
# Create contexts index
|
||||
contexts_index_name = "search_contexts"
|
||||
contexts_index_body = {
|
||||
# Create knowledge filters index
|
||||
knowledge_filter_index_name = "knowledge_filters"
|
||||
knowledge_filter_index_body = {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"id": {"type": "keyword"},
|
||||
|
|
@ -82,11 +82,11 @@ async def init_index():
|
|||
}
|
||||
}
|
||||
|
||||
if not await clients.opensearch.indices.exists(index=contexts_index_name):
|
||||
await clients.opensearch.indices.create(index=contexts_index_name, body=contexts_index_body)
|
||||
print(f"Created index '{contexts_index_name}'")
|
||||
if not await clients.opensearch.indices.exists(index=knowledge_filter_index_name):
|
||||
await clients.opensearch.indices.create(index=knowledge_filter_index_name, body=knowledge_filter_index_body)
|
||||
print(f"Created index '{knowledge_filter_index_name}'")
|
||||
else:
|
||||
print(f"Index '{contexts_index_name}' already exists, skipping creation.")
|
||||
print(f"Index '{knowledge_filter_index_name}' already exists, skipping creation.")
|
||||
|
||||
async def init_index_when_ready():
|
||||
"""Initialize OpenSearch index when it becomes available"""
|
||||
|
|
@ -111,7 +111,7 @@ def initialize_services():
|
|||
search_service = SearchService(session_manager)
|
||||
task_service = TaskService(document_service, process_pool)
|
||||
chat_service = ChatService()
|
||||
contexts_service = ContextsService(session_manager)
|
||||
knowledge_filter_service = KnowledgeFilterService(session_manager)
|
||||
|
||||
# Set process pool for document service
|
||||
document_service.process_pool = process_pool
|
||||
|
|
@ -136,7 +136,7 @@ def initialize_services():
|
|||
'chat_service': chat_service,
|
||||
'auth_service': auth_service,
|
||||
'connector_service': connector_service,
|
||||
'contexts_service': contexts_service,
|
||||
'knowledge_filter_service': knowledge_filter_service,
|
||||
'session_manager': session_manager
|
||||
}
|
||||
|
||||
|
|
@ -198,39 +198,39 @@ def create_app():
|
|||
session_manager=services['session_manager'])
|
||||
), methods=["POST"]),
|
||||
|
||||
# Contexts endpoints
|
||||
Route("/contexts",
|
||||
# Knowledge Filter endpoints
|
||||
Route("/knowledge-filter",
|
||||
require_auth(services['session_manager'])(
|
||||
partial(contexts.create_context,
|
||||
contexts_service=services['contexts_service'],
|
||||
partial(knowledge_filter.create_knowledge_filter,
|
||||
knowledge_filter_service=services['knowledge_filter_service'],
|
||||
session_manager=services['session_manager'])
|
||||
), methods=["POST"]),
|
||||
|
||||
Route("/contexts/search",
|
||||
Route("/knowledge-filter/search",
|
||||
require_auth(services['session_manager'])(
|
||||
partial(contexts.search_contexts,
|
||||
contexts_service=services['contexts_service'],
|
||||
partial(knowledge_filter.search_knowledge_filters,
|
||||
knowledge_filter_service=services['knowledge_filter_service'],
|
||||
session_manager=services['session_manager'])
|
||||
), methods=["POST"]),
|
||||
|
||||
Route("/contexts/{context_id}",
|
||||
Route("/knowledge-filter/{filter_id}",
|
||||
require_auth(services['session_manager'])(
|
||||
partial(contexts.get_context,
|
||||
contexts_service=services['contexts_service'],
|
||||
partial(knowledge_filter.get_knowledge_filter,
|
||||
knowledge_filter_service=services['knowledge_filter_service'],
|
||||
session_manager=services['session_manager'])
|
||||
), methods=["GET"]),
|
||||
|
||||
Route("/contexts/{context_id}",
|
||||
Route("/knowledge-filter/{filter_id}",
|
||||
require_auth(services['session_manager'])(
|
||||
partial(contexts.update_context,
|
||||
contexts_service=services['contexts_service'],
|
||||
partial(knowledge_filter.update_knowledge_filter,
|
||||
knowledge_filter_service=services['knowledge_filter_service'],
|
||||
session_manager=services['session_manager'])
|
||||
), methods=["PUT"]),
|
||||
|
||||
Route("/contexts/{context_id}",
|
||||
Route("/knowledge-filter/{filter_id}",
|
||||
require_auth(services['session_manager'])(
|
||||
partial(contexts.delete_context,
|
||||
contexts_service=services['contexts_service'],
|
||||
partial(knowledge_filter.delete_knowledge_filter,
|
||||
knowledge_filter_service=services['knowledge_filter_service'],
|
||||
session_manager=services['session_manager'])
|
||||
), methods=["DELETE"]),
|
||||
|
||||
|
|
|
|||
|
|
@ -77,8 +77,8 @@ class ChatService:
|
|||
|
||||
# Pass the complete filter expression as a single header to Langflow (only if we have something to send)
|
||||
if filter_expression:
|
||||
print(f"Sending GenDB query filter to Langflow: {json.dumps(filter_expression, indent=2)}")
|
||||
extra_headers['X-LANGFLOW-GLOBAL-VAR-GENDB-QUERY-FILTER'] = json.dumps(filter_expression)
|
||||
print(f"Sending OpenRAG query filter to Langflow: {json.dumps(filter_expression, indent=2)}")
|
||||
extra_headers['X-LANGFLOW-GLOBAL-VAR-OPENRAG-QUERY-FILTER'] = json.dumps(filter_expression)
|
||||
|
||||
if stream:
|
||||
return async_langflow_stream(clients.langflow_client, FLOW_ID, prompt, extra_headers=extra_headers, previous_response_id=previous_response_id)
|
||||
|
|
|
|||
|
|
@ -1,34 +1,34 @@
|
|||
from typing import Any, Dict, Optional
|
||||
|
||||
CONTEXTS_INDEX_NAME = "search_contexts"
|
||||
KNOWLEDGE_FILTERS_INDEX_NAME = "knowledge_filters"
|
||||
|
||||
class ContextsService:
|
||||
class KnowledgeFilterService:
|
||||
def __init__(self, session_manager=None):
|
||||
self.session_manager = session_manager
|
||||
|
||||
async def create_context(self, context_doc: Dict[str, Any], user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
|
||||
"""Create a new search context"""
|
||||
async def create_knowledge_filter(self, filter_doc: Dict[str, Any], user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
|
||||
"""Create a new knowledge filter"""
|
||||
try:
|
||||
# Get user's OpenSearch client with JWT for OIDC auth
|
||||
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
|
||||
|
||||
# Index the context document
|
||||
# Index the knowledge filter document
|
||||
result = await opensearch_client.index(
|
||||
index=CONTEXTS_INDEX_NAME,
|
||||
id=context_doc["id"],
|
||||
body=context_doc
|
||||
index=KNOWLEDGE_FILTERS_INDEX_NAME,
|
||||
id=filter_doc["id"],
|
||||
body=filter_doc
|
||||
)
|
||||
|
||||
if result.get("result") == "created":
|
||||
return {"success": True, "id": context_doc["id"], "context": context_doc}
|
||||
return {"success": True, "id": filter_doc["id"], "filter": filter_doc}
|
||||
else:
|
||||
return {"success": False, "error": "Failed to create context"}
|
||||
return {"success": False, "error": "Failed to create knowledge filter"}
|
||||
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
async def search_contexts(self, query: str, user_id: str = None, jwt_token: str = None, limit: int = 20) -> Dict[str, Any]:
|
||||
"""Search for contexts by name, description, or query content"""
|
||||
async def search_knowledge_filters(self, query: str, user_id: str = None, jwt_token: str = None, limit: int = 20) -> Dict[str, Any]:
|
||||
"""Search for knowledge filters by name, description, or query content"""
|
||||
try:
|
||||
# Get user's OpenSearch client with JWT for OIDC auth
|
||||
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
|
||||
|
|
@ -52,7 +52,7 @@ class ContextsService:
|
|||
"size": limit
|
||||
}
|
||||
else:
|
||||
# No query - return all contexts sorted by most recent
|
||||
# No query - return all knowledge filters sorted by most recent
|
||||
search_body = {
|
||||
"query": {"match_all": {}},
|
||||
"sort": [{"updated_at": {"order": "desc"}}],
|
||||
|
|
@ -60,72 +60,72 @@ class ContextsService:
|
|||
"size": limit
|
||||
}
|
||||
|
||||
result = await opensearch_client.search(index=CONTEXTS_INDEX_NAME, body=search_body)
|
||||
result = await opensearch_client.search(index=KNOWLEDGE_FILTERS_INDEX_NAME, body=search_body)
|
||||
|
||||
# Transform results
|
||||
contexts = []
|
||||
filters = []
|
||||
for hit in result["hits"]["hits"]:
|
||||
context = hit["_source"]
|
||||
context["score"] = hit.get("_score")
|
||||
contexts.append(context)
|
||||
knowledge_filter = hit["_source"]
|
||||
knowledge_filter["score"] = hit.get("_score")
|
||||
filters.append(knowledge_filter)
|
||||
|
||||
return {"success": True, "contexts": contexts}
|
||||
return {"success": True, "filters": filters}
|
||||
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e), "contexts": []}
|
||||
return {"success": False, "error": str(e), "filters": []}
|
||||
|
||||
async def get_context(self, context_id: str, user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
|
||||
"""Get a specific context by ID"""
|
||||
async def get_knowledge_filter(self, filter_id: str, user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
|
||||
"""Get a specific knowledge filter by ID"""
|
||||
try:
|
||||
# Get user's OpenSearch client with JWT for OIDC auth
|
||||
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
|
||||
|
||||
result = await opensearch_client.get(index=CONTEXTS_INDEX_NAME, id=context_id)
|
||||
result = await opensearch_client.get(index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id)
|
||||
|
||||
if result.get("found"):
|
||||
context = result["_source"]
|
||||
return {"success": True, "context": context}
|
||||
knowledge_filter = result["_source"]
|
||||
return {"success": True, "filter": knowledge_filter}
|
||||
else:
|
||||
return {"success": False, "error": "Context not found"}
|
||||
return {"success": False, "error": "Knowledge filter not found"}
|
||||
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
async def update_context(self, context_id: str, updates: Dict[str, Any], user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
|
||||
"""Update an existing context"""
|
||||
async def update_knowledge_filter(self, filter_id: str, updates: Dict[str, Any], user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
|
||||
"""Update an existing knowledge filter"""
|
||||
try:
|
||||
# Get user's OpenSearch client with JWT for OIDC auth
|
||||
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
|
||||
|
||||
# Update the document
|
||||
result = await opensearch_client.update(
|
||||
index=CONTEXTS_INDEX_NAME,
|
||||
id=context_id,
|
||||
index=KNOWLEDGE_FILTERS_INDEX_NAME,
|
||||
id=filter_id,
|
||||
body={"doc": updates}
|
||||
)
|
||||
|
||||
if result.get("result") in ["updated", "noop"]:
|
||||
# Get the updated document
|
||||
updated_doc = await opensearch_client.get(index=CONTEXTS_INDEX_NAME, id=context_id)
|
||||
return {"success": True, "context": updated_doc["_source"]}
|
||||
updated_doc = await opensearch_client.get(index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id)
|
||||
return {"success": True, "filter": updated_doc["_source"]}
|
||||
else:
|
||||
return {"success": False, "error": "Failed to update context"}
|
||||
return {"success": False, "error": "Failed to update knowledge filter"}
|
||||
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
async def delete_context(self, context_id: str, user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
|
||||
"""Delete a context"""
|
||||
async def delete_knowledge_filter(self, filter_id: str, user_id: str = None, jwt_token: str = None) -> Dict[str, Any]:
|
||||
"""Delete a knowledge filter"""
|
||||
try:
|
||||
# Get user's OpenSearch client with JWT for OIDC auth
|
||||
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
|
||||
|
||||
result = await opensearch_client.delete(index=CONTEXTS_INDEX_NAME, id=context_id)
|
||||
result = await opensearch_client.delete(index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id)
|
||||
|
||||
if result.get("result") == "deleted":
|
||||
return {"success": True, "message": "Context deleted successfully"}
|
||||
return {"success": True, "message": "Knowledge filter deleted successfully"}
|
||||
else:
|
||||
return {"success": False, "error": "Failed to delete context"}
|
||||
return {"success": False, "error": "Failed to delete knowledge filter"}
|
||||
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
|
@ -109,7 +109,7 @@ class SessionManager:
|
|||
# OIDC standard claims
|
||||
"iss": issuer, # Issuer from request
|
||||
"sub": user_id, # Subject (user ID)
|
||||
"aud": ["opensearch", "gendb"], # Audience
|
||||
"aud": ["opensearch", "openrag"], # Audience
|
||||
"exp": now + timedelta(days=7), # Expiration
|
||||
"iat": now, # Issued at
|
||||
"auth_time": int(now.timestamp()), # Authentication time
|
||||
|
|
@ -120,7 +120,7 @@ class SessionManager:
|
|||
"name": user.name,
|
||||
"preferred_username": user.email,
|
||||
"email_verified": True,
|
||||
"roles": ["gendb_user"] # Backend role for OpenSearch
|
||||
"roles": ["openrag_user"] # Backend role for OpenSearch
|
||||
}
|
||||
|
||||
token = jwt.encode(token_payload, self.private_key, algorithm="RS256")
|
||||
|
|
@ -133,7 +133,7 @@ class SessionManager:
|
|||
token,
|
||||
self.public_key,
|
||||
algorithms=["RS256"],
|
||||
audience=["opensearch", "gendb"]
|
||||
audience=["opensearch", "openrag"]
|
||||
)
|
||||
return payload
|
||||
except jwt.ExpiredSignatureError:
|
||||
|
|
|
|||
78
uv.lock
generated
78
uv.lock
generated
|
|
@ -483,45 +483,6 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/bb/61/78c7b3851add1481b048b5fdc29067397a1784e2910592bc81bb3f608635/fsspec-2025.5.1-py3-none-any.whl", hash = "sha256:24d3a2e663d5fc735ab256263c4075f374a174c3410c0b25e5bd1970bceaa462", size = 199052, upload-time = "2025-05-24T12:03:21.66Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gendb"
|
||||
version = "0.1.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "agentd" },
|
||||
{ name = "aiofiles" },
|
||||
{ name = "cryptography" },
|
||||
{ name = "docling" },
|
||||
{ name = "google-api-python-client" },
|
||||
{ name = "google-auth-httplib2" },
|
||||
{ name = "google-auth-oauthlib" },
|
||||
{ name = "httpx" },
|
||||
{ name = "opensearch-py", extra = ["async"] },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "python-multipart" },
|
||||
{ name = "starlette" },
|
||||
{ name = "torch" },
|
||||
{ name = "uvicorn" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "agentd", specifier = ">=0.2.2" },
|
||||
{ name = "aiofiles", specifier = ">=24.1.0" },
|
||||
{ name = "cryptography", specifier = ">=45.0.6" },
|
||||
{ name = "docling", specifier = ">=2.41.0" },
|
||||
{ name = "google-api-python-client", specifier = ">=2.143.0" },
|
||||
{ name = "google-auth-httplib2", specifier = ">=0.2.0" },
|
||||
{ name = "google-auth-oauthlib", specifier = ">=1.2.0" },
|
||||
{ name = "httpx", specifier = ">=0.27.0" },
|
||||
{ name = "opensearch-py", extras = ["async"], specifier = ">=3.0.0" },
|
||||
{ name = "pyjwt", specifier = ">=2.8.0" },
|
||||
{ name = "python-multipart", specifier = ">=0.0.20" },
|
||||
{ name = "starlette", specifier = ">=0.47.1" },
|
||||
{ name = "torch", specifier = ">=2.7.1", index = "https://download.pytorch.org/whl/cu128" },
|
||||
{ name = "uvicorn", specifier = ">=0.35.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-api-core"
|
||||
version = "2.25.1"
|
||||
|
|
@ -1354,6 +1315,45 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openrag"
|
||||
version = "0.1.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "agentd" },
|
||||
{ name = "aiofiles" },
|
||||
{ name = "cryptography" },
|
||||
{ name = "docling" },
|
||||
{ name = "google-api-python-client" },
|
||||
{ name = "google-auth-httplib2" },
|
||||
{ name = "google-auth-oauthlib" },
|
||||
{ name = "httpx" },
|
||||
{ name = "opensearch-py", extra = ["async"] },
|
||||
{ name = "pyjwt" },
|
||||
{ name = "python-multipart" },
|
||||
{ name = "starlette" },
|
||||
{ name = "torch" },
|
||||
{ name = "uvicorn" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "agentd", specifier = ">=0.2.2" },
|
||||
{ name = "aiofiles", specifier = ">=24.1.0" },
|
||||
{ name = "cryptography", specifier = ">=45.0.6" },
|
||||
{ name = "docling", specifier = ">=2.41.0" },
|
||||
{ name = "google-api-python-client", specifier = ">=2.143.0" },
|
||||
{ name = "google-auth-httplib2", specifier = ">=0.2.0" },
|
||||
{ name = "google-auth-oauthlib", specifier = ">=1.2.0" },
|
||||
{ name = "httpx", specifier = ">=0.27.0" },
|
||||
{ name = "opensearch-py", extras = ["async"], specifier = ">=3.0.0" },
|
||||
{ name = "pyjwt", specifier = ">=2.8.0" },
|
||||
{ name = "python-multipart", specifier = ">=0.0.20" },
|
||||
{ name = "starlette", specifier = ">=0.47.1" },
|
||||
{ name = "torch", specifier = ">=2.7.1", index = "https://download.pytorch.org/whl/cu128" },
|
||||
{ name = "uvicorn", specifier = ">=0.35.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opensearch-py"
|
||||
version = "3.0.0"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue