gmakstutis/openrag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'ingestion-flow' into langflow-ingestion-modes

Browse source
This commit is contained in:
Edwin Jose 2025-09-08 14:44:33 -04:00
commit 1e97ec693a
3 changed files with 108 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
flows/ingestion_flow.json
View file

@ -884,7 +884,7 @@
], ],
"frozen": false, "frozen": false,
"icon": "file-text", "icon": "file-text",
"last_updated": "2025-09-08T11:35:39.784Z", "last_updated": "2025-09-08T17:45:33.714Z",
"legacy": false, "legacy": false,
"lf_version": "1.5.0.post2", "lf_version": "1.5.0.post2",
"metadata": {}, "metadata": {},
@ -906,23 +906,6 @@
"Message" "Message"
], ],
"value": "__UNDEFINED__" "value": "__UNDEFINED__"
},
{
"allows_loop": false,
"cache": true,
"display_name": "File Path",
"group_outputs": false,
"hidden": null,
"method": "load_files_path",
"name": "path",
"options": null,
"required_inputs": null,
"selected": "Message",
"tool_mode": true,
"types": [
"Message"
],
"value": "__UNDEFINED__"
} }
], ],
"pinned": false, "pinned": false,
@ -1069,9 +1052,7 @@
"bz2", "bz2",
"gz" "gz"
], ],
"file_path": [ "file_path": [],
"242b5797-4104-4a01-8da1-b8036813100d/test_ingestion.txt"
],
"info": "Supported file extensions: txt, md, mdx, csv, json, yaml, yml, xml, html, htm, pdf, docx, py, sh, sql, js, ts, tsx; optionally bundled in file extensions: zip, tar, tgz, bz2, gz", "info": "Supported file extensions: txt, md, mdx, csv, json, yaml, yml, xml, html, htm, pdf, docx, py, sh, sql, js, ts, tsx; optionally bundled in file extensions: zip, tar, tgz, bz2, gz",
"list": true, "list": true,
"list_add_label": "Add More", "list_add_label": "Add More",
@ -1151,7 +1132,7 @@
"dragging": false, "dragging": false,
"id": "File-PSU37", "id": "File-PSU37",
"measured": { "measured": {
"height": 234, "height": 230,
"width": 320 "width": 320
}, },
"position": { "position": {
@ -1735,7 +1716,7 @@
"x": 2218.9287723423276, "x": 2218.9287723423276,
"y": 1332.2598463956504 "y": 1332.2598463956504
}, },
"selected": true, "selected": false,
"type": "genericNode" "type": "genericNode"
} }
], ],

112
src/services/chat_service.py
View file

@ -1,4 +1,5 @@
import json import json
from utils.logging_config import get_logger from utils.logging_config import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@ -178,9 +179,9 @@ class ChatService:
"Langflow client not initialized. Ensure LANGFLOW is reachable or set LANGFLOW_KEY." "Langflow client not initialized. Ensure LANGFLOW is reachable or set LANGFLOW_KEY."
) )
response_text, response_id = await async_langflow( response_text, response_id = await async_langflow(
langflow_client, langflow_client=langflow_client,
LANGFLOW_CHAT_FLOW_ID, flow_id=LANGFLOW_CHAT_FLOW_ID,
document_prompt, prompt=document_prompt,
extra_headers=extra_headers, extra_headers=extra_headers,
previous_response_id=previous_response_id, previous_response_id=previous_response_id,
) )
@ -199,7 +200,7 @@ class ChatService:
async def get_chat_history(self, user_id: str): async def get_chat_history(self, user_id: str):
"""Get chat conversation history for a user""" """Get chat conversation history for a user"""
from agent import get_user_conversations, active_conversations from agent import active_conversations, get_user_conversations
if not user_id: if not user_id:
return {"error": "User ID is required", "conversations": []} return {"error": "User ID is required", "conversations": []}
@ -275,7 +276,7 @@ class ChatService:
"previous_response_id" "previous_response_id"
), ),
"total_messages": len(messages), "total_messages": len(messages),
"source": "in_memory" "source": "in_memory",
} }
) )
@ -283,17 +284,19 @@ class ChatService:
for response_id, metadata in conversations_dict.items(): for response_id, metadata in conversations_dict.items():
if response_id not in in_memory_conversations: if response_id not in in_memory_conversations:
# This is metadata-only conversation (no function calls) # This is metadata-only conversation (no function calls)
conversations.append({ conversations.append(
"response_id": response_id, {
"title": metadata.get("title", "New Chat"), "response_id": response_id,
"endpoint": "chat", "title": metadata.get("title", "New Chat"),
"messages": [], # No messages in metadata-only "endpoint": "chat",
"created_at": metadata.get("created_at"), "messages": [], # No messages in metadata-only
"last_activity": metadata.get("last_activity"), "created_at": metadata.get("created_at"),
"previous_response_id": metadata.get("previous_response_id"), "last_activity": metadata.get("last_activity"),
"total_messages": metadata.get("total_messages", 0), "previous_response_id": metadata.get("previous_response_id"),
"source": "metadata_only" "total_messages": metadata.get("total_messages", 0),
}) "source": "metadata_only",
}
)
# Sort by last activity (most recent first) # Sort by last activity (most recent first)
conversations.sort(key=lambda c: c.get("last_activity", ""), reverse=True) conversations.sort(key=lambda c: c.get("last_activity", ""), reverse=True)
@ -326,7 +329,11 @@ class ChatService:
# 2. Get actual conversations from Langflow database (source of truth for messages) # 2. Get actual conversations from Langflow database (source of truth for messages)
print(f"[DEBUG] Attempting to fetch Langflow history for user: {user_id}") print(f"[DEBUG] Attempting to fetch Langflow history for user: {user_id}")
langflow_history = await langflow_history_service.get_user_conversation_history(user_id, flow_id=FLOW_ID) langflow_history = (
await langflow_history_service.get_user_conversation_history(
user_id, flow_id=LANGFLOW_CHAT_FLOW_ID
)
)
if langflow_history.get("conversations"): if langflow_history.get("conversations"):
for conversation in langflow_history["conversations"]: for conversation in langflow_history["conversations"]:
@ -344,7 +351,7 @@ class ChatService:
"content": msg["content"], "content": msg["content"],
"timestamp": msg.get("timestamp"), "timestamp": msg.get("timestamp"),
"langflow_message_id": msg.get("langflow_message_id"), "langflow_message_id": msg.get("langflow_message_id"),
"source": "langflow" "source": "langflow",
} }
# Include function call data if present # Include function call data if present
@ -360,10 +367,13 @@ class ChatService:
metadata = local_metadata.get(session_id, {}) metadata = local_metadata.get(session_id, {})
if not metadata.get("title"): if not metadata.get("title"):
first_user_msg = next((msg for msg in messages if msg["role"] == "user"), None) first_user_msg = next(
(msg for msg in messages if msg["role"] == "user"), None
)
title = ( title = (
first_user_msg["content"][:50] + "..." first_user_msg["content"][:50] + "..."
if first_user_msg and len(first_user_msg["content"]) > 50 if first_user_msg
and len(first_user_msg["content"]) > 50
else first_user_msg["content"] else first_user_msg["content"]
if first_user_msg if first_user_msg
else "Langflow chat" else "Langflow chat"
@ -371,37 +381,47 @@ class ChatService:
else: else:
title = metadata["title"] title = metadata["title"]
all_conversations.append({ all_conversations.append(
"response_id": session_id, {
"title": title, "response_id": session_id,
"endpoint": "langflow", "title": title,
"messages": messages, # Function calls preserved from Langflow "endpoint": "langflow",
"created_at": metadata.get("created_at") or conversation.get("created_at"), "messages": messages, # Function calls preserved from Langflow
"last_activity": metadata.get("last_activity") or conversation.get("last_activity"), "created_at": metadata.get("created_at")
"total_messages": len(messages), or conversation.get("created_at"),
"source": "langflow_enhanced", "last_activity": metadata.get("last_activity")
"langflow_session_id": session_id, or conversation.get("last_activity"),
"langflow_flow_id": conversation.get("flow_id") "total_messages": len(messages),
}) "source": "langflow_enhanced",
"langflow_session_id": session_id,
"langflow_flow_id": conversation.get("flow_id"),
}
)
# 3. Add any local metadata that doesn't have Langflow data yet (recent conversations) # 3. Add any local metadata that doesn't have Langflow data yet (recent conversations)
for response_id, metadata in local_metadata.items(): for response_id, metadata in local_metadata.items():
if not any(c["response_id"] == response_id for c in all_conversations): if not any(c["response_id"] == response_id for c in all_conversations):
all_conversations.append({ all_conversations.append(
"response_id": response_id, {
"title": metadata.get("title", "New Chat"), "response_id": response_id,
"endpoint": "langflow", "title": metadata.get("title", "New Chat"),
"messages": [], # Will be filled when Langflow sync catches up "endpoint": "langflow",
"created_at": metadata.get("created_at"), "messages": [], # Will be filled when Langflow sync catches up
"last_activity": metadata.get("last_activity"), "created_at": metadata.get("created_at"),
"total_messages": metadata.get("total_messages", 0), "last_activity": metadata.get("last_activity"),
"source": "metadata_only" "total_messages": metadata.get("total_messages", 0),
}) "source": "metadata_only",
}
)
if langflow_history.get("conversations"): if langflow_history.get("conversations"):
print(f"[DEBUG] Added {len(langflow_history['conversations'])} historical conversations from Langflow") print(
f"[DEBUG] Added {len(langflow_history['conversations'])} historical conversations from Langflow"
)
elif langflow_history.get("error"): elif langflow_history.get("error"):
print(f"[DEBUG] Could not fetch Langflow history for user {user_id}: {langflow_history['error']}") print(
f"[DEBUG] Could not fetch Langflow history for user {user_id}: {langflow_history['error']}"
)
else: else:
print(f"[DEBUG] No Langflow conversations found for user {user_id}") print(f"[DEBUG] No Langflow conversations found for user {user_id}")
@ -412,7 +432,9 @@ class ChatService:
# Sort by last activity (most recent first) # Sort by last activity (most recent first)
all_conversations.sort(key=lambda c: c.get("last_activity", ""), reverse=True) all_conversations.sort(key=lambda c: c.get("last_activity", ""), reverse=True)
print(f"[DEBUG] Returning {len(all_conversations)} conversations ({len(local_metadata)} from local metadata)") print(
f"[DEBUG] Returning {len(all_conversations)} conversations ({len(local_metadata)} from local metadata)"
)
return { return {
"user_id": user_id, "user_id": user_id,

22
src/tui/managers/env_manager.py
View file

@ -1,23 +1,23 @@
"""Environment configuration manager for OpenRAG TUI.""" """Environment configuration manager for OpenRAG TUI."""
import os
import secrets import secrets
import string import string
from dataclasses import dataclass, field
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Dict, Optional, List from typing import Dict, List, Optional
from dataclasses import dataclass, field
from utils.logging_config import get_logger from utils.logging_config import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
from ..utils.validation import ( from ..utils.validation import (
validate_openai_api_key, sanitize_env_value,
validate_documents_paths,
validate_google_oauth_client_id, validate_google_oauth_client_id,
validate_non_empty, validate_non_empty,
validate_openai_api_key,
validate_url, validate_url,
validate_documents_paths,
sanitize_env_value,
) )
@ -31,7 +31,7 @@ class EnvConfig:
langflow_secret_key: str = "" langflow_secret_key: str = ""
langflow_superuser: str = "admin" langflow_superuser: str = "admin"
langflow_superuser_password: str = "" langflow_superuser_password: str = ""
flow_id: str = "1098eea1-6649-4e1d-aed1-b77249fb8dd0" langflow_chat_flow_id: str = "1098eea1-6649-4e1d-aed1-b77249fb8dd0"
langflow_ingest_flow_id: str = "5488df7c-b93f-4f87-a446-b67028bc0813" langflow_ingest_flow_id: str = "5488df7c-b93f-4f87-a446-b67028bc0813"
# OAuth settings # OAuth settings
@ -99,7 +99,7 @@ class EnvManager:
"LANGFLOW_SECRET_KEY": "langflow_secret_key", "LANGFLOW_SECRET_KEY": "langflow_secret_key",
"LANGFLOW_SUPERUSER": "langflow_superuser", "LANGFLOW_SUPERUSER": "langflow_superuser",
"LANGFLOW_SUPERUSER_PASSWORD": "langflow_superuser_password", "LANGFLOW_SUPERUSER_PASSWORD": "langflow_superuser_password",
"FLOW_ID": "flow_id", "LANGFLOW_CHAT_FLOW_ID": "langflow_chat_flow_id",
"LANGFLOW_INGEST_FLOW_ID": "langflow_ingest_flow_id", "LANGFLOW_INGEST_FLOW_ID": "langflow_ingest_flow_id",
"GOOGLE_OAUTH_CLIENT_ID": "google_oauth_client_id", "GOOGLE_OAUTH_CLIENT_ID": "google_oauth_client_id",
"GOOGLE_OAUTH_CLIENT_SECRET": "google_oauth_client_secret", "GOOGLE_OAUTH_CLIENT_SECRET": "google_oauth_client_secret",
@ -236,8 +236,10 @@ class EnvManager:
f.write( f.write(
f"LANGFLOW_SUPERUSER_PASSWORD={self.config.langflow_superuser_password}\n" f"LANGFLOW_SUPERUSER_PASSWORD={self.config.langflow_superuser_password}\n"
) )
f.write(f"FLOW_ID={self.config.flow_id}\n") f.write(f"LANGFLOW_CHAT_FLOW_ID={self.config.langflow_chat_flow_id}\n")
f.write(f"LANGFLOW_INGEST_FLOW_ID={self.config.langflow_ingest_flow_id}\n") f.write(
f"LANGFLOW_INGEST_FLOW_ID={self.config.langflow_ingest_flow_id}\n"
)
f.write(f"OPENSEARCH_PASSWORD={self.config.opensearch_password}\n") f.write(f"OPENSEARCH_PASSWORD={self.config.opensearch_password}\n")
f.write(f"OPENAI_API_KEY={self.config.openai_api_key}\n") f.write(f"OPENAI_API_KEY={self.config.openai_api_key}\n")
f.write( f.write(