From 795cbe3b60f69f299bab23f531ac2bc18e2731f8 Mon Sep 17 00:00:00 2001 From: phact Date: Mon, 15 Dec 2025 16:12:44 -0500 Subject: [PATCH] expand compose paths, and fix .env on startup --- docker-compose.yml | 14 ++++----- src/tui/main.py | 43 +++++++++++++++++++++++++-- src/tui/managers/container_manager.py | 39 +++++------------------- src/tui/managers/env_manager.py | 20 ++++++++----- src/utils/paths.py | 20 ++++++++++++- 5 files changed, 86 insertions(+), 50 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 48a97a2a..2a73da89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: - "9200:9200" - "9600:9600" volumes: - - ${OPENSEARCH_DATA_PATH:-$HOME/.openrag/data/opensearch-data}:/usr/share/opensearch/data:Z + - ${OPENSEARCH_DATA_PATH:-./opensearch-data}:/usr/share/opensearch/data:Z dashboards: image: opensearchproject/opensearch-dashboards:3.0.0 @@ -80,11 +80,11 @@ services: - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} volumes: - - ${OPENRAG_DOCUMENTS_PATH:-$HOME/.openrag/documents}:/app/openrag-documents:Z - - ${OPENRAG_KEYS_PATH:-$HOME/.openrag/keys}:/app/keys:Z - - ${OPENRAG_FLOWS_PATH:-$HOME/.openrag/flows}:/app/flows:U,z - - ${OPENRAG_CONFIG_PATH:-$HOME/.openrag/config}:/app/config:Z - - ${OPENRAG_DATA_PATH:-$HOME/.openrag/data}:/app/data:Z + - ${OPENRAG_DOCUMENTS_PATH:-./openrag-documents}:/app/openrag-documents:Z + - ${OPENRAG_KEYS_PATH:-./keys}:/app/keys:Z + - ${OPENRAG_FLOWS_PATH:-./flows}:/app/flows:U,z + - ${OPENRAG_CONFIG_PATH:-./config}:/app/config:Z + - ${OPENRAG_DATA_PATH:-./data}:/app/data:Z openrag-frontend: image: langflowai/openrag-frontend:${OPENRAG_VERSION:-latest} @@ -101,7 +101,7 @@ services: langflow: volumes: - - ${OPENRAG_FLOWS_PATH:-$HOME/.openrag/flows}:/app/flows:U,z + - ${OPENRAG_FLOWS_PATH:-./flows}:/app/flows:U,z image: langflowai/openrag-langflow:${OPENRAG_VERSION:-latest} build: context: . diff --git a/src/tui/main.py b/src/tui/main.py index 85da0a2d..cd498c6e 100644 --- a/src/tui/main.py +++ b/src/tui/main.py @@ -576,9 +576,27 @@ def migrate_legacy_data_directories(): sources_to_migrate = [(s, t, d) for s, t, d in migrations if s.exists()] if not sources_to_migrate: - # No legacy data to migrate, just mark as done + # No legacy data to migrate, just mark as done and update .env paths marker.parent.mkdir(parents=True, exist_ok=True) marker.touch() + # Still need to update .env with centralized paths + try: + from managers.env_manager import EnvManager + env_manager = EnvManager() + env_manager.load_existing_env() + # Explicitly set centralized paths (overrides any old CWD-relative paths) + home = str(Path.home()) + env_manager.config.openrag_documents_paths = f"{home}/.openrag/documents" + env_manager.config.openrag_documents_path = f"{home}/.openrag/documents" + env_manager.config.openrag_keys_path = f"{home}/.openrag/keys" + env_manager.config.openrag_flows_path = f"{home}/.openrag/flows" + env_manager.config.openrag_config_path = f"{home}/.openrag/config" + env_manager.config.openrag_data_path = f"{home}/.openrag/data" + env_manager.config.opensearch_data_path = f"{home}/.openrag/data/opensearch-data" + env_manager.save_env() + logger.info("Updated .env file with centralized paths") + except Exception as e: + logger.warning(f"Failed to update .env paths: {e}") return # Prompt user for confirmation @@ -640,6 +658,27 @@ def migrate_legacy_data_directories(): marker.parent.mkdir(parents=True, exist_ok=True) marker.touch() + # Update .env file with centralized paths + try: + from managers.env_manager import EnvManager + env_manager = EnvManager() + env_manager.load_existing_env() + # Explicitly set centralized paths (overrides any old CWD-relative paths) + home = str(Path.home()) + env_manager.config.openrag_documents_paths = f"{home}/.openrag/documents" + env_manager.config.openrag_documents_path = f"{home}/.openrag/documents" + env_manager.config.openrag_keys_path = f"{home}/.openrag/keys" + env_manager.config.openrag_flows_path = f"{home}/.openrag/flows" + env_manager.config.openrag_config_path = f"{home}/.openrag/config" + env_manager.config.openrag_data_path = f"{home}/.openrag/data" + env_manager.config.opensearch_data_path = f"{home}/.openrag/data/opensearch-data" + env_manager.save_env() + print(" Updated .env with centralized paths") + logger.info("Updated .env file with centralized paths") + except Exception as e: + logger.warning(f"Failed to update .env paths: {e}") + print(f" Warning: Failed to update .env paths: {e}") + print("\nMigration complete!\n") logger.info("Data migration completed successfully") @@ -688,7 +727,7 @@ def run_tui(): try: # Migrate legacy data directories from CWD to ~/.openrag/ migrate_legacy_data_directories() - + # Initialize host directory structure setup_host_directories() diff --git a/src/tui/managers/container_manager.py b/src/tui/managers/container_manager.py index 1f46ff09..d1ac00b4 100644 --- a/src/tui/managers/container_manager.py +++ b/src/tui/managers/container_manager.py @@ -1173,43 +1173,18 @@ class ContainerManager: env_manager = EnvManager() env_manager.load_existing_env() opensearch_data_path = Path(env_manager.config.opensearch_data_path.replace("$HOME", str(Path.home()))).expanduser().absolute() - + if not opensearch_data_path.exists(): yield True, "OpenSearch data directory does not exist, skipping" return - - # Use the opensearch container with proper volume mount flags - # :Z flag ensures proper SELinux labeling and UID mapping for rootless containers - cmd = [ - "run", - "--rm", - "-v", f"{opensearch_data_path}:/usr/share/opensearch/data:Z", - "langflowai/openrag-opensearch:latest", - "bash", "-c", - "rm -rf /usr/share/opensearch/data/* /usr/share/opensearch/data/.[!.]* && echo 'Cleared successfully'" - ] - - success, stdout, stderr = await self._run_runtime_command(cmd) - - if success and "Cleared successfully" in stdout: + + # Use alpine with root to clear container-owned files + success, msg = await self.clear_directory_with_container(opensearch_data_path) + + if success: yield True, "OpenSearch data cleared successfully" else: - # If it fails, try with the base opensearch image - yield False, "Retrying with base OpenSearch image..." - cmd = [ - "run", - "--rm", - "-v", f"{opensearch_data_path}:/usr/share/opensearch/data:Z", - "opensearchproject/opensearch:3.0.0", - "bash", "-c", - "rm -rf /usr/share/opensearch/data/* /usr/share/opensearch/data/.[!.]* && echo 'Cleared successfully'" - ] - success, stdout, stderr = await self._run_runtime_command(cmd) - - if success and "Cleared successfully" in stdout: - yield True, "OpenSearch data cleared successfully" - else: - yield False, f"Failed to clear OpenSearch data: {stderr if stderr else 'Unknown error'}" + yield False, f"Failed to clear OpenSearch data: {msg}" async def reset_services(self) -> AsyncIterator[tuple[bool, str]]: """Reset all services (stop, remove containers/volumes, clear data) and yield progress updates.""" diff --git a/src/tui/managers/env_manager.py b/src/tui/managers/env_manager.py index a853ee72..aa5dc2eb 100644 --- a/src/tui/managers/env_manager.py +++ b/src/tui/managers/env_manager.py @@ -377,30 +377,34 @@ class EnvManager: f.write(f"LANGFLOW_URL_INGEST_FLOW_ID={self._quote_env_value(self.config.langflow_url_ingest_flow_id)}\n") f.write(f"NUDGES_FLOW_ID={self._quote_env_value(self.config.nudges_flow_id)}\n") f.write(f"OPENSEARCH_PASSWORD={self._quote_env_value(self.config.opensearch_password)}\n") + + # Expand $HOME in paths before writing to .env + # This ensures paths work with all compose implementations (docker, podman) + from utils.paths import expand_path f.write( - f"OPENRAG_DOCUMENTS_PATHS={self._quote_env_value(self.config.openrag_documents_paths)}\n" + f"OPENRAG_DOCUMENTS_PATHS={self._quote_env_value(expand_path(self.config.openrag_documents_paths))}\n" ) f.write("\n") - + # Volume mount paths for Docker Compose f.write("# Volume mount paths for Docker Compose\n") f.write( - f"OPENRAG_DOCUMENTS_PATH={self._quote_env_value(self.config.openrag_documents_path)}\n" + f"OPENRAG_DOCUMENTS_PATH={self._quote_env_value(expand_path(self.config.openrag_documents_path))}\n" ) f.write( - f"OPENRAG_KEYS_PATH={self._quote_env_value(self.config.openrag_keys_path)}\n" + f"OPENRAG_KEYS_PATH={self._quote_env_value(expand_path(self.config.openrag_keys_path))}\n" ) f.write( - f"OPENRAG_FLOWS_PATH={self._quote_env_value(self.config.openrag_flows_path)}\n" + f"OPENRAG_FLOWS_PATH={self._quote_env_value(expand_path(self.config.openrag_flows_path))}\n" ) f.write( - f"OPENRAG_CONFIG_PATH={self._quote_env_value(self.config.openrag_config_path)}\n" + f"OPENRAG_CONFIG_PATH={self._quote_env_value(expand_path(self.config.openrag_config_path))}\n" ) f.write( - f"OPENRAG_DATA_PATH={self._quote_env_value(self.config.openrag_data_path)}\n" + f"OPENRAG_DATA_PATH={self._quote_env_value(expand_path(self.config.openrag_data_path))}\n" ) f.write( - f"OPENSEARCH_DATA_PATH={self._quote_env_value(self.config.opensearch_data_path)}\n" + f"OPENSEARCH_DATA_PATH={self._quote_env_value(expand_path(self.config.opensearch_data_path))}\n" ) # Set OPENRAG_VERSION to TUI version if self.config.openrag_version: diff --git a/src/utils/paths.py b/src/utils/paths.py index c621c8f8..3e97a4bd 100644 --- a/src/utils/paths.py +++ b/src/utils/paths.py @@ -55,7 +55,7 @@ def get_tui_compose_file(gpu: bool = False) -> Path: def get_legacy_paths() -> dict: """Get legacy (CWD-based) paths for migration purposes. - + Returns: Dictionary mapping resource names to their old CWD-based paths """ @@ -65,3 +65,21 @@ def get_legacy_paths() -> dict: "tui_compose": cwd / "docker-compose.yml", "tui_compose_gpu": cwd / "docker-compose.gpu.yml", } + + +def expand_path(path: str) -> str: + """Expand $HOME and ~ in a path string to the actual home directory. + + Args: + path: Path string that may contain $HOME or ~ + + Returns: + Path string with $HOME and ~ expanded to actual home directory + """ + if not path: + return path + expanded = path.replace("$HOME", str(Path.home())) + # Also handle ~ at start of path + if expanded.startswith("~"): + expanded = str(Path.home()) + expanded[1:] + return expanded