diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index ca39b6e3..00000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.env.example b/.env.example index 8d412670..f790ce09 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,9 @@ AWS_SECRET_ACCESS_KEY= # OPTIONAL url for openrag link to langflow in the UI LANGFLOW_PUBLIC_URL= +# OPTIONAL: Override host for docling service (for special networking setups) +# HOST_DOCKER_INTERNAL=host.containers.internal + # Langflow auth LANGFLOW_AUTO_LOGIN=False LANGFLOW_SUPERUSER= diff --git a/.github/workflows/build-langflow-responses.yml b/.github/workflows/build-langflow-responses.yml deleted file mode 100644 index 0f9d3d08..00000000 --- a/.github/workflows/build-langflow-responses.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Build Langflow Responses Multi-Arch - -on: - workflow_dispatch: - -jobs: - build: - strategy: - fail-fast: false - matrix: - include: - - platform: linux/amd64 - arch: amd64 - runs-on: ubuntu-latest - - platform: linux/arm64 - arch: arm64 - runs-on: [self-hosted, linux, ARM64, langflow-ai-arm64-2] - - runs-on: ${{ matrix.runs-on }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push langflow (${{ matrix.arch }}) - uses: docker/build-push-action@v5 - with: - context: . - file: ./Dockerfile.langflow - platforms: ${{ matrix.platform }} - push: true - tags: phact/langflow:responses-${{ matrix.arch }} - cache-from: type=gha,scope=langflow-responses-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=langflow-responses-${{ matrix.arch }} - - manifest: - needs: build - runs-on: ubuntu-latest - steps: - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Create and push multi-arch manifest - run: | - docker buildx imagetools create -t phact/langflow:responses \ - phact/langflow:responses-amd64 \ - phact/langflow:responses-arm64 \ No newline at end of file diff --git a/.github/workflows/build-multiarch.yml b/.github/workflows/build-multiarch.yml index 620bcf3b..64c13b91 100644 --- a/.github/workflows/build-multiarch.yml +++ b/.github/workflows/build-multiarch.yml @@ -1,16 +1,95 @@ -name: Build Multi-Architecture Docker Images +name: Release + Docker Images (multi-arch) on: + push: + branches: + - main + paths: + - 'pyproject.toml' workflow_dispatch: - inputs: - update_latest: - description: 'Update latest tags (production release)' - required: false - default: false - type: boolean jobs: + build-python-packages: + runs-on: ubuntu-latest + outputs: + skip_release: ${{ steps.version.outputs.skip_release }} + version: ${{ steps.version.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Extract version from pyproject.toml + id: version + run: | + VERSION=$(grep '^version = ' pyproject.toml | cut -d '"' -f 2) + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Version: $VERSION" + + # Check if tag already exists + if git rev-parse "v$VERSION" >/dev/null 2>&1; then + echo "Tag v$VERSION already exists, skipping release" + echo "skip_release=true" >> $GITHUB_OUTPUT + exit 0 + fi + echo "skip_release=false" >> $GITHUB_OUTPUT + + # Check if version is numeric (e.g., 0.1.16) vs prerelease (e.g., 0.1.16-rc1) + if [[ "$VERSION" =~ ^[0-9.-]+$ ]]; then + echo "is_prerelease=false" >> $GITHUB_OUTPUT + echo "Release type: Production" + else + echo "is_prerelease=true" >> $GITHUB_OUTPUT + echo "Release type: Prerelease" + fi + + - name: Build wheel and source distribution + if: steps.version.outputs.skip_release != 'true' + run: | + uv build + + - name: List built artifacts + if: steps.version.outputs.skip_release != 'true' + run: | + ls -la dist/ + echo "Built artifacts:" + for file in dist/*; do + echo " - $(basename $file) ($(stat -c%s $file | numfmt --to=iec-i)B)" + done + + - name: Upload build artifacts + if: steps.version.outputs.skip_release != 'true' + uses: actions/upload-artifact@v4 + with: + name: python-packages + path: dist/ + retention-days: 30 + + - name: Create Release + if: steps.version.outputs.skip_release != 'true' + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.version.outputs.version }} + name: Release ${{ steps.version.outputs.version }} + draft: false + prerelease: ${{ steps.version.outputs.is_prerelease }} + generate_release_notes: true + files: | + dist/*.whl + dist/*.tar.gz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + build: + needs: build-python-packages + if: needs.build-python-packages.outputs.skip_release != 'true' strategy: fail-fast: false matrix: @@ -106,9 +185,9 @@ jobs: cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.arch }} manifest: - needs: build + needs: [build, build-python-packages] runs-on: ubuntu-latest - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && needs.build-python-packages.outputs.skip_release != 'true' steps: - name: Checkout uses: actions/checkout@v4 @@ -146,8 +225,8 @@ jobs: phact/openrag-opensearch:$VERSION-amd64 \ phact/openrag-opensearch:$VERSION-arm64 - # Only update latest tags if version is numeric AND checkbox is checked - if [[ "$VERSION" =~ ^[0-9.-]+$ ]] && [[ "${{ github.event.inputs.update_latest }}" == "true" ]]; then + # Only update latest tags if version is numeric + if [[ "$VERSION" =~ ^[0-9.-]+$ ]]; then echo "Updating latest tags for production release: $VERSION" docker buildx imagetools create -t phact/openrag-backend:latest \ phact/openrag-backend:$VERSION-amd64 \ @@ -165,5 +244,5 @@ jobs: phact/openrag-opensearch:$VERSION-amd64 \ phact/openrag-opensearch:$VERSION-arm64 else - echo "Skipping latest tags - version: $VERSION, update_latest: ${{ github.event.inputs.update_latest }}" + echo "Skipping latest tags - version: $VERSION (not numeric)" fi diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml new file mode 100644 index 00000000..16f33c41 --- /dev/null +++ b/.github/workflows/test-integration.yml @@ -0,0 +1,54 @@ +name: Integration Tests + +on: + pull_request: + push: + branches: + - main + +jobs: + tests: + runs-on: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + env: + # Prefer repository/environment variable first, then secret, then a sane fallback + OPENSEARCH_PASSWORD: ${{ vars.OPENSEARCH_PASSWORD || secrets.OPENSEARCH_PASSWORD || 'OpenRag#2025!' }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + steps: + - run: df -h + #- name: "node-cleanup" + #run: | + # sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + # sudo docker image prune --all --force + # sudo docker builder prune -a + - run: df -h + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up UV + uses: astral-sh/setup-uv@v3 + with: + version: latest + + - name: Python version + run: uv python install 3.13 + + - name: Install dependencies + run: uv sync + + - name: Run integration tests + env: + OPENSEARCH_HOST: localhost + OPENSEARCH_PORT: 9200 + OPENSEARCH_USERNAME: admin + OPENSEARCH_PASSWORD: ${{ env.OPENSEARCH_PASSWORD }} + LOG_LEVEL: DEBUG + # Force no-auth mode so tests bypass OAuth + GOOGLE_OAUTH_CLIENT_ID: "" + GOOGLE_OAUTH_CLIENT_SECRET: "" + # Disable startup ingest noise unless a test enables it + DISABLE_STARTUP_INGEST: "true" + run: | + make test-ci + echo "Keys directory after tests:" + ls -la keys/ || echo "No keys directory" diff --git a/.gitignore b/.gitignore index 484db58d..625097a6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ wheels/ 1001*.pdf *.json !flows/*.json +!src/tui/_assets/flows/*.json +!src/tui/_assets/flows/components/*.json .DS_Store config/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 19b01709..6b8cd832 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,20 +11,48 @@ Thank you for your interest in contributing to OpenRAG! This guide will help you - Python 3.13+ with uv package manager - Node.js 18+ and npm -### Environment Setup +### Set up OpenRAG for development + +1. Set up your development environment. ```bash -# Clone the repository -git clone +# Clone and setup environment +git clone https://github.com/langflow-ai/openrag.git cd openrag - -# Setup development environment make setup # Creates .env and installs dependencies ``` -### Configuration +2. Configure the `.env` file with your API keys and credentials. -Edit `.env` with your API keys and credentials. See the main README for required environment variables. +```bash +# Required +OPENAI_API_KEY=your_openai_api_key +OPENSEARCH_PASSWORD=your_secure_password +LANGFLOW_SUPERUSER=admin +LANGFLOW_SUPERUSER_PASSWORD=your_secure_password +LANGFLOW_CHAT_FLOW_ID=your_chat_flow_id +LANGFLOW_INGEST_FLOW_ID=your_ingest_flow_id +NUDGES_FLOW_ID=your_nudges_flow_id +``` + +For extended configuration, including ingestion and optional variables, see [docs/reference/configuration.mdx](docs/docs/reference/configuration.mdx). + +3. Start OpenRAG. + +```bash +# Full stack with GPU support +make dev + +# Or CPU only +make dev-cpu +``` + +Access the services: +- **Frontend**: http://localhost:3000 +- **Backend API**: http://localhost:8000 +- **Langflow**: http://localhost:7860 +- **OpenSearch**: http://localhost:9200 +- **OpenSearch Dashboards**: http://localhost:5601 ## 🔧 Development Commands diff --git a/Dockerfile.langflow b/Dockerfile.langflow index 71baf447..bdae1f70 100644 --- a/Dockerfile.langflow +++ b/Dockerfile.langflow @@ -1,4 +1,4 @@ -FROM langflowai/langflow-nightly:1.6.3.dev0 +FROM langflowai/langflow-nightly:1.6.3.dev1 EXPOSE 7860 diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..d0f089fc --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include src/tui/_assets * \ No newline at end of file diff --git a/Makefile b/Makefile index e8b08a1b..b30f77fc 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,17 @@ # OpenRAG Development Makefile # Provides easy commands for development workflow -.PHONY: help dev dev-cpu dev-local infra stop clean build logs shell-backend shell-frontend install test backend frontend install-be install-fe build-be build-fe logs-be logs-fe logs-lf logs-os shell-be shell-lf shell-os restart status health db-reset flow-upload quick setup +# Load variables from .env if present so `make` commands pick them up +ifneq (,$(wildcard .env)) + include .env + # Export all simple KEY=VALUE pairs to the environment for child processes + export $(shell sed -n 's/^\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' .env) +endif + +.PHONY: help dev dev-cpu dev-local infra stop clean build logs shell-backend shell-frontend install \ + test test-integration test-ci \ + backend frontend install-be install-fe build-be build-fe logs-be logs-fe logs-lf logs-os \ + shell-be shell-lf shell-os restart status health db-reset flow-upload quick setup # Default target help: @@ -32,14 +42,16 @@ help: @echo " shell-lf - Shell into langflow container" @echo "" @echo "Testing:" - @echo " test - Run backend tests" + @echo " test - Run all backend tests" + @echo " test-integration - Run integration tests (requires infra)" + @echo " test-ci - Start infra, run integration tests, tear down" @echo " lint - Run linting checks" @echo "" # Development environments dev: @echo "🚀 Starting OpenRAG with GPU support..." - docker-compose up -d + docker compose up -d @echo "✅ Services started!" @echo " Backend: http://localhost:8000" @echo " Frontend: http://localhost:3000" @@ -49,7 +61,7 @@ dev: dev-cpu: @echo "🚀 Starting OpenRAG with CPU only..." - docker-compose -f docker-compose-cpu.yml up -d + docker compose -f docker-compose-cpu.yml up -d @echo "✅ Services started!" @echo " Backend: http://localhost:8000" @echo " Frontend: http://localhost:3000" @@ -59,7 +71,7 @@ dev-cpu: dev-local: @echo "🔧 Starting infrastructure only (for local development)..." - docker-compose up -d opensearch dashboards langflow + docker compose up -d opensearch dashboards langflow @echo "✅ Infrastructure started!" @echo " Langflow: http://localhost:7860" @echo " OpenSearch: http://localhost:9200" @@ -69,7 +81,7 @@ dev-local: infra: @echo "🔧 Starting infrastructure services only..." - docker-compose up -d opensearch dashboards langflow + docker compose up -d opensearch dashboards langflow @echo "✅ Infrastructure services started!" @echo " Langflow: http://localhost:7860" @echo " OpenSearch: http://localhost:9200" @@ -86,15 +98,15 @@ infra-cpu: # Container management stop: @echo "🛑 Stopping all containers..." - docker-compose down - docker-compose -f docker-compose-cpu.yml down 2>/dev/null || true + docker compose down + docker compose -f docker-compose-cpu.yml down 2>/dev/null || true restart: stop dev clean: stop @echo "🧹 Cleaning up containers and volumes..." - docker-compose down -v --remove-orphans - docker-compose -f docker-compose-cpu.yml down -v --remove-orphans 2>/dev/null || true + docker compose down -v --remove-orphans + docker compose -f docker-compose-cpu.yml down -v --remove-orphans 2>/dev/null || true docker system prune -f # Local development @@ -114,7 +126,7 @@ install: install-be install-fe install-be: @echo "📦 Installing backend dependencies..." - uv sync + uv sync --extra torch-cu128 install-fe: @echo "📦 Installing frontend dependencies..." @@ -123,7 +135,7 @@ install-fe: # Building build: @echo "🔨 Building Docker images..." - docker-compose build + docker compose build build-be: @echo "🔨 Building backend image..." @@ -136,41 +148,124 @@ build-fe: # Logging and debugging logs: @echo "📋 Showing all container logs..." - docker-compose logs -f + docker compose logs -f logs-be: @echo "📋 Showing backend logs..." - docker-compose logs -f openrag-backend + docker compose logs -f openrag-backend logs-fe: @echo "📋 Showing frontend logs..." - docker-compose logs -f openrag-frontend + docker compose logs -f openrag-frontend logs-lf: @echo "📋 Showing langflow logs..." - docker-compose logs -f langflow + docker compose logs -f langflow logs-os: @echo "📋 Showing opensearch logs..." - docker-compose logs -f opensearch + docker compose logs -f opensearch # Shell access shell-be: @echo "🐚 Opening shell in backend container..." - docker-compose exec openrag-backend /bin/bash + docker compose exec openrag-backend /bin/bash shell-lf: @echo "🐚 Opening shell in langflow container..." - docker-compose exec langflow /bin/bash + docker compose exec langflow /bin/bash shell-os: @echo "🐚 Opening shell in opensearch container..." - docker-compose exec opensearch /bin/bash + docker compose exec opensearch /bin/bash # Testing and quality test: - @echo "🧪 Running backend tests..." - uv run pytest + @echo "🧪 Running all backend tests..." + uv run pytest tests/ -v + +test-integration: + @echo "🧪 Running integration tests (requires infrastructure)..." + @echo "💡 Make sure to run 'make infra' first!" + uv run pytest tests/integration/ -v + +# CI-friendly integration test target: brings up infra, waits, runs tests, tears down +test-ci: + @set -e; \ + echo "Installing test dependencies..."; \ + uv sync --group dev; \ + if [ ! -f keys/private_key.pem ]; then \ + echo "Generating RSA keys for JWT signing..."; \ + uv run python -c "from src.main import generate_jwt_keys; generate_jwt_keys()"; \ + else \ + echo "RSA keys already exist, ensuring correct permissions..."; \ + chmod 600 keys/private_key.pem 2>/dev/null || true; \ + chmod 644 keys/public_key.pem 2>/dev/null || true; \ + fi; \ + echo "Cleaning up old containers and volumes..."; \ + docker compose -f docker-compose-cpu.yml down -v 2>/dev/null || true; \ + echo "Pulling latest images..."; \ + docker compose -f docker-compose-cpu.yml pull; \ + echo "Starting infra (OpenSearch + Dashboards + Langflow) with CPU containers"; \ + docker compose -f docker-compose-cpu.yml up -d opensearch dashboards langflow; \ + echo "Starting docling-serve..."; \ + DOCLING_ENDPOINT=$$(uv run python scripts/docling_ctl.py start --port 5001 | grep "Endpoint:" | awk '{print $$2}'); \ + echo "Docling-serve started at $$DOCLING_ENDPOINT"; \ + echo "Waiting for backend OIDC endpoint..."; \ + for i in $$(seq 1 60); do \ + docker exec openrag-backend curl -s http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1 && break || sleep 2; \ + done; \ + echo "Waiting for OpenSearch security config to be fully applied..."; \ + for i in $$(seq 1 60); do \ + if docker logs os 2>&1 | grep -q "Security configuration applied successfully"; then \ + echo "✓ Security configuration applied"; \ + break; \ + fi; \ + sleep 2; \ + done; \ + echo "Verifying OIDC authenticator is active in OpenSearch..."; \ + AUTHC_CONFIG=$$(curl -k -s -u admin:$${OPENSEARCH_PASSWORD} https://localhost:9200/_opendistro/_security/api/securityconfig 2>/dev/null); \ + if echo "$$AUTHC_CONFIG" | grep -q "openid_auth_domain"; then \ + echo "✓ OIDC authenticator configured"; \ + echo "$$AUTHC_CONFIG" | grep -A 5 "openid_auth_domain"; \ + else \ + echo "✗ OIDC authenticator NOT found in security config!"; \ + echo "Security config:"; \ + echo "$$AUTHC_CONFIG" | head -50; \ + exit 1; \ + fi; \ + echo "Waiting for Langflow..."; \ + for i in $$(seq 1 60); do \ + curl -s http://localhost:7860/ >/dev/null 2>&1 && break || sleep 2; \ + done; \ + echo "Waiting for docling-serve at $$DOCLING_ENDPOINT..."; \ + for i in $$(seq 1 60); do \ + curl -s $${DOCLING_ENDPOINT}/health >/dev/null 2>&1 && break || sleep 2; \ + done; \ + echo "Running integration tests"; \ + LOG_LEVEL=$${LOG_LEVEL:-DEBUG} \ + GOOGLE_OAUTH_CLIENT_ID="" \ + GOOGLE_OAUTH_CLIENT_SECRET="" \ + OPENSEARCH_HOST=localhost OPENSEARCH_PORT=9200 \ + OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD=$${OPENSEARCH_PASSWORD} \ + DISABLE_STARTUP_INGEST=$${DISABLE_STARTUP_INGEST:-true} \ + uv run pytest tests/integration -vv -s -o log_cli=true --log-cli-level=DEBUG; \ + TEST_RESULT=$$?; \ + echo ""; \ + echo "=== Post-test JWT diagnostics ==="; \ + echo "Generating test JWT token..."; \ + TEST_TOKEN=$$(uv run python -c "from src.session_manager import SessionManager, AnonymousUser; sm = SessionManager('test'); print(sm.create_jwt_token(AnonymousUser()))" 2>/dev/null || echo ""); \ + if [ -n "$$TEST_TOKEN" ]; then \ + echo "Testing JWT against OpenSearch..."; \ + HTTP_CODE=$$(curl -k -s -w "%{http_code}" -o /tmp/os_diag.txt -H "Authorization: Bearer $$TEST_TOKEN" -H "Content-Type: application/json" https://localhost:9200/documents/_search -d '{"query":{"match_all":{}}}' 2>&1); \ + echo "HTTP $$HTTP_CODE: $$(cat /tmp/os_diag.txt | head -c 150)"; \ + fi; \ + echo "================================="; \ + echo ""; \ + echo "Tearing down infra"; \ + uv run python scripts/docling_ctl.py stop || true; \ + docker compose down -v || true; \ + exit $$TEST_RESULT lint: @echo "🔍 Running linting checks..." @@ -180,19 +275,19 @@ lint: # Service status status: @echo "📊 Container status:" - @docker-compose ps 2>/dev/null || echo "No containers running" + @docker compose ps 2>/dev/null || echo "No containers running" health: @echo "🏥 Health check:" @echo "Backend: $$(curl -s http://localhost:8000/health 2>/dev/null || echo 'Not responding')" @echo "Langflow: $$(curl -s http://localhost:7860/health 2>/dev/null || echo 'Not responding')" - @echo "OpenSearch: $$(curl -s -k -u admin:$(shell grep OPENSEARCH_PASSWORD .env | cut -d= -f2) https://localhost:9200 2>/dev/null | jq -r .tagline 2>/dev/null || echo 'Not responding')" + @echo "OpenSearch: $$(curl -s -k -u admin:$${OPENSEARCH_PASSWORD} https://localhost:9200 2>/dev/null | jq -r .tagline 2>/dev/null || echo 'Not responding')" # Database operations db-reset: @echo "🗄️ Resetting OpenSearch indices..." - curl -X DELETE "http://localhost:9200/documents" -u admin:$$(grep OPENSEARCH_PASSWORD .env | cut -d= -f2) || true - curl -X DELETE "http://localhost:9200/knowledge_filters" -u admin:$$(grep OPENSEARCH_PASSWORD .env | cut -d= -f2) || true + curl -X DELETE "http://localhost:9200/documents" -u admin:$${OPENSEARCH_PASSWORD} || true + curl -X DELETE "http://localhost:9200/knowledge_filters" -u admin:$${OPENSEARCH_PASSWORD} || true @echo "Indices reset. Restart backend to recreate." # Flow management @@ -215,4 +310,4 @@ setup: @echo "⚙️ Setting up development environment..." @if [ ! -f .env ]; then cp .env.example .env && echo "📝 Created .env from template"; fi @$(MAKE) install - @echo "✅ Setup complete! Run 'make dev' to start." \ No newline at end of file + @echo "✅ Setup complete! Run 'make dev' to start." diff --git a/README.md b/README.md index a0178f28..a7abbbe6 100644 --- a/README.md +++ b/README.md @@ -2,20 +2,6 @@ # OpenRAG - -
- 🚀 Quick Start   |   - 💻 TUI Interface   |   - 🐳 Docker Deployment   |   - ⚙️ Development   |   - 🔧 Troubleshooting -
- - - -OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations. Users can upload, process, and query documents through a chat interface backed by large language models and semantic search capabilities. The system utilizes Langflow for document ingestion, retrieval workflows, and intelligent nudges, providing a seamless RAG experience. Built with Starlette, Next.js, OpenSearch, and Langflow integration. [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/phact/openrag) - -
Langflow    @@ -24,144 +10,124 @@ OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables Starlette    Next.js - +    + Ask DeepWiki
+OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations. Users can upload, process, and query documents through a chat interface backed by large language models and semantic search capabilities. The system utilizes Langflow for document ingestion, retrieval workflows, and intelligent nudges, providing a seamless RAG experience. Built with Starlette, Next.js, OpenSearch, and Langflow integration. + +
+ Quickstart   |   + TUI Interface   |   + Docker Deployment   |   + Development   |   + Troubleshooting +
+## Quickstart +Use the OpenRAG Terminal User Interface (TUI) to manage your OpenRAG installation without complex command-line operations. +To launch OpenRAG with the TUI, do the following: -## 🚀 Quick Start +1. Clone the OpenRAG repository. + ```bash + git clone https://github.com/langflow-ai/openrag.git + cd openrag + ``` -### Prerequisites +2. To start the TUI, from the repository root, run: + ```bash + # Install dependencies first + uv sync + + # Launch the TUI + uv run openrag + ``` -- Docker or Podman with Compose installed -- Make (for development commands) + The TUI opens and guides you through OpenRAG setup. -### 1. Environment Setup +For the full TUI guide, see [TUI](docs/docs/get-started/tui.mdx). -```bash -# Clone and setup environment -git clone https://github.com/langflow-ai/openrag.git -cd openrag -make setup # Creates .env and installs dependencies -``` +## Docker Deployment -### 2. Configure Environment +If you prefer to use Docker to run OpenRAG, the repository includes two Docker Compose `.yml` files. +They deploy the same applications and containers, but to different environments. -Edit `.env` with your API keys and credentials: +- [`docker-compose.yml`](https://github.com/langflow-ai/openrag/blob/main/docker-compose.yml) is an OpenRAG deployment for environments with GPU support. GPU support requires an NVIDIA GPU with CUDA support and compatible NVIDIA drivers installed on the OpenRAG host machine. -```bash -# Required -OPENAI_API_KEY=your_openai_api_key -OPENSEARCH_PASSWORD=your_secure_password -LANGFLOW_SUPERUSER=admin -LANGFLOW_SUPERUSER_PASSWORD=your_secure_password -LANGFLOW_CHAT_FLOW_ID=your_chat_flow_id -LANGFLOW_INGEST_FLOW_ID=your_ingest_flow_id -NUDGES_FLOW_ID=your_nudges_flow_id -``` -See extended configuration, including ingestion and optional variables: [docs/reference/configuration.md](docs/docs/reference/configuration.md) -### 3. Start OpenRAG +- [`docker-compose-cpu.yml`](https://github.com/langflow-ai/openrag/blob/main/docker-compose-cpu.yml) is a CPU-only version of OpenRAG for systems without GPU support. Use this Docker compose file for environments where GPU drivers aren't available. -```bash -# Full stack with GPU support -make dev +Both Docker deployments depend on `docling serve` to be running on port `5001` on the host machine. This enables [Mac MLX](https://opensource.apple.com/projects/mlx/) support for document processing. Installing OpenRAG with the TUI starts `docling serve` automatically, but for a Docker deployment you must manually start the `docling serve` process. -# Or CPU only -make dev-cpu -``` +To deploy OpenRAG with Docker: -Access the services: -- **Frontend**: http://localhost:3000 -- **Backend API**: http://localhost:8000 -- **Langflow**: http://localhost:7860 -- **OpenSearch**: http://localhost:9200 -- **OpenSearch Dashboards**: http://localhost:5601 +1. Clone the OpenRAG repository. + ```bash + git clone https://github.com/langflow-ai/openrag.git + cd openrag + ``` -## 🖥️ TUI Interface +2. Install dependencies. + ```bash + uv sync + ``` -OpenRAG includes a powerful Terminal User Interface (TUI) for easy setup, configuration, and monitoring. The TUI provides a user-friendly way to manage your OpenRAG installation without complex command-line operations. +3. Start `docling serve` on the host machine. + ```bash + uv run python scripts/docling_ctl.py start --port 5001 + ``` + +4. Confirm `docling serve` is running. + ``` + uv run python scripts/docling_ctl.py status + ``` -![OpenRAG TUI Interface](assets/OpenRAG_TUI_2025-09-10T13_04_11_757637.svg) + Successful result: + ```bash + Status: running + Endpoint: http://127.0.0.1:5001 + Docs: http://127.0.0.1:5001/docs + PID: 27746 + ``` -### Launching the TUI +5. Build and start all services. -```bash -# Install dependencies first -uv sync + For the GPU-accelerated deployment, run: + ```bash + docker compose build + docker compose up -d + ``` -# Launch the TUI -uv run openrag -``` + For environments without GPU support, run: + ```bash + docker compose -f docker-compose-cpu.yml up -d + ``` -### TUI Features + The OpenRAG Docker Compose file starts five containers: + | Container Name | Default Address | Purpose | + |---|---|---| + | OpenRAG Backend | http://localhost:8000 | FastAPI server and core functionality. | + | OpenRAG Frontend | http://localhost:3000 | React web interface for users. | + | Langflow | http://localhost:7860 | AI workflow engine and flow management. | + | OpenSearch | http://localhost:9200 | Vector database for document storage. | + | OpenSearch Dashboards | http://localhost:5601 | Database administration interface. | -See the full TUI guide for features, navigation, and benefits: [docs/get-started/tui.mdx](docs/docs/get-started/tui.mdx) +6. Access the OpenRAG application at `http://localhost:3000` and continue with the [Quickstart](docs/docs/get-started/quickstart.mdx). + To stop `docling serve`, run: + + ```bash + uv run python scripts/docling_ctl.py stop + ``` +For more information, see [Deploy with Docker](docs/docs/get-started/docker.mdx). +## Troubleshooting -## 🐳 Docker Deployment +For common issues and fixes, see [Troubleshoot](docs/docs/support/troubleshoot.mdx). -### Standard Deployment +## Development -```bash -# Build and start all services -docker compose build -docker compose up -d -``` - -### CPU-Only Deployment - -For environments without GPU support: - -```bash -docker compose -f docker-compose-cpu.yml up -d -``` - -More deployment commands and tips: [docs/get-started/docker.mdx](docs/docs/get-started/docker.mdx) - -## 🔧 Troubleshooting - -### Podman on macOS - -If using Podman on macOS, you may need to increase VM memory: - -```bash -podman machine stop -podman machine rm -podman machine init --memory 8192 # 8 GB example -podman machine start -``` - -### Common Issues - -See common issues and fixes: [docs/support/troubleshoot.mdx](docs/docs/reference/troubleshoot.mdx) - - - -## 🛠️ Development - -For developers wanting to contribute to OpenRAG or set up a development environment, please see our comprehensive development guide: - -**[📚 See CONTRIBUTING.md for detailed development instructions](CONTRIBUTING.md)** - -The contributing guide includes: -- Complete development environment setup -- Local development workflows -- Testing and debugging procedures -- Code style guidelines -- Architecture overview -- Pull request guidelines - -### Quick Development Commands - -```bash -make help # See all available commands -make setup # Initial development setup -make infra # Start infrastructure services -make backend # Run backend locally -make frontend # Run frontend locally -``` \ No newline at end of file +For developers wanting to contribute to OpenRAG or set up a development environment, see [CONTRIBUTING.md](CONTRIBUTING.md). \ No newline at end of file diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml index 0c09254a..c0af8a01 100644 --- a/docker-compose-cpu.yml +++ b/docker-compose-cpu.yml @@ -74,7 +74,7 @@ services: volumes: - ./documents:/app/documents:Z - ./keys:/app/keys:Z - - ./flows:/app/flows:Z + - ./flows:/app/flows:U,z openrag-frontend: image: phact/openrag-frontend:${OPENRAG_VERSION:-latest} @@ -91,7 +91,7 @@ services: langflow: volumes: - - ./flows:/app/flows:Z + - ./flows:/app/flows:U,z image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest} # build: # context: . @@ -108,6 +108,7 @@ services: - OWNER_NAME=None - OWNER_EMAIL=None - CONNECTOR_TYPE=system + - CONNECTOR_TYPE_URL=url - OPENRAG-QUERY-FILTER="{}" - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} - FILENAME=None diff --git a/docker-compose.yml b/docker-compose.yml index be9bcbc9..df8a3228 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,7 +73,7 @@ services: volumes: - ./documents:/app/documents:Z - ./keys:/app/keys:Z - - ./flows:/app/flows:z + - ./flows:/app/flows:U,z gpus: all openrag-frontend: @@ -81,7 +81,6 @@ services: # build: # context: . # dockerfile: Dockerfile.frontend - #dockerfile: Dockerfile.frontend container_name: openrag-frontend depends_on: - openrag-backend @@ -92,7 +91,7 @@ services: langflow: volumes: - - ./flows:/app/flows:z + - ./flows:/app/flows:U,z image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest} # build: # context: . @@ -109,6 +108,7 @@ services: - OWNER_NAME=None - OWNER_EMAIL=None - CONNECTOR_TYPE=system + - CONNECTOR_TYPE_URL=url - OPENRAG-QUERY-FILTER="{}" - FILENAME=None - MIMETYPE=None diff --git a/docs/docs/_partial-external-preview.mdx b/docs/docs/_partial-external-preview.mdx deleted file mode 100644 index 8563720c..00000000 --- a/docs/docs/_partial-external-preview.mdx +++ /dev/null @@ -1,4 +0,0 @@ -:::info -OpenRAG is is currently in public preview. -Development is ongoing, and the features and functionality are subject to change. -::: \ No newline at end of file diff --git a/docs/docs/core-components/agents.mdx b/docs/docs/core-components/agents.mdx index 3ee4617b..ea4c05bd 100644 --- a/docs/docs/core-components/agents.mdx +++ b/docs/docs/core-components/agents.mdx @@ -7,9 +7,6 @@ import Icon from "@site/src/components/icon/icon"; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import PartialModifyFlows from '@site/docs/_partial-modify-flows.mdx'; -import PartialExternalPreview from '@site/docs/_partial-external-preview.mdx'; - - OpenRAG leverages Langflow's Agent component to power the OpenRAG OpenSearch Agent flow. @@ -34,11 +31,11 @@ In an agentic context, tools are functions that the agent can run to perform tas -## Use the OpenRAG OpenSearch Agent flow +## Use the OpenRAG OpenSearch Agent flow {#flow} If you've chatted with your knowledge in OpenRAG, you've already experienced the OpenRAG OpenSearch Agent chat flow. To switch OpenRAG over to the [Langflow visual editor](https://docs.langflow.org/concepts-overview) and view the OpenRAG OpenSearch Agentflow, click