Merge branch 'main' into issue-502-monolithic-pages
This commit is contained in:
April M
commit
0f02b5fbc0
49 changed files with 13012 additions and 5541 deletions
77
.github/workflows/build-multiarch.yml
vendored
77
.github/workflows/build-multiarch.yml
vendored
|
|
@ -53,58 +53,63 @@ jobs:
|
|||
# backend
|
||||
- image: backend
|
||||
file: ./Dockerfile.backend
|
||||
tag: phact/openrag-backend
|
||||
tag: langflowai/openrag-backend
|
||||
platform: linux/amd64
|
||||
arch: amd64
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
- image: backend
|
||||
file: ./Dockerfile.backend
|
||||
tag: phact/openrag-backend
|
||||
tag: langflowai/openrag-backend
|
||||
platform: linux/arm64
|
||||
arch: arm64
|
||||
runs-on: [self-hosted, linux, ARM64, langflow-ai-arm64-2]
|
||||
#runs-on: [self-hosted, linux, ARM64, langflow-ai-arm64-2]
|
||||
runs-on: RagRunner
|
||||
|
||||
# frontend
|
||||
- image: frontend
|
||||
file: ./Dockerfile.frontend
|
||||
tag: phact/openrag-frontend
|
||||
tag: langflowai/openrag-frontend
|
||||
platform: linux/amd64
|
||||
arch: amd64
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
- image: frontend
|
||||
file: ./Dockerfile.frontend
|
||||
tag: phact/openrag-frontend
|
||||
tag: langflowai/openrag-frontend
|
||||
platform: linux/arm64
|
||||
arch: arm64
|
||||
runs-on: [self-hosted, linux, ARM64, langflow-ai-arm64-2]
|
||||
#runs-on: [self-hosted, linux, ARM64, langflow-ai-arm64-2]
|
||||
runs-on: RagRunner
|
||||
|
||||
# langflow
|
||||
- image: langflow
|
||||
file: ./Dockerfile.langflow
|
||||
tag: phact/openrag-langflow
|
||||
tag: langflowai/openrag-langflow
|
||||
platform: linux/amd64
|
||||
arch: amd64
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
- image: langflow
|
||||
file: ./Dockerfile.langflow
|
||||
tag: phact/openrag-langflow
|
||||
tag: langflowai/openrag-langflow
|
||||
platform: linux/arm64
|
||||
arch: arm64
|
||||
runs-on: self-hosted
|
||||
#runs-on: self-hosted
|
||||
runs-on: RagRunner
|
||||
|
||||
# opensearch
|
||||
- image: opensearch
|
||||
file: ./Dockerfile
|
||||
tag: phact/openrag-opensearch
|
||||
tag: langflowai/openrag-opensearch
|
||||
platform: linux/amd64
|
||||
arch: amd64
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
- image: opensearch
|
||||
file: ./Dockerfile
|
||||
tag: phact/openrag-opensearch
|
||||
tag: langflowai/openrag-opensearch
|
||||
platform: linux/arm64
|
||||
arch: arm64
|
||||
runs-on: [self-hosted, linux, ARM64, langflow-ai-arm64-2]
|
||||
#runs-on: [self-hosted, linux, ARM64, langflow-ai-arm64-2]
|
||||
#runs-on: self-hosted
|
||||
runs-on: RagRunner
|
||||
|
||||
runs-on: ${{ matrix.runs-on }}
|
||||
|
||||
|
|
@ -165,40 +170,40 @@ jobs:
|
|||
VERSION=${{ steps.version.outputs.version }}
|
||||
|
||||
# Create versioned tags
|
||||
docker buildx imagetools create -t phact/openrag-backend:$VERSION \
|
||||
phact/openrag-backend:$VERSION-amd64 \
|
||||
phact/openrag-backend:$VERSION-arm64
|
||||
docker buildx imagetools create -t langflowai/openrag-backend:$VERSION \
|
||||
langflowai/openrag-backend:$VERSION-amd64 \
|
||||
langflowai/openrag-backend:$VERSION-arm64
|
||||
|
||||
docker buildx imagetools create -t phact/openrag-frontend:$VERSION \
|
||||
phact/openrag-frontend:$VERSION-amd64 \
|
||||
phact/openrag-frontend:$VERSION-arm64
|
||||
docker buildx imagetools create -t langflowai/openrag-frontend:$VERSION \
|
||||
langflowai/openrag-frontend:$VERSION-amd64 \
|
||||
langflowai/openrag-frontend:$VERSION-arm64
|
||||
|
||||
docker buildx imagetools create -t phact/openrag-langflow:$VERSION \
|
||||
phact/openrag-langflow:$VERSION-amd64 \
|
||||
phact/openrag-langflow:$VERSION-arm64
|
||||
docker buildx imagetools create -t langflowai/openrag-langflow:$VERSION \
|
||||
langflowai/openrag-langflow:$VERSION-amd64 \
|
||||
langflowai/openrag-langflow:$VERSION-arm64
|
||||
|
||||
docker buildx imagetools create -t phact/openrag-opensearch:$VERSION \
|
||||
phact/openrag-opensearch:$VERSION-amd64 \
|
||||
phact/openrag-opensearch:$VERSION-arm64
|
||||
docker buildx imagetools create -t langflowai/openrag-opensearch:$VERSION \
|
||||
langflowai/openrag-opensearch:$VERSION-amd64 \
|
||||
langflowai/openrag-opensearch:$VERSION-arm64
|
||||
|
||||
# Only update latest tags if version is numeric
|
||||
if [[ "$VERSION" =~ ^[0-9.-]+$ ]]; then
|
||||
echo "Updating latest tags for production release: $VERSION"
|
||||
docker buildx imagetools create -t phact/openrag-backend:latest \
|
||||
phact/openrag-backend:$VERSION-amd64 \
|
||||
phact/openrag-backend:$VERSION-arm64
|
||||
docker buildx imagetools create -t langflowai/openrag-backend:latest \
|
||||
langflowai/openrag-backend:$VERSION-amd64 \
|
||||
langflowai/openrag-backend:$VERSION-arm64
|
||||
|
||||
docker buildx imagetools create -t phact/openrag-frontend:latest \
|
||||
phact/openrag-frontend:$VERSION-amd64 \
|
||||
phact/openrag-frontend:$VERSION-arm64
|
||||
docker buildx imagetools create -t langflowai/openrag-frontend:latest \
|
||||
langflowai/openrag-frontend:$VERSION-amd64 \
|
||||
langflowai/openrag-frontend:$VERSION-arm64
|
||||
|
||||
docker buildx imagetools create -t phact/openrag-langflow:latest \
|
||||
phact/openrag-langflow:$VERSION-amd64 \
|
||||
phact/openrag-langflow:$VERSION-arm64
|
||||
docker buildx imagetools create -t langflowai/openrag-langflow:latest \
|
||||
langflowai/openrag-langflow:$VERSION-amd64 \
|
||||
langflowai/openrag-langflow:$VERSION-arm64
|
||||
|
||||
docker buildx imagetools create -t phact/openrag-opensearch:latest \
|
||||
phact/openrag-opensearch:$VERSION-amd64 \
|
||||
phact/openrag-opensearch:$VERSION-arm64
|
||||
docker buildx imagetools create -t langflowai/openrag-opensearch:latest \
|
||||
langflowai/openrag-opensearch:$VERSION-amd64 \
|
||||
langflowai/openrag-opensearch:$VERSION-arm64
|
||||
else
|
||||
echo "Skipping latest tags - version: $VERSION (not numeric)"
|
||||
fi
|
||||
|
|
|
|||
19
.github/workflows/test-integration.yml
vendored
19
.github/workflows/test-integration.yml
vendored
|
|
@ -31,14 +31,23 @@ jobs:
|
|||
|
||||
steps:
|
||||
- run: df -h
|
||||
#- name: "node-cleanup"
|
||||
#run: |
|
||||
# sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
# sudo docker image prune --all --force
|
||||
# sudo docker builder prune -a
|
||||
|
||||
- name: Cleanup Docker cache
|
||||
run: |
|
||||
docker system prune -af || true
|
||||
docker builder prune -af || true
|
||||
docker-compose -f docker-compose.yml down -v --remove-orphans || true
|
||||
|
||||
- run: df -h
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Verify workspace
|
||||
run: |
|
||||
echo "Current directory: $(pwd)"
|
||||
echo "Workspace: ${GITHUB_WORKSPACE}"
|
||||
ls -la
|
||||
|
||||
- name: Set up UV
|
||||
uses: astral-sh/setup-uv@v3
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ COPY pyproject.toml uv.lock ./
|
|||
RUN uv sync
|
||||
|
||||
# Copy sample document and warmup script for docling
|
||||
COPY documents/warmup_ocr.pdf ./
|
||||
COPY openrag-documents/warmup_ocr.pdf ./
|
||||
COPY warm_up_docling.py ./
|
||||
RUN uv run docling-tools models download
|
||||
RUN uv run python - <<'PY'
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
FROM langflowai/langflow-nightly:1.7.0.dev5
|
||||
FROM langflowai/langflow-nightly:1.7.0.dev21
|
||||
|
||||
EXPOSE 7860
|
||||
|
||||
|
|
|
|||
10
Makefile
10
Makefile
|
|
@ -210,7 +210,7 @@ test-ci:
|
|||
echo "Pulling latest images..."; \
|
||||
docker compose -f docker-compose-cpu.yml pull; \
|
||||
echo "Building OpenSearch image override..."; \
|
||||
docker build --no-cache -t phact/openrag-opensearch:latest -f Dockerfile .; \
|
||||
docker build --no-cache -t langflowai/openrag-opensearch:latest -f Dockerfile .; \
|
||||
echo "Starting infra (OpenSearch + Dashboards + Langflow) with CPU containers"; \
|
||||
docker compose -f docker-compose-cpu.yml up -d opensearch dashboards langflow; \
|
||||
echo "Starting docling-serve..."; \
|
||||
|
|
@ -288,10 +288,10 @@ test-ci-local:
|
|||
echo "Cleaning up old containers and volumes..."; \
|
||||
docker compose -f docker-compose-cpu.yml down -v 2>/dev/null || true; \
|
||||
echo "Building all images locally..."; \
|
||||
docker build -t phact/openrag-opensearch:latest -f Dockerfile .; \
|
||||
docker build -t phact/openrag-backend:latest -f Dockerfile.backend .; \
|
||||
docker build -t phact/openrag-frontend:latest -f Dockerfile.frontend .; \
|
||||
docker build -t phact/openrag-langflow:latest -f Dockerfile.langflow .; \
|
||||
docker build -t langflowai/openrag-opensearch:latest -f Dockerfile .; \
|
||||
docker build -t langflowai/openrag-backend:latest -f Dockerfile.backend .; \
|
||||
docker build -t langflowai/openrag-frontend:latest -f Dockerfile.frontend .; \
|
||||
docker build -t langflowai/openrag-langflow:latest -f Dockerfile.langflow .; \
|
||||
echo "Starting infra (OpenSearch + Dashboards + Langflow) with CPU containers"; \
|
||||
docker compose -f docker-compose-cpu.yml up -d opensearch dashboards langflow; \
|
||||
echo "Starting docling-serve..."; \
|
||||
|
|
|
|||
42
README.md
42
README.md
|
|
@ -18,27 +18,29 @@ OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables
|
|||
</div>
|
||||
<div align="center">
|
||||
<a href="#quickstart" style="color: #0366d6;">Quickstart</a> |
|
||||
<a href="#tui-interface" style="color: #0366d6;">TUI Interface</a> |
|
||||
<a href="#docker-deployment" style="color: #0366d6;">Docker Deployment</a> |
|
||||
<a href="#install-python-package" style="color: #0366d6;">Python package</a> |
|
||||
<a href="#docker-or-podman-installation" style="color: #0366d6;">Docker or Podman</a> |
|
||||
<a href="#development" style="color: #0366d6;">Development</a> |
|
||||
<a href="#troubleshooting" style="color: #0366d6;">Troubleshooting</a>
|
||||
</div>
|
||||
|
||||
## Quickstart
|
||||
|
||||
To quickly run OpenRAG without creating or modifying any project files, use `uvx`:
|
||||
To run OpenRAG without creating or modifying any project files, use `uvx`:
|
||||
|
||||
```bash
|
||||
uvx openrag
|
||||
```
|
||||
This runs OpenRAG without installing it to your project or globally.
|
||||
To run a specific version of OpenRAG, add the version to the command, such as: `uvx --from openrag==0.1.25 openrag`.
|
||||
|
||||
This command runs OpenRAG without installing it to your project or globally.
|
||||
|
||||
To run a specific version of OpenRAG, run `uvx --from openrag==VERSION openrag`.
|
||||
|
||||
## Install Python package
|
||||
|
||||
To first set up a project and then install the OpenRAG Python package, do the following:
|
||||
To add the OpenRAG Python package to a Python project, use `uv`:
|
||||
|
||||
1. Create a new project with a virtual environment using `uv init`.
|
||||
1. Create a new project with a virtual environment using `uv init`:
|
||||
|
||||
```bash
|
||||
uv init YOUR_PROJECT_NAME
|
||||
|
|
@ -48,33 +50,33 @@ To first set up a project and then install the OpenRAG Python package, do the fo
|
|||
The `(venv)` prompt doesn't change, but `uv` commands will automatically use the project's virtual environment.
|
||||
For more information on virtual environments, see the [uv documentation](https://docs.astral.sh/uv/pip/environments).
|
||||
|
||||
2. Add OpenRAG to your project.
|
||||
2. Add OpenRAG to your project:
|
||||
|
||||
```bash
|
||||
uv add openrag
|
||||
```
|
||||
|
||||
To add a specific version of OpenRAG:
|
||||
```bash
|
||||
uv add openrag==0.1.25
|
||||
```
|
||||
To add a specific version of OpenRAG, run `uv add openrag==VERSION`.
|
||||
|
||||
3. Start the OpenRAG terminal user interface (TUI):
|
||||
|
||||
3. Start the OpenRAG TUI.
|
||||
```bash
|
||||
uv run openrag
|
||||
```
|
||||
|
||||
4. Continue with the [Quickstart](https://docs.openr.ag/quickstart).
|
||||
|
||||
For the full TUI installation guide, see [TUI](https://docs.openr.ag/install).
|
||||
For all installation options, see the [OpenRAG installation guide](https://docs.openr.ag/install).
|
||||
|
||||
## Docker or Podman installation
|
||||
|
||||
For more information, see [Install OpenRAG containers](https://docs.openr.ag/docker).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
For common issues and fixes, see [Troubleshoot](https://docs.openr.ag/support/troubleshoot).
|
||||
By default, OpenRAG automatically starts the required containers and helps you manage them.
|
||||
To install OpenRAG with self-managed containers, see the [OpenRAG installation guide](https://docs.openr.ag/docker).
|
||||
|
||||
## Development
|
||||
|
||||
For developers wanting to contribute to OpenRAG or set up a development environment, see [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||
For developers wanting to contribute to OpenRAG or set up a development environment, see [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
For common issues and fixes, see [Troubleshoot OpenRAG](https://docs.openr.ag/support/troubleshoot).
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
services:
|
||||
opensearch:
|
||||
image: phact/openrag-opensearch:${OPENRAG_VERSION:-latest}
|
||||
image: langflowai/openrag-opensearch:${OPENRAG_VERSION:-latest}
|
||||
#build:
|
||||
# context: .
|
||||
# dockerfile: Dockerfile
|
||||
|
|
@ -44,7 +44,7 @@ services:
|
|||
- "5601:5601"
|
||||
|
||||
openrag-backend:
|
||||
image: phact/openrag-backend:${OPENRAG_VERSION:-latest}
|
||||
image: langflowai/openrag-backend:${OPENRAG_VERSION:-latest}
|
||||
# build:
|
||||
# context: .
|
||||
# dockerfile: Dockerfile.backend
|
||||
|
|
@ -81,12 +81,12 @@ services:
|
|||
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
|
||||
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
|
||||
volumes:
|
||||
- ./documents:/app/documents:Z
|
||||
- ./openrag-documents:/app/documents:Z
|
||||
- ./keys:/app/keys:Z
|
||||
- ./flows:/app/flows:U,z
|
||||
|
||||
openrag-frontend:
|
||||
image: phact/openrag-frontend:${OPENRAG_VERSION:-latest}
|
||||
image: langflowai/openrag-frontend:${OPENRAG_VERSION:-latest}
|
||||
# build:
|
||||
# context: .
|
||||
# dockerfile: Dockerfile.frontend
|
||||
|
|
@ -101,7 +101,7 @@ services:
|
|||
langflow:
|
||||
volumes:
|
||||
- ./flows:/app/flows:U,z
|
||||
image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest}
|
||||
image: langflowai/openrag-langflow:${LANGFLOW_VERSION:-latest}
|
||||
# build:
|
||||
# context: .
|
||||
# dockerfile: Dockerfile.langflow
|
||||
|
|
@ -129,7 +129,8 @@ services:
|
|||
- FILENAME=None
|
||||
- MIMETYPE=None
|
||||
- FILESIZE=0
|
||||
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE
|
||||
- SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-}
|
||||
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL
|
||||
- LANGFLOW_LOG_LEVEL=DEBUG
|
||||
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
|
||||
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
|
||||
|
|
|
|||
7
docker-compose.gpu.yml
Normal file
7
docker-compose.gpu.yml
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
services:
|
||||
openrag-backend:
|
||||
environment:
|
||||
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
- NVIDIA_VISIBLE_DEVICES=all
|
||||
gpus: all
|
||||
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
services:
|
||||
opensearch:
|
||||
image: phact/openrag-opensearch:${OPENRAG_VERSION:-latest}
|
||||
image: langflowai/openrag-opensearch:${OPENRAG_VERSION:-latest}
|
||||
#build:
|
||||
#context: .
|
||||
#dockerfile: Dockerfile
|
||||
|
|
@ -44,10 +44,10 @@ services:
|
|||
- "5601:5601"
|
||||
|
||||
openrag-backend:
|
||||
image: phact/openrag-backend:${OPENRAG_VERSION:-latest}
|
||||
# build:
|
||||
# context: .
|
||||
# dockerfile: Dockerfile.backend
|
||||
image: langflowai/openrag-backend:${OPENRAG_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.backend
|
||||
container_name: openrag-backend
|
||||
depends_on:
|
||||
- langflow
|
||||
|
|
@ -72,8 +72,6 @@ services:
|
|||
- WATSONX_ENDPOINT=${WATSONX_ENDPOINT}
|
||||
- WATSONX_PROJECT_ID=${WATSONX_PROJECT_ID}
|
||||
- OLLAMA_ENDPOINT=${OLLAMA_ENDPOINT}
|
||||
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
- NVIDIA_VISIBLE_DEVICES=all
|
||||
- GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID}
|
||||
- GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET}
|
||||
- MICROSOFT_GRAPH_OAUTH_CLIENT_ID=${MICROSOFT_GRAPH_OAUTH_CLIENT_ID}
|
||||
|
|
@ -82,16 +80,15 @@ services:
|
|||
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
|
||||
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
|
||||
volumes:
|
||||
- ./documents:/app/documents:Z
|
||||
- ./openrag-documents:/app/documents:Z
|
||||
- ./keys:/app/keys:Z
|
||||
- ./flows:/app/flows:U,z
|
||||
gpus: all
|
||||
|
||||
openrag-frontend:
|
||||
image: phact/openrag-frontend:${OPENRAG_VERSION:-latest}
|
||||
# build:
|
||||
# context: .
|
||||
# dockerfile: Dockerfile.frontend
|
||||
image: langflowai/openrag-frontend:${OPENRAG_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.frontend
|
||||
container_name: openrag-frontend
|
||||
depends_on:
|
||||
- openrag-backend
|
||||
|
|
@ -103,21 +100,21 @@ services:
|
|||
langflow:
|
||||
volumes:
|
||||
- ./flows:/app/flows:U,z
|
||||
image: phact/openrag-langflow:${LANGFLOW_VERSION:-latest}
|
||||
# build:
|
||||
# context: .
|
||||
# dockerfile: Dockerfile.langflow
|
||||
image: langflowai/openrag-langflow:${LANGFLOW_VERSION:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.langflow
|
||||
container_name: langflow
|
||||
ports:
|
||||
- "7860:7860"
|
||||
environment:
|
||||
- LANGFLOW_DEACTIVATE_TRACING=true
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
||||
- WATSONX_API_KEY=${WATSONX_API_KEY}
|
||||
- WATSONX_ENDPOINT=${WATSONX_ENDPOINT}
|
||||
- WATSONX_PROJECT_ID=${WATSONX_PROJECT_ID}
|
||||
- OLLAMA_BASE_URL=${OLLAMA_ENDPOINT}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY:-None}
|
||||
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-None}
|
||||
- WATSONX_API_KEY=${WATSONX_API_KEY:-None}
|
||||
- WATSONX_ENDPOINT=${WATSONX_ENDPOINT:-None}
|
||||
- WATSONX_PROJECT_ID=${WATSONX_PROJECT_ID:-None}
|
||||
- OLLAMA_BASE_URL=${OLLAMA_ENDPOINT:-None}
|
||||
- LANGFLOW_LOAD_FLOWS_PATH=/app/flows
|
||||
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
|
||||
- JWT=None
|
||||
|
|
@ -127,11 +124,13 @@ services:
|
|||
- CONNECTOR_TYPE=system
|
||||
- CONNECTOR_TYPE_URL=url
|
||||
- OPENRAG-QUERY-FILTER="{}"
|
||||
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
|
||||
- FILENAME=None
|
||||
- MIMETYPE=None
|
||||
- FILESIZE=0
|
||||
- SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-}
|
||||
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
|
||||
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE
|
||||
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL
|
||||
- LANGFLOW_LOG_LEVEL=DEBUG
|
||||
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
|
||||
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ If you are using GitHub pages for hosting, this command is a convenient way to b
|
|||
|
||||
## Update the OpenRAG documentation PDF
|
||||
|
||||
The documentation PDF at `openrag/documents/openrag-documentation.pdf` is used by the OpenRAG application, so keep it up to date.
|
||||
The documentation PDF at `openrag/openrag-documents/openrag-documentation.pdf` is used by the OpenRAG application, so keep it up to date.
|
||||
|
||||
To update the PDF, do the following:
|
||||
|
||||
|
|
@ -68,7 +68,7 @@ To remove these items, give the following prompt or something similar to your ID
|
|||
2. Check your `.mdx` files to confirm these elements are removed.
|
||||
Don't commit the changes.
|
||||
|
||||
3. From `openrag/docs`, run this command to build the site with the changes, and create a PDF at `openrag/documents`.
|
||||
3. From `openrag/docs`, run this command to build the site with the changes, and create a PDF at `openrag/openrag-documents`.
|
||||
|
||||
```
|
||||
npm run build:pdf
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ The default value is 200 characters, which represents an overlap of 20 percent i
|
|||
|
||||
### Set the local documents path {#set-the-local-documents-path}
|
||||
|
||||
The default path for local uploads is the `./documents` subdirectory in your OpenRAG installation directory. This is mounted to the `/app/documents/` directory inside the OpenRAG container. Files added to the host or container directory are visible in both locations.
|
||||
The default path for local uploads is the `./openrag-documents` subdirectory in your OpenRAG installation directory. This is mounted to the `/app/documents/` directory inside the OpenRAG container. Files added to the host or container directory are visible in both locations.
|
||||
|
||||
To change this location, modify the **Documents Paths** variable in either the [**Advanced Setup** menu](/install#setup) or in the `.env` used by Docker Compose.
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ OpenRAG has two Docker Compose files. Both files deploy the same applications an
|
|||
|
||||
- Install the following:
|
||||
|
||||
- [Python](https://www.python.org/downloads/release/python-3100/) version 3.10 to 3.13.
|
||||
- [Python](https://www.python.org/downloads/release/python-3100/) version 3.13 or later.
|
||||
- [uv](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
- [Podman](https://podman.io/docs/installation) (recommended) or [Docker](https://docs.docker.com/get-docker/).
|
||||
- [`podman-compose`](https://docs.podman.io/en/latest/markdown/podman-compose.1.html) or [Docker Compose](https://docs.docker.com/compose/install/). To use Docker Compose with Podman, you must alias Docker Compose commands to Podman commands.
|
||||
|
|
@ -187,7 +187,7 @@ docker compose up -d --force-recreate
|
|||
|
||||
Reset state by rebuilding all of your containers.
|
||||
Your OpenSearch and Langflow databases will be lost.
|
||||
Documents stored in the `./documents` directory will persist, since the directory is mounted as a volume in the OpenRAG backend container.
|
||||
Documents stored in the `./openrag-documents` directory will persist, since the directory is mounted as a volume in the OpenRAG backend container.
|
||||
|
||||
```bash
|
||||
docker compose up --build --force-recreate --remove-orphans
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ If you prefer running Podman or Docker containers and manually editing `.env` fi
|
|||
|
||||
## Prerequisites
|
||||
|
||||
- All OpenRAG installations require [Python](https://www.python.org/downloads/release/python-3100/) version 3.10 to 3.13.
|
||||
- All OpenRAG installations require [Python](https://www.python.org/downloads/release/python-3100/) version 3.13 or later.
|
||||
|
||||
- If you aren't using the automatic installer script, install the following:
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ This quickstart requires the following:
|
|||
This quickstart uses OpenAI for simplicity.
|
||||
For other providers, see the complete [installation guide](/install).
|
||||
|
||||
- [Python](https://www.python.org/downloads/release/python-3100/) version 3.10 to 3.13.
|
||||
- [Python](https://www.python.org/downloads/release/python-3100/) version 3.13 or later.
|
||||
|
||||
- Microsoft Windows only: To run OpenRAG on Windows, you must use the Windows Subsystem for Linux (WSL).
|
||||
|
||||
|
|
@ -102,7 +102,7 @@ You can click a document to view the chunks of the document as they are stored i
|
|||
|
||||
For this quickstart, use either the <Icon name="File" aria-hidden="true"/> **File** or <Icon name="Folder" aria-hidden="true"/> **Folder** upload options to load documents from your local machine.
|
||||
**Folder** uploads an entire directory.
|
||||
The default directory is the `/documents` subdirectory in your OpenRAG installation directory.
|
||||
The default directory is the `/openrag-documents` subdirectory in your OpenRAG installation directory.
|
||||
|
||||
For information about the cloud storage provider options, see [Ingest files with OAuth connectors](/ingestion#oauth-ingestion).
|
||||
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ Control how OpenRAG [processes and ingests documents](/ingestion) into your know
|
|||
| `DISABLE_INGEST_WITH_LANGFLOW` | `false` | Disable Langflow ingestion pipeline. |
|
||||
| `DOCLING_OCR_ENGINE` | - | OCR engine for document processing. |
|
||||
| `OCR_ENABLED` | `false` | Enable OCR for image processing. |
|
||||
| `OPENRAG_DOCUMENTS_PATHS` | `./documents` | Document paths for ingestion. |
|
||||
| `OPENRAG_DOCUMENTS_PATHS` | `./openrag-documents` | Document paths for ingestion. |
|
||||
| `PICTURE_DESCRIPTIONS_ENABLED` | `false` | Enable picture descriptions. |
|
||||
|
||||
### Langflow settings
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
"docusaurus": "docusaurus",
|
||||
"start": "docusaurus start",
|
||||
"build": "docusaurus build",
|
||||
"build:pdf": "rm -f ../documents/openrag-documentation.pdf && npm run build && npm run serve & sleep 10 && npx docusaurus-to-pdf && pkill -f 'docusaurus serve'",
|
||||
"build:pdf": "rm -f ../openrag-documents/openrag-documentation.pdf && npm run build && npm run serve & sleep 10 && npx docusaurus-to-pdf && pkill -f 'docusaurus serve'",
|
||||
"swizzle": "docusaurus swizzle",
|
||||
"deploy": "docusaurus deploy",
|
||||
"clear": "docusaurus clear",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"baseUrl": "http://localhost:3000",
|
||||
"entryPoint": "http://localhost:3000",
|
||||
"outputDir": "../documents/openrag-documentation.pdf",
|
||||
"outputDir": "../openrag-documents/openrag-documentation.pdf",
|
||||
"customStyles": "table { max-width: 3500px !important; } .navbar, .footer, .breadcrumbs { display: none !important; }",
|
||||
"forceImages": true
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -4,6 +4,7 @@ import {
|
|||
useQueryClient,
|
||||
} from "@tanstack/react-query";
|
||||
import type { ParsedQueryData } from "@/contexts/knowledge-filter-context";
|
||||
import { SEARCH_CONSTANTS } from "@/lib/constants";
|
||||
|
||||
export interface SearchPayload {
|
||||
query: string;
|
||||
|
|
@ -70,13 +71,16 @@ export const useGetSearchQuery = (
|
|||
|
||||
async function getFiles(): Promise<File[]> {
|
||||
try {
|
||||
// For wildcard queries, use a high limit to get all files
|
||||
// Otherwise use the limit from queryData or default to 100
|
||||
const isWildcardQuery = effectiveQuery.trim() === "*" || effectiveQuery.trim() === "";
|
||||
const searchLimit = isWildcardQuery
|
||||
? SEARCH_CONSTANTS.WILDCARD_QUERY_LIMIT
|
||||
: (queryData?.limit || 100);
|
||||
|
||||
const searchPayload: SearchPayload = {
|
||||
query: effectiveQuery,
|
||||
limit:
|
||||
queryData?.limit ||
|
||||
(effectiveQuery.trim() === "*" || effectiveQuery.trim() === ""
|
||||
? 10000
|
||||
: 10), // Maximum allowed limit for wildcard searches
|
||||
limit: searchLimit,
|
||||
scoreThreshold: queryData?.scoreThreshold || 0,
|
||||
};
|
||||
if (queryData?.filters) {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,11 @@ import {
|
|||
DialogTrigger,
|
||||
} from "@/components/ui/dialog";
|
||||
import { StatusBadge } from "@/components/ui/status-badge";
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import {
|
||||
DeleteConfirmationDialog,
|
||||
formatFilesToDelete,
|
||||
|
|
@ -156,9 +161,16 @@ function SearchPage() {
|
|||
}}
|
||||
>
|
||||
{getSourceIcon(data?.connector_type)}
|
||||
<span className="font-medium text-foreground truncate">
|
||||
{value}
|
||||
</span>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<span className="font-medium text-foreground truncate">
|
||||
{value}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top" align="start">
|
||||
{value}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -91,6 +91,10 @@ export function Navigation({
|
|||
|
||||
const { loading } = useLoadingStore();
|
||||
|
||||
useEffect(() => {
|
||||
console.log("loading", loading);
|
||||
}, [loading]);
|
||||
|
||||
const [previousConversationCount, setPreviousConversationCount] = useState(0);
|
||||
const [deleteModalOpen, setDeleteModalOpen] = useState(false);
|
||||
const [conversationToDelete, setConversationToDelete] =
|
||||
|
|
@ -391,8 +395,70 @@ export function Navigation({
|
|||
No conversations yet
|
||||
</div>
|
||||
) : (
|
||||
conversations.map((conversation) => (
|
||||
<button
|
||||
<>
|
||||
{/* Optimistic rendering: Show placeholder conversation button while loading */}
|
||||
{(() => {
|
||||
// Show placeholder when:
|
||||
// 1. Loading is true AND conversation doesn't exist yet (creating new conversation), OR
|
||||
// 2. currentConversationId exists but isn't in conversations yet (gap between response and list update)
|
||||
const conversationExists = currentConversationId
|
||||
? conversations.some(
|
||||
(conv) => conv.response_id === currentConversationId,
|
||||
)
|
||||
: false;
|
||||
|
||||
const shouldShowPlaceholder =
|
||||
!conversationExists &&
|
||||
(loading ||
|
||||
(currentConversationId !== null &&
|
||||
currentConversationId !== undefined));
|
||||
|
||||
// Use placeholderConversation if available
|
||||
// Otherwise create a placeholder with currentConversationId if it exists
|
||||
// Or use a temporary ID if we're loading but don't have an ID yet
|
||||
const placeholderToShow =
|
||||
placeholderConversation
|
||||
? placeholderConversation
|
||||
: currentConversationId
|
||||
? {
|
||||
response_id: currentConversationId,
|
||||
title: "",
|
||||
endpoint: endpoint,
|
||||
messages: [],
|
||||
total_messages: 0,
|
||||
}
|
||||
: loading
|
||||
? {
|
||||
response_id: `loading-${Date.now()}`,
|
||||
title: "",
|
||||
endpoint: endpoint,
|
||||
messages: [],
|
||||
total_messages: 0,
|
||||
}
|
||||
: null;
|
||||
|
||||
return (
|
||||
shouldShowPlaceholder &&
|
||||
placeholderToShow && (
|
||||
<button
|
||||
key={placeholderToShow.response_id}
|
||||
type="button"
|
||||
className="w-full px-3 h-11 rounded-lg bg-accent group relative text-left cursor-not-allowed"
|
||||
disabled
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="text-sm font-medium text-muted-foreground truncate">
|
||||
<span className="thinking-dots"></span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
)
|
||||
);
|
||||
})()}
|
||||
{conversations.map((conversation) => (
|
||||
<button
|
||||
key={conversation.response_id}
|
||||
type="button"
|
||||
className={`w-full px-3 h-11 rounded-lg group relative text-left ${
|
||||
|
|
@ -467,7 +533,8 @@ export function Navigation({
|
|||
</DropdownMenu>
|
||||
</div>
|
||||
</button>
|
||||
))
|
||||
))}
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,13 @@ export const UI_CONSTANTS = {
|
|||
MAX_SYSTEM_PROMPT_CHARS: 4000,
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Search Constants
|
||||
*/
|
||||
export const SEARCH_CONSTANTS = {
|
||||
WILDCARD_QUERY_LIMIT: 10000, // Maximum allowed limit for wildcard searches
|
||||
} as const;
|
||||
|
||||
export const ANIMATION_DURATION = 0.4;
|
||||
export const SIDEBAR_WIDTH = 280;
|
||||
export const HEADER_HEIGHT = 54;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||
|
||||
[project]
|
||||
name = "openrag"
|
||||
version = "0.1.38"
|
||||
version = "0.1.42"
|
||||
description = "Add your description here"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.13"
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from config.settings import (
|
|||
clients,
|
||||
get_openrag_config,
|
||||
config_manager,
|
||||
is_no_auth_mode,
|
||||
)
|
||||
from api.provider_validation import validate_provider_setup
|
||||
|
||||
|
|
@ -614,7 +615,7 @@ async def update_settings(request, session_manager):
|
|||
)
|
||||
logger.info("Set OLLAMA_BASE_URL global variable in Langflow")
|
||||
|
||||
# Update model values across flows if provider or model changed
|
||||
# Update LLM model values across flows if provider or model changed
|
||||
if "llm_provider" in body or "llm_model" in body:
|
||||
flows_service = _get_flows_service()
|
||||
llm_provider = current_config.agent.llm_provider.lower()
|
||||
|
|
@ -629,19 +630,49 @@ async def update_settings(request, session_manager):
|
|||
f"Successfully updated Langflow flows for LLM provider {llm_provider}"
|
||||
)
|
||||
|
||||
# Update SELECTED_EMBEDDING_MODEL global variable (no flow updates needed)
|
||||
if "embedding_provider" in body or "embedding_model" in body:
|
||||
flows_service = _get_flows_service()
|
||||
embedding_provider = current_config.knowledge.embedding_provider.lower()
|
||||
embedding_provider_config = current_config.get_embedding_provider_config()
|
||||
embedding_endpoint = getattr(embedding_provider_config, "endpoint", None)
|
||||
await flows_service.change_langflow_model_value(
|
||||
embedding_provider,
|
||||
embedding_model=current_config.knowledge.embedding_model,
|
||||
endpoint=embedding_endpoint,
|
||||
await clients._create_langflow_global_variable(
|
||||
"SELECTED_EMBEDDING_MODEL", current_config.knowledge.embedding_model, modify=True
|
||||
)
|
||||
logger.info(
|
||||
f"Successfully updated Langflow flows for embedding provider {embedding_provider}"
|
||||
f"Set SELECTED_EMBEDDING_MODEL global variable to {current_config.knowledge.embedding_model}"
|
||||
)
|
||||
|
||||
# Update MCP servers with provider credentials
|
||||
try:
|
||||
from services.langflow_mcp_service import LangflowMCPService
|
||||
from utils.langflow_headers import build_mcp_global_vars_from_config
|
||||
|
||||
mcp_service = LangflowMCPService()
|
||||
|
||||
# Build global vars using utility function
|
||||
mcp_global_vars = build_mcp_global_vars_from_config(current_config)
|
||||
|
||||
# In no-auth mode, add the anonymous JWT token and user details
|
||||
if is_no_auth_mode() and session_manager:
|
||||
from session_manager import AnonymousUser
|
||||
|
||||
# Create/get anonymous JWT for no-auth mode
|
||||
anonymous_jwt = session_manager.get_effective_jwt_token(None, None)
|
||||
if anonymous_jwt:
|
||||
mcp_global_vars["JWT"] = anonymous_jwt
|
||||
|
||||
# Add anonymous user details
|
||||
anonymous_user = AnonymousUser()
|
||||
mcp_global_vars["OWNER"] = anonymous_user.user_id # "anonymous"
|
||||
mcp_global_vars["OWNER_NAME"] = f'"{anonymous_user.name}"' # "Anonymous User" (quoted)
|
||||
mcp_global_vars["OWNER_EMAIL"] = anonymous_user.email # "anonymous@localhost"
|
||||
|
||||
logger.debug("Added anonymous JWT and user details to MCP servers for no-auth mode")
|
||||
|
||||
if mcp_global_vars:
|
||||
result = await mcp_service.update_mcp_servers_with_global_vars(mcp_global_vars)
|
||||
logger.info("Updated MCP servers with provider credentials after settings change", **result)
|
||||
|
||||
except Exception as mcp_error:
|
||||
logger.warning(f"Failed to update MCP servers after settings change: {str(mcp_error)}")
|
||||
# Don't fail the entire settings update if MCP update fails
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update Langflow settings: {str(e)}")
|
||||
|
|
@ -660,7 +691,7 @@ async def update_settings(request, session_manager):
|
|||
)
|
||||
|
||||
|
||||
async def onboarding(request, flows_service):
|
||||
async def onboarding(request, flows_service, session_manager=None):
|
||||
"""Handle onboarding configuration setup"""
|
||||
try:
|
||||
# Get current configuration
|
||||
|
|
@ -928,7 +959,7 @@ async def onboarding(request, flows_service):
|
|||
)
|
||||
logger.info("Set OLLAMA_BASE_URL global variable in Langflow")
|
||||
|
||||
# Update flows with model values
|
||||
# Update flows with LLM model values
|
||||
if "llm_provider" in body or "llm_model" in body:
|
||||
llm_provider = current_config.agent.llm_provider.lower()
|
||||
llm_provider_config = current_config.get_llm_provider_config()
|
||||
|
|
@ -940,16 +971,49 @@ async def onboarding(request, flows_service):
|
|||
)
|
||||
logger.info(f"Updated Langflow flows for LLM provider {llm_provider}")
|
||||
|
||||
# Set SELECTED_EMBEDDING_MODEL global variable (no flow updates needed)
|
||||
if "embedding_provider" in body or "embedding_model" in body:
|
||||
embedding_provider = current_config.knowledge.embedding_provider.lower()
|
||||
embedding_provider_config = current_config.get_embedding_provider_config()
|
||||
embedding_endpoint = getattr(embedding_provider_config, "endpoint", None)
|
||||
await flows_service.change_langflow_model_value(
|
||||
provider=embedding_provider,
|
||||
embedding_model=current_config.knowledge.embedding_model,
|
||||
endpoint=embedding_endpoint,
|
||||
await clients._create_langflow_global_variable(
|
||||
"SELECTED_EMBEDDING_MODEL", current_config.knowledge.embedding_model, modify=True
|
||||
)
|
||||
logger.info(f"Updated Langflow flows for embedding provider {embedding_provider}")
|
||||
logger.info(
|
||||
f"Set SELECTED_EMBEDDING_MODEL global variable to {current_config.knowledge.embedding_model}"
|
||||
)
|
||||
|
||||
# Update MCP servers with provider credentials during onboarding
|
||||
try:
|
||||
from services.langflow_mcp_service import LangflowMCPService
|
||||
from utils.langflow_headers import build_mcp_global_vars_from_config
|
||||
|
||||
mcp_service = LangflowMCPService()
|
||||
|
||||
# Build global vars using utility function
|
||||
mcp_global_vars = build_mcp_global_vars_from_config(current_config)
|
||||
|
||||
# In no-auth mode, add the anonymous JWT token and user details
|
||||
if is_no_auth_mode() and session_manager:
|
||||
from session_manager import AnonymousUser
|
||||
|
||||
# Create/get anonymous JWT for no-auth mode
|
||||
anonymous_jwt = session_manager.get_effective_jwt_token(None, None)
|
||||
if anonymous_jwt:
|
||||
mcp_global_vars["JWT"] = anonymous_jwt
|
||||
|
||||
# Add anonymous user details
|
||||
anonymous_user = AnonymousUser()
|
||||
mcp_global_vars["OWNER"] = anonymous_user.user_id # "anonymous"
|
||||
mcp_global_vars["OWNER_NAME"] = f'"{anonymous_user.name}"' # "Anonymous User" (quoted)
|
||||
mcp_global_vars["OWNER_EMAIL"] = anonymous_user.email # "anonymous@localhost"
|
||||
|
||||
logger.debug("Added anonymous JWT and user details to MCP servers for no-auth mode during onboarding")
|
||||
|
||||
if mcp_global_vars:
|
||||
result = await mcp_service.update_mcp_servers_with_global_vars(mcp_global_vars)
|
||||
logger.info("Updated MCP servers with provider credentials during onboarding", **result)
|
||||
|
||||
except Exception as mcp_error:
|
||||
logger.warning(f"Failed to update MCP servers during onboarding: {str(mcp_error)}")
|
||||
# Don't fail onboarding if MCP update fails
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
|
|
|
|||
79
src/main.py
79
src/main.py
|
|
@ -2,6 +2,7 @@
|
|||
from connectors.langflow_connector_service import LangflowConnectorService
|
||||
from connectors.service import ConnectorService
|
||||
from services.flows_service import FlowsService
|
||||
from utils.container_utils import detect_container_environment
|
||||
from utils.embeddings import create_dynamic_index_body
|
||||
from utils.logging_config import configure_from_env, get_logger
|
||||
|
||||
|
|
@ -13,6 +14,7 @@ import atexit
|
|||
import mimetypes
|
||||
import multiprocessing
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from functools import partial
|
||||
|
||||
|
|
@ -300,6 +302,21 @@ async def init_index_when_ready():
|
|||
)
|
||||
|
||||
|
||||
def _get_documents_dir():
|
||||
"""Get the documents directory path, handling both Docker and local environments."""
|
||||
# In Docker, the volume is mounted at /app/documents
|
||||
# Locally, we use openrag-documents
|
||||
container_env = detect_container_environment()
|
||||
if container_env:
|
||||
path = os.path.abspath("/app/documents")
|
||||
logger.debug(f"Running in {container_env}, using container path: {path}")
|
||||
return path
|
||||
else:
|
||||
path = os.path.abspath(os.path.join(os.getcwd(), "openrag-documents"))
|
||||
logger.debug(f"Running locally, using local path: {path}")
|
||||
return path
|
||||
|
||||
|
||||
async def ingest_default_documents_when_ready(services):
|
||||
"""Scan the local documents folder and ingest files like a non-auth upload."""
|
||||
try:
|
||||
|
|
@ -307,7 +324,7 @@ async def ingest_default_documents_when_ready(services):
|
|||
"Ingesting default documents when ready",
|
||||
disable_langflow_ingest=DISABLE_INGEST_WITH_LANGFLOW,
|
||||
)
|
||||
base_dir = os.path.abspath(os.path.join(os.getcwd(), "documents"))
|
||||
base_dir = _get_documents_dir()
|
||||
if not os.path.isdir(base_dir):
|
||||
logger.info(
|
||||
"Default documents directory not found; skipping ingestion",
|
||||
|
|
@ -370,7 +387,7 @@ async def _ingest_default_documents_langflow(services, file_paths):
|
|||
|
||||
# Prepare tweaks for default documents with anonymous user metadata
|
||||
default_tweaks = {
|
||||
"OpenSearchHybrid-Ve6bS": {
|
||||
"OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4": {
|
||||
"docs_metadata": [
|
||||
{"key": "owner", "value": None},
|
||||
{"key": "owner_name", "value": anonymous_user.name},
|
||||
|
|
@ -433,6 +450,55 @@ async def _ingest_default_documents_openrag(services, file_paths):
|
|||
)
|
||||
|
||||
|
||||
async def _update_mcp_servers_with_provider_credentials(services):
|
||||
"""Update MCP servers with provider credentials at startup.
|
||||
|
||||
This is especially important for no-auth mode where users don't go through
|
||||
the OAuth login flow that would normally set these credentials.
|
||||
"""
|
||||
try:
|
||||
auth_service = services.get("auth_service")
|
||||
session_manager = services.get("session_manager")
|
||||
|
||||
if not auth_service or not auth_service.langflow_mcp_service:
|
||||
logger.debug("MCP service not available, skipping credential update")
|
||||
return
|
||||
|
||||
config = get_openrag_config()
|
||||
|
||||
# Build global vars with provider credentials using utility function
|
||||
from utils.langflow_headers import build_mcp_global_vars_from_config
|
||||
|
||||
global_vars = build_mcp_global_vars_from_config(config)
|
||||
|
||||
# In no-auth mode, add the anonymous JWT token and user details
|
||||
if is_no_auth_mode() and session_manager:
|
||||
from session_manager import AnonymousUser
|
||||
|
||||
# Create/get anonymous JWT for no-auth mode
|
||||
anonymous_jwt = session_manager.get_effective_jwt_token(None, None)
|
||||
if anonymous_jwt:
|
||||
global_vars["JWT"] = anonymous_jwt
|
||||
|
||||
# Add anonymous user details
|
||||
anonymous_user = AnonymousUser()
|
||||
global_vars["OWNER"] = anonymous_user.user_id # "anonymous"
|
||||
global_vars["OWNER_NAME"] = f'"{anonymous_user.name}"' # "Anonymous User" (quoted for spaces)
|
||||
global_vars["OWNER_EMAIL"] = anonymous_user.email # "anonymous@localhost"
|
||||
|
||||
logger.info("Added anonymous JWT and user details to MCP servers for no-auth mode")
|
||||
|
||||
if global_vars:
|
||||
result = await auth_service.langflow_mcp_service.update_mcp_servers_with_global_vars(global_vars)
|
||||
logger.info("Updated MCP servers with provider credentials at startup", **result)
|
||||
else:
|
||||
logger.debug("No provider credentials configured, skipping MCP server update")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Failed to update MCP servers with provider credentials at startup", error=str(e))
|
||||
# Don't fail startup if MCP update fails
|
||||
|
||||
|
||||
async def startup_tasks(services):
|
||||
"""Startup tasks"""
|
||||
logger.info("Starting startup tasks")
|
||||
|
|
@ -445,6 +511,9 @@ async def startup_tasks(services):
|
|||
|
||||
# Configure alerting security
|
||||
await configure_alerting_security()
|
||||
|
||||
# Update MCP servers with provider credentials (especially important for no-auth mode)
|
||||
await _update_mcp_servers_with_provider_credentials(services)
|
||||
|
||||
|
||||
async def initialize_services():
|
||||
|
|
@ -1052,7 +1121,11 @@ async def create_app():
|
|||
Route(
|
||||
"/onboarding",
|
||||
require_auth(services["session_manager"])(
|
||||
partial(settings.onboarding, flows_service=services["flows_service"])
|
||||
partial(
|
||||
settings.onboarding,
|
||||
flows_service=services["flows_service"],
|
||||
session_manager=services["session_manager"]
|
||||
)
|
||||
),
|
||||
methods=["POST"],
|
||||
),
|
||||
|
|
|
|||
|
|
@ -743,9 +743,9 @@ class LangflowFileProcessor(TaskProcessor):
|
|||
|
||||
if metadata_tweaks:
|
||||
# Initialize the OpenSearch component tweaks if not already present
|
||||
if "OpenSearchHybrid-Ve6bS" not in final_tweaks:
|
||||
final_tweaks["OpenSearchHybrid-Ve6bS"] = {}
|
||||
final_tweaks["OpenSearchHybrid-Ve6bS"]["docs_metadata"] = metadata_tweaks
|
||||
if "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4" not in final_tweaks:
|
||||
final_tweaks["OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4"] = {}
|
||||
final_tweaks["OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4"]["docs_metadata"] = metadata_tweaks
|
||||
|
||||
# Process file using langflow service
|
||||
result = await self.langflow_file_service.upload_and_ingest_file(
|
||||
|
|
|
|||
|
|
@ -308,6 +308,16 @@ class AuthService:
|
|||
global_vars["OWNER_NAME"] = str(f"\"{owner_name}\"")
|
||||
if user_info.get("email"):
|
||||
global_vars["OWNER_EMAIL"] = user_info.get("email")
|
||||
|
||||
# Add provider credentials to MCP servers using utility function
|
||||
from config.settings import get_openrag_config
|
||||
from utils.langflow_headers import build_mcp_global_vars_from_config
|
||||
|
||||
config = get_openrag_config()
|
||||
provider_vars = build_mcp_global_vars_from_config(config)
|
||||
|
||||
# Merge provider credentials with user info
|
||||
global_vars.update(provider_vars)
|
||||
|
||||
# Run in background to avoid delaying login flow
|
||||
task = asyncio.create_task(
|
||||
|
|
|
|||
|
|
@ -60,11 +60,22 @@ class ChatService:
|
|||
"LANGFLOW_URL and LANGFLOW_CHAT_FLOW_ID environment variables are required"
|
||||
)
|
||||
|
||||
# Prepare extra headers for JWT authentication
|
||||
# Prepare extra headers for JWT authentication and embedding model
|
||||
extra_headers = {}
|
||||
if jwt_token:
|
||||
extra_headers["X-LANGFLOW-GLOBAL-VAR-JWT"] = jwt_token
|
||||
|
||||
# Pass the selected embedding model as a global variable
|
||||
from config.settings import get_openrag_config
|
||||
from utils.langflow_headers import add_provider_credentials_to_headers
|
||||
|
||||
config = get_openrag_config()
|
||||
embedding_model = config.knowledge.embedding_model
|
||||
extra_headers["X-LANGFLOW-GLOBAL-VAR-SELECTED_EMBEDDING_MODEL"] = embedding_model
|
||||
|
||||
# Add provider credentials to headers
|
||||
add_provider_credentials_to_headers(extra_headers, config)
|
||||
logger.debug(f"[LF] Extra headers {extra_headers}")
|
||||
# Get context variables for filters, limit, and threshold
|
||||
from auth_context import (
|
||||
get_score_threshold,
|
||||
|
|
@ -169,11 +180,22 @@ class ChatService:
|
|||
"LANGFLOW_URL and NUDGES_FLOW_ID environment variables are required"
|
||||
)
|
||||
|
||||
# Prepare extra headers for JWT authentication
|
||||
# Prepare extra headers for JWT authentication and embedding model
|
||||
extra_headers = {}
|
||||
if jwt_token:
|
||||
extra_headers["X-LANGFLOW-GLOBAL-VAR-JWT"] = jwt_token
|
||||
|
||||
# Pass the selected embedding model as a global variable
|
||||
from config.settings import get_openrag_config
|
||||
from utils.langflow_headers import add_provider_credentials_to_headers
|
||||
|
||||
config = get_openrag_config()
|
||||
embedding_model = config.knowledge.embedding_model
|
||||
extra_headers["X-LANGFLOW-GLOBAL-VAR-SELECTED_EMBEDDING_MODEL"] = embedding_model
|
||||
|
||||
# Add provider credentials to headers
|
||||
add_provider_credentials_to_headers(extra_headers, config)
|
||||
|
||||
# Build the complete filter expression like the chat service does
|
||||
filter_expression = {}
|
||||
has_user_filters = False
|
||||
|
|
@ -287,10 +309,22 @@ class ChatService:
|
|||
document_prompt = f"I'm uploading a document called '{filename}'. Here is its content:\n\n{document_content}\n\nPlease confirm you've received this document and are ready to answer questions about it."
|
||||
|
||||
if endpoint == "langflow":
|
||||
# Prepare extra headers for JWT authentication
|
||||
# Prepare extra headers for JWT authentication and embedding model
|
||||
extra_headers = {}
|
||||
if jwt_token:
|
||||
extra_headers["X-LANGFLOW-GLOBAL-VAR-JWT"] = jwt_token
|
||||
|
||||
# Pass the selected embedding model as a global variable
|
||||
from config.settings import get_openrag_config
|
||||
from utils.langflow_headers import add_provider_credentials_to_headers
|
||||
|
||||
config = get_openrag_config()
|
||||
embedding_model = config.knowledge.embedding_model
|
||||
extra_headers["X-LANGFLOW-GLOBAL-VAR-SELECTED_EMBEDDING_MODEL"] = embedding_model
|
||||
|
||||
# Add provider credentials to headers
|
||||
add_provider_credentials_to_headers(extra_headers, config)
|
||||
|
||||
# Ensure the Langflow client exists; try lazy init if needed
|
||||
langflow_client = await clients.ensure_langflow_client()
|
||||
if not langflow_client:
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ class LangflowFileService:
|
|||
# Pass JWT token via tweaks using the x-langflow-global-var- pattern
|
||||
if jwt_token:
|
||||
# Using the global variable pattern that Langflow expects for OpenSearch components
|
||||
tweaks["OpenSearchHybrid-Ve6bS"] = {"jwt_token": jwt_token}
|
||||
tweaks["OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4"] = {"jwt_token": jwt_token}
|
||||
logger.debug("[LF] Added JWT token to tweaks for OpenSearch components")
|
||||
else:
|
||||
logger.warning("[LF] No JWT token provided")
|
||||
|
|
@ -112,9 +112,9 @@ class LangflowFileService:
|
|||
logger.info(f"[LF] Metadata tweaks {metadata_tweaks}")
|
||||
# if metadata_tweaks:
|
||||
# # Initialize the OpenSearch component tweaks if not already present
|
||||
# if "OpenSearchHybrid-Ve6bS" not in tweaks:
|
||||
# tweaks["OpenSearchHybrid-Ve6bS"] = {}
|
||||
# tweaks["OpenSearchHybrid-Ve6bS"]["docs_metadata"] = metadata_tweaks
|
||||
# if "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4" not in tweaks:
|
||||
# tweaks["OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4"] = {}
|
||||
# tweaks["OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4"]["docs_metadata"] = metadata_tweaks
|
||||
# logger.debug(
|
||||
# "[LF] Added metadata to tweaks", metadata_count=len(metadata_tweaks)
|
||||
# )
|
||||
|
|
@ -140,6 +140,13 @@ class LangflowFileService:
|
|||
filename = str(file_tuples[0][0]) if file_tuples and len(file_tuples) > 0 else ""
|
||||
mimetype = str(file_tuples[0][2]) if file_tuples and len(file_tuples) > 0 and len(file_tuples[0]) > 2 else ""
|
||||
|
||||
# Get the current embedding model and provider credentials from config
|
||||
from config.settings import get_openrag_config
|
||||
from utils.langflow_headers import add_provider_credentials_to_headers
|
||||
|
||||
config = get_openrag_config()
|
||||
embedding_model = config.knowledge.embedding_model
|
||||
|
||||
headers={
|
||||
"X-Langflow-Global-Var-JWT": str(jwt_token),
|
||||
"X-Langflow-Global-Var-OWNER": str(owner),
|
||||
|
|
@ -149,7 +156,11 @@ class LangflowFileService:
|
|||
"X-Langflow-Global-Var-FILENAME": filename,
|
||||
"X-Langflow-Global-Var-MIMETYPE": mimetype,
|
||||
"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),
|
||||
"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),
|
||||
}
|
||||
|
||||
# Add provider credentials as global variables for ingestion
|
||||
add_provider_credentials_to_headers(headers, config)
|
||||
logger.info(f"[LF] Headers {headers}")
|
||||
logger.info(f"[LF] Payload {payload}")
|
||||
resp = await clients.langflow_request(
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
../../../docker-compose-cpu.yml
|
||||
1
src/tui/_assets/docker-compose.gpu.yml
Symbolic link
1
src/tui/_assets/docker-compose.gpu.yml
Symbolic link
|
|
@ -0,0 +1 @@
|
|||
../../../docker-compose.gpu.yml
|
||||
|
|
@ -455,7 +455,7 @@ def _copy_assets(resource_tree, destination: Path, allowed_suffixes: Optional[It
|
|||
|
||||
def copy_sample_documents(*, force: bool = False) -> None:
|
||||
"""Copy sample documents from package to current directory if they don't exist."""
|
||||
documents_dir = Path("documents")
|
||||
documents_dir = Path("openrag-documents")
|
||||
|
||||
try:
|
||||
assets_files = files("tui._assets.documents")
|
||||
|
|
@ -485,7 +485,7 @@ def copy_compose_files(*, force: bool = False) -> None:
|
|||
logger.debug(f"Could not access compose assets: {e}")
|
||||
return
|
||||
|
||||
for filename in ("docker-compose.yml", "docker-compose-cpu.yml"):
|
||||
for filename in ("docker-compose.yml", "docker-compose.gpu.yml"):
|
||||
destination = Path(filename)
|
||||
if destination.exists() and not force:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ class ServiceInfo:
|
|||
image: Optional[str] = None
|
||||
image_digest: Optional[str] = None
|
||||
created: Optional[str] = None
|
||||
error_message: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.ports is None:
|
||||
|
|
@ -56,15 +57,15 @@ class ContainerManager:
|
|||
self.platform_detector = PlatformDetector()
|
||||
self.runtime_info = self.platform_detector.detect_runtime()
|
||||
self.compose_file = compose_file or self._find_compose_file("docker-compose.yml")
|
||||
self.cpu_compose_file = self._find_compose_file("docker-compose-cpu.yml")
|
||||
self.gpu_compose_file = self._find_compose_file("docker-compose.gpu.yml")
|
||||
self.services_cache: Dict[str, ServiceInfo] = {}
|
||||
self.last_status_update = 0
|
||||
# Auto-select CPU compose if no GPU available
|
||||
# Auto-select GPU override if GPU is available
|
||||
try:
|
||||
has_gpu, _ = detect_gpu_devices()
|
||||
self.use_cpu_compose = not has_gpu
|
||||
self.use_gpu_compose = has_gpu
|
||||
except Exception:
|
||||
self.use_cpu_compose = True
|
||||
self.use_gpu_compose = False
|
||||
|
||||
# Expected services based on compose files
|
||||
self.expected_services = [
|
||||
|
|
@ -135,6 +136,96 @@ class ContainerManager:
|
|||
return self.platform_detector.get_compose_installation_instructions()
|
||||
return self.platform_detector.get_installation_instructions()
|
||||
|
||||
def _extract_ports_from_compose(self) -> Dict[str, List[int]]:
|
||||
"""Extract port mappings from compose files.
|
||||
|
||||
Returns:
|
||||
Dict mapping service name to list of host ports
|
||||
"""
|
||||
service_ports: Dict[str, List[int]] = {}
|
||||
|
||||
compose_files = [self.compose_file]
|
||||
if hasattr(self, 'cpu_compose_file') and self.cpu_compose_file and self.cpu_compose_file.exists():
|
||||
compose_files.append(self.cpu_compose_file)
|
||||
|
||||
for compose_file in compose_files:
|
||||
if not compose_file.exists():
|
||||
continue
|
||||
|
||||
try:
|
||||
import re
|
||||
content = compose_file.read_text()
|
||||
current_service = None
|
||||
in_ports_section = False
|
||||
|
||||
for line in content.splitlines():
|
||||
# Detect service names
|
||||
service_match = re.match(r'^ (\w[\w-]*):$', line)
|
||||
if service_match:
|
||||
current_service = service_match.group(1)
|
||||
in_ports_section = False
|
||||
if current_service not in service_ports:
|
||||
service_ports[current_service] = []
|
||||
continue
|
||||
|
||||
# Detect ports section
|
||||
if current_service and re.match(r'^ ports:$', line):
|
||||
in_ports_section = True
|
||||
continue
|
||||
|
||||
# Exit ports section on new top-level key
|
||||
if in_ports_section and re.match(r'^ \w+:', line):
|
||||
in_ports_section = False
|
||||
|
||||
# Extract port mappings
|
||||
if in_ports_section and current_service:
|
||||
# Match patterns like: - "3000:3000", - "9200:9200", - 7860:7860
|
||||
port_match = re.search(r'["\']?(\d+):\d+["\']?', line)
|
||||
if port_match:
|
||||
host_port = int(port_match.group(1))
|
||||
if host_port not in service_ports[current_service]:
|
||||
service_ports[current_service].append(host_port)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error parsing {compose_file} for ports: {e}")
|
||||
continue
|
||||
|
||||
return service_ports
|
||||
|
||||
async def check_ports_available(self) -> tuple[bool, List[tuple[str, int, str]]]:
|
||||
"""Check if required ports are available.
|
||||
|
||||
Returns:
|
||||
Tuple of (all_available, conflicts) where conflicts is a list of
|
||||
(service_name, port, error_message) tuples
|
||||
"""
|
||||
import socket
|
||||
|
||||
service_ports = self._extract_ports_from_compose()
|
||||
conflicts: List[tuple[str, int, str]] = []
|
||||
|
||||
for service_name, ports in service_ports.items():
|
||||
for port in ports:
|
||||
try:
|
||||
# Try to bind to the port to check if it's available
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(0.5)
|
||||
result = sock.connect_ex(('127.0.0.1', port))
|
||||
sock.close()
|
||||
|
||||
if result == 0:
|
||||
# Port is in use
|
||||
conflicts.append((
|
||||
service_name,
|
||||
port,
|
||||
f"Port {port} is already in use"
|
||||
))
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking port {port}: {e}")
|
||||
continue
|
||||
|
||||
return (len(conflicts) == 0, conflicts)
|
||||
|
||||
async def _run_compose_command(
|
||||
self, args: List[str], cpu_mode: Optional[bool] = None
|
||||
) -> tuple[bool, str, str]:
|
||||
|
|
@ -143,9 +234,15 @@ class ContainerManager:
|
|||
return False, "", "No container runtime available"
|
||||
|
||||
if cpu_mode is None:
|
||||
cpu_mode = self.use_cpu_compose
|
||||
compose_file = self.cpu_compose_file if cpu_mode else self.compose_file
|
||||
cmd = self.runtime_info.compose_command + ["-f", str(compose_file)] + args
|
||||
use_gpu = self.use_gpu_compose
|
||||
else:
|
||||
use_gpu = not cpu_mode
|
||||
|
||||
# Build compose command with override pattern
|
||||
cmd = self.runtime_info.compose_command + ["-f", str(self.compose_file)]
|
||||
if use_gpu and self.gpu_compose_file.exists():
|
||||
cmd.extend(["-f", str(self.gpu_compose_file)])
|
||||
cmd.extend(args)
|
||||
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
|
|
@ -179,9 +276,15 @@ class ContainerManager:
|
|||
return
|
||||
|
||||
if cpu_mode is None:
|
||||
cpu_mode = self.use_cpu_compose
|
||||
compose_file = self.cpu_compose_file if cpu_mode else self.compose_file
|
||||
cmd = self.runtime_info.compose_command + ["-f", str(compose_file)] + args
|
||||
use_gpu = self.use_gpu_compose
|
||||
else:
|
||||
use_gpu = not cpu_mode
|
||||
|
||||
# Build compose command with override pattern
|
||||
cmd = self.runtime_info.compose_command + ["-f", str(self.compose_file)]
|
||||
if use_gpu and self.gpu_compose_file.exists():
|
||||
cmd.extend(["-f", str(self.gpu_compose_file)])
|
||||
cmd.extend(args)
|
||||
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
|
|
@ -242,9 +345,15 @@ class ContainerManager:
|
|||
return
|
||||
|
||||
if cpu_mode is None:
|
||||
cpu_mode = self.use_cpu_compose
|
||||
compose_file = self.cpu_compose_file if cpu_mode else self.compose_file
|
||||
cmd = self.runtime_info.compose_command + ["-f", str(compose_file)] + args
|
||||
use_gpu = self.use_gpu_compose
|
||||
else:
|
||||
use_gpu = not cpu_mode
|
||||
|
||||
# Build compose command with override pattern
|
||||
cmd = self.runtime_info.compose_command + ["-f", str(self.compose_file)]
|
||||
if use_gpu and self.gpu_compose_file.exists():
|
||||
cmd.extend(["-f", str(self.gpu_compose_file)])
|
||||
cmd.extend(args)
|
||||
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
|
|
@ -551,44 +660,61 @@ class ContainerManager:
|
|||
"""Get resolved image names from compose files using docker/podman compose, with robust fallbacks."""
|
||||
images: set[str] = set()
|
||||
|
||||
compose_files = [self.compose_file, self.cpu_compose_file]
|
||||
for compose_file in compose_files:
|
||||
# Try both GPU and CPU modes to get all images
|
||||
for use_gpu in [True, False]:
|
||||
try:
|
||||
if not compose_file or not compose_file.exists():
|
||||
continue
|
||||
# Build compose command with override pattern
|
||||
cmd = self.runtime_info.compose_command + ["-f", str(self.compose_file)]
|
||||
if use_gpu and self.gpu_compose_file.exists():
|
||||
cmd.extend(["-f", str(self.gpu_compose_file)])
|
||||
cmd.extend(["config", "--format", "json"])
|
||||
|
||||
cpu_mode = (compose_file == self.cpu_compose_file)
|
||||
|
||||
# Try JSON format first
|
||||
success, stdout, _ = await self._run_compose_command(
|
||||
["config", "--format", "json"],
|
||||
cpu_mode=cpu_mode
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=Path.cwd(),
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
stdout_text = stdout.decode() if stdout else ""
|
||||
|
||||
if success and stdout.strip():
|
||||
from_cfg = self._extract_images_from_compose_config(stdout, tried_json=True)
|
||||
if process.returncode == 0 and stdout_text.strip():
|
||||
from_cfg = self._extract_images_from_compose_config(stdout_text, tried_json=True)
|
||||
if from_cfg:
|
||||
images.update(from_cfg)
|
||||
continue # this compose file succeeded; move to next file
|
||||
continue
|
||||
|
||||
# Fallback to YAML output (for older compose versions)
|
||||
success, stdout, _ = await self._run_compose_command(
|
||||
["config"],
|
||||
cpu_mode=cpu_mode
|
||||
)
|
||||
cmd = self.runtime_info.compose_command + ["-f", str(self.compose_file)]
|
||||
if use_gpu and self.gpu_compose_file.exists():
|
||||
cmd.extend(["-f", str(self.gpu_compose_file)])
|
||||
cmd.append("config")
|
||||
|
||||
if success and stdout.strip():
|
||||
from_cfg = self._extract_images_from_compose_config(stdout, tried_json=False)
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=Path.cwd(),
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
stdout_text = stdout.decode() if stdout else ""
|
||||
|
||||
if process.returncode == 0 and stdout_text.strip():
|
||||
from_cfg = self._extract_images_from_compose_config(stdout_text, tried_json=False)
|
||||
if from_cfg:
|
||||
images.update(from_cfg)
|
||||
continue
|
||||
|
||||
except Exception:
|
||||
# Keep behavior resilient—just continue to next file
|
||||
# Keep behavior resilient—just continue to next mode
|
||||
continue
|
||||
|
||||
# Fallback: manual parsing if compose config didn't work
|
||||
if not images:
|
||||
compose_files = [self.compose_file]
|
||||
if self.gpu_compose_file.exists():
|
||||
compose_files.append(self.gpu_compose_file)
|
||||
|
||||
for compose in compose_files:
|
||||
try:
|
||||
if not compose.exists():
|
||||
|
|
@ -638,8 +764,11 @@ class ContainerManager:
|
|||
yield False, "No container runtime available"
|
||||
return
|
||||
|
||||
# Diagnostic info about compose files
|
||||
compose_file = self.cpu_compose_file if (cpu_mode if cpu_mode is not None else self.use_cpu_compose) else self.compose_file
|
||||
# Determine GPU mode
|
||||
if cpu_mode is None:
|
||||
use_gpu = self.use_gpu_compose
|
||||
else:
|
||||
use_gpu = not cpu_mode
|
||||
|
||||
# Show the search process for debugging
|
||||
if hasattr(self, '_compose_search_log'):
|
||||
|
|
@ -650,9 +779,23 @@ class ContainerManager:
|
|||
# Show runtime detection info
|
||||
runtime_cmd_str = " ".join(self.runtime_info.compose_command)
|
||||
yield False, f"Using compose command: {runtime_cmd_str}", False
|
||||
yield False, f"Final compose file: {compose_file.absolute()}", False
|
||||
if not compose_file.exists():
|
||||
yield False, f"ERROR: Compose file not found at {compose_file.absolute()}", False
|
||||
compose_files_str = str(self.compose_file.absolute())
|
||||
if use_gpu and self.gpu_compose_file.exists():
|
||||
compose_files_str += f" + {self.gpu_compose_file.absolute()}"
|
||||
yield False, f"Compose files: {compose_files_str}", False
|
||||
if not self.compose_file.exists():
|
||||
yield False, f"ERROR: Base compose file not found at {self.compose_file.absolute()}", False
|
||||
return
|
||||
|
||||
# Check for port conflicts before starting
|
||||
yield False, "Checking port availability...", False
|
||||
ports_available, conflicts = await self.check_ports_available()
|
||||
if not ports_available:
|
||||
yield False, "ERROR: Port conflicts detected:", False
|
||||
for service_name, port, error_msg in conflicts:
|
||||
yield False, f" - {service_name}: {error_msg}", False
|
||||
yield False, "Please stop the conflicting services and try again.", False
|
||||
yield False, "Services not started due to port conflicts.", False
|
||||
return
|
||||
|
||||
yield False, "Starting OpenRAG services...", False
|
||||
|
|
@ -677,13 +820,37 @@ class ContainerManager:
|
|||
|
||||
yield False, "Creating and starting containers...", False
|
||||
up_success = {"value": True}
|
||||
error_messages = []
|
||||
|
||||
async for message, replace_last in self._stream_compose_command(["up", "-d"], up_success, cpu_mode):
|
||||
# Detect error patterns in the output
|
||||
import re
|
||||
lower_msg = message.lower()
|
||||
|
||||
# Check for common error patterns
|
||||
if any(pattern in lower_msg for pattern in [
|
||||
"port.*already.*allocated",
|
||||
"address already in use",
|
||||
"bind.*address already in use",
|
||||
"port is already allocated"
|
||||
]):
|
||||
error_messages.append("Port conflict detected")
|
||||
up_success["value"] = False
|
||||
elif "error" in lower_msg or "failed" in lower_msg:
|
||||
# Generic error detection
|
||||
if message not in error_messages:
|
||||
error_messages.append(message)
|
||||
|
||||
yield False, message, replace_last
|
||||
|
||||
if up_success["value"]:
|
||||
yield True, "Services started successfully", False
|
||||
else:
|
||||
yield False, "Failed to start services. See output above for details.", False
|
||||
if error_messages:
|
||||
yield False, "\nDetected errors:", False
|
||||
for err in error_messages[:5]: # Limit to first 5 errors
|
||||
yield False, f" - {err}", False
|
||||
|
||||
async def stop_services(self) -> AsyncIterator[tuple[bool, str]]:
|
||||
"""Stop all services and yield progress updates."""
|
||||
|
|
@ -786,16 +953,11 @@ class ContainerManager:
|
|||
yield "No container runtime available"
|
||||
return
|
||||
|
||||
compose_file = (
|
||||
self.cpu_compose_file if self.use_cpu_compose else self.compose_file
|
||||
)
|
||||
cmd = self.runtime_info.compose_command + [
|
||||
"-f",
|
||||
str(compose_file),
|
||||
"logs",
|
||||
"-f",
|
||||
service_name,
|
||||
]
|
||||
# Build compose command with override pattern
|
||||
cmd = self.runtime_info.compose_command + ["-f", str(self.compose_file)]
|
||||
if self.use_gpu_compose and self.gpu_compose_file.exists():
|
||||
cmd.extend(["-f", str(self.gpu_compose_file)])
|
||||
cmd.extend(["logs", "-f", service_name])
|
||||
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
|
|
|
|||
|
|
@ -143,6 +143,29 @@ class DoclingManager:
|
|||
self._external_process = False
|
||||
return False
|
||||
|
||||
def check_port_available(self) -> tuple[bool, Optional[str]]:
|
||||
"""Check if the native service port is available.
|
||||
|
||||
Returns:
|
||||
Tuple of (available, error_message)
|
||||
"""
|
||||
import socket
|
||||
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(0.5)
|
||||
result = sock.connect_ex(('127.0.0.1', self._port))
|
||||
sock.close()
|
||||
|
||||
if result == 0:
|
||||
# Port is in use
|
||||
return False, f"Port {self._port} is already in use"
|
||||
return True, None
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking port {self._port}: {e}")
|
||||
# If we can't check, assume it's available
|
||||
return True, None
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
"""Get current status of docling serve."""
|
||||
# Check for starting state first
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ class EnvConfig:
|
|||
nudges_flow_id: str = "ebc01d31-1976-46ce-a385-b0240327226c"
|
||||
|
||||
# Document paths (comma-separated)
|
||||
openrag_documents_paths: str = "./documents"
|
||||
openrag_documents_paths: str = "./openrag-documents"
|
||||
|
||||
# OpenSearch data path
|
||||
opensearch_data_path: str = "./opensearch-data"
|
||||
|
|
@ -454,7 +454,7 @@ class EnvManager:
|
|||
(
|
||||
"openrag_documents_paths",
|
||||
"Documents Paths",
|
||||
"./documents,/path/to/more/docs",
|
||||
"./openrag-documents,/path/to/more/docs",
|
||||
False,
|
||||
),
|
||||
]
|
||||
|
|
@ -521,7 +521,7 @@ class EnvManager:
|
|||
)
|
||||
|
||||
if not is_valid:
|
||||
return ["./documents:/app/documents:Z"] # fallback
|
||||
return ["./openrag-documents:/app/documents:Z"] # fallback
|
||||
|
||||
volume_mounts = []
|
||||
for i, path in enumerate(validated_paths):
|
||||
|
|
|
|||
|
|
@ -523,7 +523,7 @@ class ConfigScreen(Screen):
|
|||
yield Label("Documents Paths")
|
||||
current_value = getattr(self.env_manager.config, "openrag_documents_paths", "")
|
||||
input_widget = Input(
|
||||
placeholder="./documents,/path/to/more/docs",
|
||||
placeholder="./openrag-documents,/path/to/more/docs",
|
||||
value=current_value,
|
||||
validators=[DocumentsPathValidator()],
|
||||
id="input-openrag_documents_paths",
|
||||
|
|
|
|||
|
|
@ -33,13 +33,14 @@ class MonitorScreen(Screen):
|
|||
("u", "upgrade", "Upgrade"),
|
||||
("x", "reset", "Reset"),
|
||||
("l", "logs", "View Logs"),
|
||||
("g", "toggle_mode", "Toggle GPU/CPU"),
|
||||
("j", "cursor_down", "Move Down"),
|
||||
("k", "cursor_up", "Move Up"),
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.container_manager = ContainerManager()
|
||||
self._container_manager = None # Use app's shared instance
|
||||
self.docling_manager = DoclingManager()
|
||||
self.services_table = None
|
||||
self.docling_table = None
|
||||
|
|
@ -52,6 +53,13 @@ class MonitorScreen(Screen):
|
|||
# Track which table was last selected for mutual exclusion
|
||||
self._last_selected_table = None
|
||||
|
||||
@property
|
||||
def container_manager(self) -> ContainerManager:
|
||||
"""Get the shared container manager from the app."""
|
||||
if self._container_manager is None:
|
||||
self._container_manager = self.app.container_manager
|
||||
return self._container_manager
|
||||
|
||||
def on_unmount(self) -> None:
|
||||
"""Clean up when the screen is unmounted."""
|
||||
if hasattr(self, 'docling_manager'):
|
||||
|
|
@ -69,10 +77,10 @@ class MonitorScreen(Screen):
|
|||
|
||||
def _create_services_tab(self) -> ComposeResult:
|
||||
"""Create the services monitoring tab."""
|
||||
# Current mode indicator + toggle
|
||||
# GPU/CPU mode section
|
||||
yield Static("GPU Mode", id="mode-indicator", classes="tab-header")
|
||||
yield Horizontal(
|
||||
Static("", id="mode-indicator"),
|
||||
Button("Toggle Mode", id="toggle-mode-btn"),
|
||||
Button("Switch to CPU Mode", id="toggle-mode-btn"),
|
||||
classes="button-row",
|
||||
id="mode-row",
|
||||
)
|
||||
|
|
@ -311,17 +319,46 @@ class MonitorScreen(Screen):
|
|||
"""Start services with progress updates."""
|
||||
self.operation_in_progress = True
|
||||
try:
|
||||
# Check for port conflicts before attempting to start
|
||||
ports_available, conflicts = await self.container_manager.check_ports_available()
|
||||
if not ports_available:
|
||||
# Show error notification instead of modal
|
||||
conflict_msgs = []
|
||||
for service_name, port, error_msg in conflicts[:3]: # Show first 3
|
||||
conflict_msgs.append(f"{service_name} (port {port})")
|
||||
|
||||
conflict_str = ", ".join(conflict_msgs)
|
||||
if len(conflicts) > 3:
|
||||
conflict_str += f" and {len(conflicts) - 3} more"
|
||||
|
||||
self.notify(
|
||||
f"Cannot start services: Port conflicts detected for {conflict_str}. "
|
||||
f"Please stop the conflicting services first.",
|
||||
severity="error",
|
||||
timeout=10
|
||||
)
|
||||
# Refresh to show current state
|
||||
await self._refresh_services()
|
||||
return
|
||||
|
||||
# Show command output in modal dialog
|
||||
command_generator = self.container_manager.start_services(cpu_mode)
|
||||
modal = CommandOutputModal(
|
||||
"Starting Services",
|
||||
command_generator,
|
||||
on_complete=None, # We'll refresh in on_screen_resume instead
|
||||
on_complete=self._on_start_complete, # Refresh after completion
|
||||
)
|
||||
self.app.push_screen(modal)
|
||||
except Exception as e:
|
||||
self.notify(f"Error starting services: {str(e)}", severity="error")
|
||||
await self._refresh_services()
|
||||
finally:
|
||||
self.operation_in_progress = False
|
||||
|
||||
async def _on_start_complete(self) -> None:
|
||||
"""Callback after service start completes."""
|
||||
await self._refresh_services()
|
||||
|
||||
async def _stop_services(self) -> None:
|
||||
"""Stop services with progress updates."""
|
||||
self.operation_in_progress = True
|
||||
|
|
@ -386,6 +423,19 @@ class MonitorScreen(Screen):
|
|||
"""Start docling serve."""
|
||||
self.operation_in_progress = True
|
||||
try:
|
||||
# Check for port conflicts before attempting to start
|
||||
port_available, error_msg = self.docling_manager.check_port_available()
|
||||
if not port_available:
|
||||
self.notify(
|
||||
f"Cannot start docling serve: {error_msg}. "
|
||||
f"Please stop the conflicting service first.",
|
||||
severity="error",
|
||||
timeout=10
|
||||
)
|
||||
# Refresh to show current state
|
||||
await self._refresh_services()
|
||||
return
|
||||
|
||||
# Start the service (this sets _starting = True internally at the start)
|
||||
# Create task and let it begin executing (which sets the flag)
|
||||
start_task = asyncio.create_task(self.docling_manager.start())
|
||||
|
|
@ -581,22 +631,21 @@ class MonitorScreen(Screen):
|
|||
def _update_mode_row(self) -> None:
|
||||
"""Update the mode indicator and toggle button label."""
|
||||
try:
|
||||
use_cpu = getattr(self.container_manager, "use_cpu_compose", True)
|
||||
use_gpu = getattr(self.container_manager, "use_gpu_compose", False)
|
||||
indicator = self.query_one("#mode-indicator", Static)
|
||||
mode_text = "Mode: CPU (no GPU detected)" if use_cpu else "Mode: GPU"
|
||||
indicator.update(mode_text)
|
||||
indicator.update("GPU Mode" if use_gpu else "CPU Mode")
|
||||
toggle_btn = self.query_one("#toggle-mode-btn", Button)
|
||||
toggle_btn.label = "Switch to GPU Mode" if use_cpu else "Switch to CPU Mode"
|
||||
toggle_btn.label = "Switch to CPU Mode" if use_gpu else "Switch to GPU Mode"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def action_toggle_mode(self) -> None:
|
||||
"""Toggle between CPU/GPU compose files and refresh view."""
|
||||
try:
|
||||
current = getattr(self.container_manager, "use_cpu_compose", True)
|
||||
self.container_manager.use_cpu_compose = not current
|
||||
current = getattr(self.container_manager, "use_gpu_compose", False)
|
||||
self.container_manager.use_gpu_compose = not current
|
||||
self.notify(
|
||||
"Switched to GPU compose" if not current else "Switched to CPU compose",
|
||||
"Switched to GPU mode" if not current else "Switched to CPU mode",
|
||||
severity="information",
|
||||
)
|
||||
self._update_mode_row()
|
||||
|
|
|
|||
|
|
@ -101,9 +101,22 @@ class WelcomeScreen(Screen):
|
|||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
# Check if any services are running
|
||||
running_services = [s for s in services if isinstance(s, dict) and s.get('State') == 'running']
|
||||
self.services_running = len(running_services) > 0
|
||||
# Check if services are running (exclude starting/created states)
|
||||
# State can be lowercase or mixed case, so normalize it
|
||||
running_services = []
|
||||
starting_services = []
|
||||
for s in services:
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
state = str(s.get('State', '')).lower()
|
||||
if state == 'running':
|
||||
running_services.append(s)
|
||||
elif 'starting' in state or 'created' in state:
|
||||
starting_services.append(s)
|
||||
|
||||
# Only consider services running if we have running services AND no starting services
|
||||
# This prevents showing the button when containers are still coming up
|
||||
self.services_running = len(running_services) > 0 and len(starting_services) == 0
|
||||
else:
|
||||
self.services_running = False
|
||||
except Exception:
|
||||
|
|
@ -220,7 +233,12 @@ class WelcomeScreen(Screen):
|
|||
running_services = [
|
||||
s.name for s in services.values() if s.status == ServiceStatus.RUNNING
|
||||
]
|
||||
self.services_running = len(running_services) > 0
|
||||
starting_services = [
|
||||
s.name for s in services.values() if s.status == ServiceStatus.STARTING
|
||||
]
|
||||
# Only consider services running if we have running services AND no starting services
|
||||
# This prevents showing the button when containers are still coming up
|
||||
self.services_running = len(running_services) > 0 and len(starting_services) == 0
|
||||
else:
|
||||
self.services_running = False
|
||||
|
||||
|
|
@ -385,6 +403,34 @@ class WelcomeScreen(Screen):
|
|||
|
||||
async def _start_all_services(self) -> None:
|
||||
"""Start all services: containers first, then native services."""
|
||||
# Check for port conflicts before attempting to start anything
|
||||
conflicts = []
|
||||
|
||||
# Check container ports
|
||||
if self.container_manager.is_available():
|
||||
ports_available, port_conflicts = await self.container_manager.check_ports_available()
|
||||
if not ports_available:
|
||||
for service_name, port, error_msg in port_conflicts[:3]: # Show first 3
|
||||
conflicts.append(f"{service_name} (port {port})")
|
||||
if len(port_conflicts) > 3:
|
||||
conflicts.append(f"and {len(port_conflicts) - 3} more")
|
||||
|
||||
# Check native service port
|
||||
port_available, error_msg = self.docling_manager.check_port_available()
|
||||
if not port_available:
|
||||
conflicts.append(f"docling (port {self.docling_manager._port})")
|
||||
|
||||
# If there are any conflicts, show error and return
|
||||
if conflicts:
|
||||
conflict_str = ", ".join(conflicts)
|
||||
self.notify(
|
||||
f"Cannot start services: Port conflicts detected for {conflict_str}. "
|
||||
f"Please stop the conflicting services first.",
|
||||
severity="error",
|
||||
timeout=10
|
||||
)
|
||||
return
|
||||
|
||||
# Step 1: Start container services first (to create the network)
|
||||
if self.container_manager.is_available():
|
||||
command_generator = self.container_manager.start_services()
|
||||
|
|
@ -410,6 +456,20 @@ class WelcomeScreen(Screen):
|
|||
async def _start_native_services_after_containers(self) -> None:
|
||||
"""Start native services after containers have been started."""
|
||||
if not self.docling_manager.is_running():
|
||||
# Check for port conflicts before attempting to start
|
||||
port_available, error_msg = self.docling_manager.check_port_available()
|
||||
if not port_available:
|
||||
self.notify(
|
||||
f"Cannot start native services: {error_msg}. "
|
||||
f"Please stop the conflicting service first.",
|
||||
severity="error",
|
||||
timeout=10
|
||||
)
|
||||
# Update state and return
|
||||
self.docling_running = False
|
||||
await self._refresh_welcome_content()
|
||||
return
|
||||
|
||||
self.notify("Starting native services...", severity="information")
|
||||
success, message = await self.docling_manager.start()
|
||||
if success:
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ class CommandOutputModal(ModalScreen):
|
|||
("p", "pause_waves", "Pause"),
|
||||
("f", "speed_up", "Faster"),
|
||||
("s", "speed_down", "Slower"),
|
||||
("escape", "close_modal", "Close"),
|
||||
]
|
||||
|
||||
DEFAULT_CSS = """
|
||||
|
|
@ -188,6 +189,8 @@ class CommandOutputModal(ModalScreen):
|
|||
self._output_lines: list[str] = []
|
||||
self._layer_line_map: dict[str, int] = {} # Maps layer ID to line index
|
||||
self._status_task: Optional[asyncio.Task] = None
|
||||
self._error_detected = False
|
||||
self._command_complete = False
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
"""Create the modal dialog layout."""
|
||||
|
|
@ -254,6 +257,12 @@ class CommandOutputModal(ModalScreen):
|
|||
for w in waves.wavelets:
|
||||
w.speed = max(0.1, w.speed * 0.8)
|
||||
|
||||
def action_close_modal(self) -> None:
|
||||
"""Close the modal (only if error detected or command complete)."""
|
||||
close_btn = self.query_one("#close-btn", Button)
|
||||
if not close_btn.disabled:
|
||||
self.dismiss()
|
||||
|
||||
async def _run_command(self) -> None:
|
||||
"""Run the command and update the output in real-time."""
|
||||
output = self.query_one("#command-output", TextArea)
|
||||
|
|
@ -273,8 +282,25 @@ class CommandOutputModal(ModalScreen):
|
|||
# Move cursor to end to trigger scroll
|
||||
output.move_cursor((len(self._output_lines), 0))
|
||||
|
||||
# Detect error patterns in messages
|
||||
import re
|
||||
lower_msg = message.lower() if message else ""
|
||||
if not self._error_detected and any(pattern in lower_msg for pattern in [
|
||||
"error:",
|
||||
"failed",
|
||||
"port.*already.*allocated",
|
||||
"address already in use",
|
||||
"not found",
|
||||
"permission denied"
|
||||
]):
|
||||
self._error_detected = True
|
||||
# Enable close button when error detected
|
||||
close_btn = self.query_one("#close-btn", Button)
|
||||
close_btn.disabled = False
|
||||
|
||||
# If command is complete, update UI
|
||||
if is_complete:
|
||||
self._command_complete = True
|
||||
self._update_output("Command completed successfully", False)
|
||||
output.text = "\n".join(self._output_lines)
|
||||
output.move_cursor((len(self._output_lines), 0))
|
||||
|
|
|
|||
71
src/utils/langflow_headers.py
Normal file
71
src/utils/langflow_headers.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
"""Utility functions for building Langflow request headers."""
|
||||
|
||||
from typing import Dict
|
||||
from utils.container_utils import transform_localhost_url
|
||||
|
||||
|
||||
def add_provider_credentials_to_headers(headers: Dict[str, str], config) -> None:
|
||||
"""Add provider credentials to headers as Langflow global variables.
|
||||
|
||||
Args:
|
||||
headers: Dictionary of headers to add credentials to
|
||||
config: OpenRAGConfig object containing provider configurations
|
||||
"""
|
||||
# Add OpenAI credentials
|
||||
if config.providers.openai.api_key:
|
||||
headers["X-LANGFLOW-GLOBAL-VAR-OPENAI_API_KEY"] = str(config.providers.openai.api_key)
|
||||
|
||||
# Add Anthropic credentials
|
||||
if config.providers.anthropic.api_key:
|
||||
headers["X-LANGFLOW-GLOBAL-VAR-ANTHROPIC_API_KEY"] = str(config.providers.anthropic.api_key)
|
||||
|
||||
# Add WatsonX credentials
|
||||
if config.providers.watsonx.api_key:
|
||||
headers["X-LANGFLOW-GLOBAL-VAR-WATSONX_API_KEY"] = str(config.providers.watsonx.api_key)
|
||||
|
||||
if config.providers.watsonx.project_id:
|
||||
headers["X-LANGFLOW-GLOBAL-VAR-WATSONX_PROJECT_ID"] = str(config.providers.watsonx.project_id)
|
||||
|
||||
# Add Ollama endpoint (with localhost transformation)
|
||||
if config.providers.ollama.endpoint:
|
||||
ollama_endpoint = transform_localhost_url(config.providers.ollama.endpoint)
|
||||
headers["X-LANGFLOW-GLOBAL-VAR-OLLAMA_BASE_URL"] = str(ollama_endpoint)
|
||||
|
||||
|
||||
def build_mcp_global_vars_from_config(config) -> Dict[str, str]:
|
||||
"""Build MCP global variables dictionary from OpenRAG configuration.
|
||||
|
||||
Args:
|
||||
config: OpenRAGConfig object containing provider configurations
|
||||
|
||||
Returns:
|
||||
Dictionary of global variables for MCP servers (without X-Langflow-Global-Var prefix)
|
||||
"""
|
||||
global_vars = {}
|
||||
|
||||
# Add OpenAI credentials
|
||||
if config.providers.openai.api_key:
|
||||
global_vars["OPENAI_API_KEY"] = config.providers.openai.api_key
|
||||
|
||||
# Add Anthropic credentials
|
||||
if config.providers.anthropic.api_key:
|
||||
global_vars["ANTHROPIC_API_KEY"] = config.providers.anthropic.api_key
|
||||
|
||||
# Add WatsonX credentials
|
||||
if config.providers.watsonx.api_key:
|
||||
global_vars["WATSONX_API_KEY"] = config.providers.watsonx.api_key
|
||||
|
||||
if config.providers.watsonx.project_id:
|
||||
global_vars["WATSONX_PROJECT_ID"] = config.providers.watsonx.project_id
|
||||
|
||||
# Add Ollama endpoint (with localhost transformation)
|
||||
if config.providers.ollama.endpoint:
|
||||
ollama_endpoint = transform_localhost_url(config.providers.ollama.endpoint)
|
||||
global_vars["OLLAMA_BASE_URL"] = ollama_endpoint
|
||||
|
||||
# Add selected embedding model
|
||||
if config.knowledge.embedding_model:
|
||||
global_vars["SELECTED_EMBEDDING_MODEL"] = config.knowledge.embedding_model
|
||||
|
||||
return global_vars
|
||||
|
||||
|
|
@ -29,7 +29,7 @@ async def wait_for_ready(client: httpx.AsyncClient, timeout_s: float = 30.0):
|
|||
|
||||
|
||||
def count_files_in_documents() -> int:
|
||||
base_dir = Path(os.getcwd()) / "documents"
|
||||
base_dir = Path(os.getcwd()) / "openrag-documents"
|
||||
if not base_dir.is_dir():
|
||||
return 0
|
||||
return sum(1 for _ in base_dir.rglob("*") if _.is_file() and _.name not in EXCLUDED_INGESTION_FILES)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue