diff --git a/.env.example b/.env.example index 081c9026..5d231931 100644 --- a/.env.example +++ b/.env.example @@ -40,13 +40,28 @@ GOOGLE_OAUTH_CLIENT_SECRET= MICROSOFT_GRAPH_OAUTH_CLIENT_ID= MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET= +# AWS Access Key ID and Secret Access Key with access to your S3 instance +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + # OPTIONAL: dns routable from google (etc.) to handle continous ingest (something like ngrok works). This enables continous ingestion WEBHOOK_BASE_URL= +# Model Provider API Keys OPENAI_API_KEY= +ANTHROPIC_API_KEY= +OLLAMA_ENDPOINT= +WATSONX_API_KEY= +WATSONX_ENDPOINT= +WATSONX_PROJECT_ID= -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= +# LLM Provider configuration. Providers can be "anthropic", "watsonx", "ibm" or "ollama". +LLM_PROVIDER= +LLM_MODEL= + +# Embedding provider configuration. Providers can be "watsonx", "ibm" or "ollama". +EMBEDDING_PROVIDER= +EMBEDDING_MODEL= # OPTIONAL url for openrag link to langflow in the UI LANGFLOW_PUBLIC_URL= diff --git a/.github/workflows/build-multiarch.yml b/.github/workflows/build-multiarch.yml index f9a83400..336f8df9 100644 --- a/.github/workflows/build-multiarch.yml +++ b/.github/workflows/build-multiarch.yml @@ -14,6 +14,7 @@ jobs: outputs: skip_release: ${{ steps.version.outputs.skip_release }} version: ${{ steps.version.outputs.version }} + docker_version: ${{ steps.version.outputs.docker_version }} is_prerelease: ${{ steps.version.outputs.is_prerelease }} steps: - name: Checkout @@ -26,6 +27,12 @@ jobs: echo "version=$VERSION" >> $GITHUB_OUTPUT echo "Version: $VERSION" + # Normalize version per PEP 440 for Docker tags + # e.g., "0.1.53-rc2" -> "0.1.53rc2" to match Python's importlib.metadata + DOCKER_VERSION=$(echo "$VERSION" | sed -E 's/-?(rc|alpha|beta|dev|post)/\1/g') + echo "docker_version=$DOCKER_VERSION" >> $GITHUB_OUTPUT + echo "Docker Version: $DOCKER_VERSION" + # Check if tag already exists if git rev-parse "v$VERSION" >/dev/null 2>&1; then echo "Tag v$VERSION already exists, skipping release" @@ -117,13 +124,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Extract version from pyproject.toml - id: version - run: | - VERSION=$(grep '^version = ' pyproject.toml | cut -d '"' -f 2) - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "Version: $VERSION" - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -141,7 +141,7 @@ jobs: file: ${{ matrix.file }} platforms: ${{ matrix.platform }} push: ${{ github.event_name != 'pull_request' }} - tags: ${{ matrix.tag }}:${{ steps.version.outputs.version }}-${{ matrix.arch }} + tags: ${{ matrix.tag }}:${{ needs.check-version.outputs.docker_version }}-${{ matrix.arch }} cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.arch }} @@ -153,12 +153,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Extract version from pyproject.toml - id: version - run: | - VERSION=$(grep '^version = ' pyproject.toml | cut -d '"' -f 2) - echo "version=$VERSION" >> $GITHUB_OUTPUT - - name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -167,7 +161,7 @@ jobs: - name: Create and push multi-arch manifests run: | - VERSION=${{ steps.version.outputs.version }} + VERSION=${{ needs.check-version.outputs.docker_version }} # Create versioned tags docker buildx imagetools create -t langflowai/openrag-backend:$VERSION \ @@ -224,13 +218,6 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v3 - - name: Extract version from pyproject.toml - id: version - run: | - VERSION=$(grep '^version = ' pyproject.toml | cut -d '"' -f 2) - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "Version: $VERSION" - - name: Build wheel and source distribution run: | uv build @@ -253,8 +240,8 @@ jobs: - name: Create Release uses: softprops/action-gh-release@v2 with: - tag_name: v${{ steps.version.outputs.version }} - name: Release ${{ steps.version.outputs.version }} + tag_name: v${{ needs.check-version.outputs.version }} + name: Release ${{ needs.check-version.outputs.version }} draft: false prerelease: ${{ needs.check-version.outputs.is_prerelease }} generate_release_notes: true diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index a70dd24d..544b846e 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -38,8 +38,12 @@ jobs: docker builder prune -af || true docker-compose -f docker-compose.yml down -v --remove-orphans || true + - name: Cleanup OpenSearch data (root-owned files) + run: | + docker run --rm -v $(pwd):/work alpine rm -rf /work/opensearch-data || true + - run: df -h - + - name: Checkout uses: actions/checkout@v4 diff --git a/docker-compose.yml b/docker-compose.yml index a0b1ca2b..2a73da89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -80,10 +80,11 @@ services: - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} volumes: - - ./openrag-documents:/app/openrag-documents:Z - - ./keys:/app/keys:Z - - ./flows:/app/flows:U,z - - ./config:/app/config:Z + - ${OPENRAG_DOCUMENTS_PATH:-./openrag-documents}:/app/openrag-documents:Z + - ${OPENRAG_KEYS_PATH:-./keys}:/app/keys:Z + - ${OPENRAG_FLOWS_PATH:-./flows}:/app/flows:U,z + - ${OPENRAG_CONFIG_PATH:-./config}:/app/config:Z + - ${OPENRAG_DATA_PATH:-./data}:/app/data:Z openrag-frontend: image: langflowai/openrag-frontend:${OPENRAG_VERSION:-latest} @@ -100,7 +101,7 @@ services: langflow: volumes: - - ./flows:/app/flows:U,z + - ${OPENRAG_FLOWS_PATH:-./flows}:/app/flows:U,z image: langflowai/openrag-langflow:${OPENRAG_VERSION:-latest} build: context: . diff --git a/docs/docs/_partial-docker-compose-down-and-prune.mdx b/docs/docs/_partial-docker-compose-down-and-prune.mdx new file mode 100644 index 00000000..179c0525 --- /dev/null +++ b/docs/docs/_partial-docker-compose-down-and-prune.mdx @@ -0,0 +1,9 @@ +```bash title="Docker" +docker compose down --volumes --remove-orphans --rmi local +docker system prune -f +``` + +```bash title="Podman" +podman compose down --volumes --remove-orphans --rmi local +podman system prune -f +``` \ No newline at end of file diff --git a/docs/docs/_partial-docker-compose-up.mdx b/docs/docs/_partial-docker-compose-up.mdx new file mode 100644 index 00000000..535effb3 --- /dev/null +++ b/docs/docs/_partial-docker-compose-up.mdx @@ -0,0 +1,7 @@ +```bash title="Docker" +docker compose up -d +``` + +```bash title="Podman" +podman compose up -d +``` \ No newline at end of file diff --git a/docs/docs/_partial-docker-remove-and-cleanup-steps.mdx b/docs/docs/_partial-docker-remove-and-cleanup-steps.mdx new file mode 100644 index 00000000..ccc91713 --- /dev/null +++ b/docs/docs/_partial-docker-remove-and-cleanup-steps.mdx @@ -0,0 +1,49 @@ +2. Remove all containers, including stopped containers: + + ```bash title="Docker" + docker rm --force $(docker ps -aq) + ``` + + ```bash title="Podman" + podman rm --all --force + ``` + +3. Remove all images: + + ```bash title="Docker" + docker rmi --force $(docker images -q) + ``` + + ```bash title="Podman" + podman rmi --all --force + ``` + +4. Remove all volumes: + + ```bash title="Docker" + docker volume prune --force + ``` + + ```bash title="Podman" + podman volume prune --force + ``` + +5. Remove all networks except the default network: + + ```bash title="Docker" + docker network prune --force + ``` + + ```bash title="Podman" + podman network prune --force + ``` + +6. Clean up any leftover data: + + ```bash title="Docker" + docker system prune --all --force --volumes + ``` + + ```bash title="Podman" + podman system prune --all --force --volumes + ``` \ No newline at end of file diff --git a/docs/docs/_partial-docker-stop-all.mdx b/docs/docs/_partial-docker-stop-all.mdx new file mode 100644 index 00000000..3fbf17f9 --- /dev/null +++ b/docs/docs/_partial-docker-stop-all.mdx @@ -0,0 +1,7 @@ +```bash title="Docker" +docker stop $(docker ps -q) +``` + +```bash title="Podman" +podman stop --all +``` \ No newline at end of file diff --git a/docs/docs/_partial-factory-reset-warning.mdx b/docs/docs/_partial-factory-reset-warning.mdx new file mode 100644 index 00000000..97f2ca7e --- /dev/null +++ b/docs/docs/_partial-factory-reset-warning.mdx @@ -0,0 +1,25 @@ +:::warning +This is a destructive action that does the following: + +* Destroys all OpenRAG containers, volumes, and local images with `docker compose down --volumes --remove-orphans --rmi local`. +* Prunes any additional container objects with `docker system prune -f`. +* Deletes the contents of OpenRAG's `config` and `./opensearch-data` directories. +* Deletes the `conversations.json` file. + +

Destroyed containers and deleted data are lost and cannot be recovered after running this operation. + +This operation _doesn't_ remove the `.env` file or the contents of the `./openrag-documents` directory. +::: + + \ No newline at end of file diff --git a/docs/docs/_partial-install-next-steps.mdx b/docs/docs/_partial-install-next-steps.mdx new file mode 100644 index 00000000..ad5938be --- /dev/null +++ b/docs/docs/_partial-install-next-steps.mdx @@ -0,0 +1,5 @@ +## Next steps + +* Try some of OpenRAG's core features in the [quickstart](/quickstart#chat-with-documents). +* Learn how to [manage OpenRAG services](/manage-services). +* [Upload documents](/ingestion), and then use the [**Chat**](/chat) to explore your data. \ No newline at end of file diff --git a/docs/docs/_partial-integrate-chat.mdx b/docs/docs/_partial-integrate-chat.mdx index de3d9a62..867ecd27 100644 --- a/docs/docs/_partial-integrate-chat.mdx +++ b/docs/docs/_partial-integrate-chat.mdx @@ -4,8 +4,8 @@ import TabItem from '@theme/TabItem'; 1. Open the **OpenRAG OpenSearch Agent** flow in the Langflow visual editor: From the **Chat** window, click

- Ollama model selection and external server configuration - -
- 3. Click **Complete**. - 4. To complete the onboarding tasks, click **What is OpenRAG**, and then click **Add a Document**. - 5. Continue with the [Quickstart](/quickstart). +6. Continue through the overview slides for a brief introduction to OpenRAG, or click