From 75c7f237d05a9b8e5e6f9bc78de335b63d4a73a1 Mon Sep 17 00:00:00 2001
From: April M <36110273+aimurphy@users.noreply.github.com>
Date: Tue, 25 Nov 2025 18:21:05 -0800
Subject: [PATCH] working on knowledge topics
---
docs/docs/core-components/agents.mdx | 66 +-----
docs/docs/core-components/chat.mdx | 62 +++++
docs/docs/core-components/ingestion.mdx | 114 +---------
.../core-components/knowledge-configure.mdx | 58 +++++
.../core-components/knowledge-filters.mdx | 53 +++++
docs/docs/core-components/knowledge.mdx | 212 +++++++++++++-----
docs/docs/get-started/install.mdx | 8 +-
docs/docs/get-started/quickstart.mdx | 4 +-
docs/docs/reference/configuration.mdx | 3 +-
docs/sidebars.js | 20 +-
10 files changed, 352 insertions(+), 248 deletions(-)
create mode 100644 docs/docs/core-components/chat.mdx
create mode 100644 docs/docs/core-components/knowledge-configure.mdx
create mode 100644 docs/docs/core-components/knowledge-filters.mdx
diff --git a/docs/docs/core-components/agents.mdx b/docs/docs/core-components/agents.mdx
index 23d50016..dc6ce5d2 100644
--- a/docs/docs/core-components/agents.mdx
+++ b/docs/docs/core-components/agents.mdx
@@ -1,5 +1,5 @@
---
-title: Langflow in OpenRAG
+title: Use Langflow in OpenRAG
slug: /agents
---
@@ -10,62 +10,14 @@ import TabItem from '@theme/TabItem';
OpenRAG includes a built-in [Langflow](https://docs.langflow.org/) instance for creating and managing application workflows called [_flows_](https://docs.langflow.org/concepts-overview).
In a flow, the individual workflow steps are represented by [_components_](https://docs.langflow.org/concepts-components) that are connected together to form a complete process.
-OpenRAG includes several built-in flows:
+OpenRAG includes several built-in flows that you can customize.
+You can also create your own flows using OpenRAG's embedded Langflow visual editor.
-* The [**OpenRAG OpenSearch Agent** flow](/agents#flow) powers the **Chat** feature in OpenRAG.
-* The [**OpenSearch Ingestion** and **OpenSearch URL Ingestion** flows](/ingestion#knowledge-ingestion-flows) process documents and web content for storage in your OpenSearch knowledge bases.
+## Built-in flows
-You can customize the built-in flows or create your own flows using OpenRAG's embedded Langflow visual editor.
-
-## About the OpenRAG Chat flow (OpenRAG OpenSearch Agent flow) {#flow}
-
-When you **Chat** with your knowledge in OpenRAG, the **OpenRAG OpenSearch Agent** flow runs in the background.
-
-If you [inspect the flow in Langflow](#inspect-and-modify-flows), you'll see that it is comprised of eight components that work together to ingest chat messages, retrieve relevant information from your knowledge base, and then generate responses.
-
-
-
-* The [**Agent** component](https://docs.langflow.org/agents) orchestrates the entire flow by deciding when to search the knowledge base, how to formulate search queries, and how to combine retrieved information with the user's question to generate a comprehensive response.
-The **Agent** behaves according to the prompt in the **Agent Instructions** field.
-
- The Agent component is the star of this flow because it powers decision making, tool calling, and an LLM-driven conversational experience.
-
-
- How do agents work?
-
- Agents extend Large Language Models (LLMs) by integrating tools, which are functions that provide additional context and enable autonomous task execution. These integrations make agents more specialized and powerful than standalone LLMs.
-
- Whereas an LLM might generate acceptable, inert responses to general queries and tasks, an agent can leverage the integrated context and tools to provide more relevant responses and even take action. For example, you might create an agent that can access your company's documentation, repositories, and other resources to help your team with tasks that require knowledge of your specific products, customers, and code.
-
- Agents use LLMs as a reasoning engine to process input, determine which actions to take to address the query, and then generate a response. The response could be a typical text-based LLM response, or it could involve an action, like editing a file, running a script, or calling an external API.
-
- In an agentic context, tools are functions that the agent can run to perform tasks or access external resources. A function is wrapped as a Tool object with a common interface that the agent understands. Agents become aware of tools through tool registration, which is when the agent is provided a list of available tools typically at agent initialization. The Tool object's description tells the agent what the tool can do so that it can decide whether the tool is appropriate for a given request.
-
-
-
-* The [**Chat Input** component](https://docs.langflow.org/components-io) is connected to the Agent component's Input port. This allows to flow to be triggered by an incoming prompt from a user or application.
-
-* The [**OpenSearch** component](https://docs.langflow.org/bundles-elastic#opensearch) is connected to the Agent component's Tools port. The agent might not use this database for every request; the agent only uses this connection if it decides the knowledge can help respond to the prompt.
-
-* The [**Language Model** component](https://docs.langflow.org/components-models) is connected to the Agent component's Language Model port. The agent uses the connected LLM to reason through the request sent through Chat Input.
-
-* The [**Embedding Model** component](https://docs.langflow.org/components-embedding-models) is connected to the OpenSearch component's Embedding port. This component converts text queries into vector representations that are compared with document embeddings stored in OpenSearch for semantic similarity matching. This gives your Agent's queries context.
-
-* The [**Text Input** component](https://docs.langflow.org/components-io) is populated with the global variable `OPENRAG-QUERY-FILTER`.
-This filter is the [Knowledge filter](/knowledge#create-knowledge-filters), and filters which knowledge sources to search through.
-
-* The **Agent** component's Output port is connected to the [**Chat Output** component](https://docs.langflow.org/components-io), which returns the final response to the user or application.
-
-* An [**MCP Tools** component](https://docs.langflow.org/mcp-client) is connected to the Agent's **Tools** port. This component calls the [**OpenSearch URL Ingestion** flow](/ingestion#url-flow), which Langflow uses as a [Model Context Protocol (MCP) server](https://docs.langflow.org/mcp-server) to fetch content from URLs and store in OpenSearch.
-
-### Nudges
-
-When you use the OpenRAG **Chat**, the **OpenRAG OpenSearch Nudges** flow runs in the background to pull additional context from your knowledge base and chat history.
-
-Nudges appear as prompts in the chat.
-Click a nudge to accept it and provide the nudge's context to the OpenRAG **Chat** agent (the **OpenRAG OpenSearch Agent** flow).
-
-Like OpenRAG's other built-in flows, you can [inspect the flow in Langflow](#inspect-and-modify-flows), and you can customize it if you want to change the nudge behavior.
+* The [**OpenRAG OpenSearch Agent** flow](/chat#flow) powers the **Chat** feature in OpenRAG.
+* The [**OpenSearch Ingestion** and **OpenSearch URL Ingestion** flows](/knowledge#knowledge-ingestion-flows) process documents and web content for storage in your OpenSearch knowledge base.
+* The [**OpenRAG OpenSearch Nudges** flow](/chat#nudges) provides optional contextual suggestions in the OpenRAG **Chat**.
## Inspect and modify flows {#inspect-and-modify-flows}
@@ -99,12 +51,12 @@ For example, to view and edit the built-in **Chat** flow (the **OpenRAG OpenSear
If you modify the built-in **Chat** flow, make sure you click in the **Conversations** tab to start a new conversation. This ensures that the chat doesn't persist any context from the previous conversation with the original model.
:::
-## Revert a built-in flow to the default state
+## Revert a built-in flow to its default state
After you edit a built-in flow, you can click **Restore flow** on the **Settings** page to revert the flow to its original state when you first installed OpenRAG.
This is a destructive action that discards all customizations to the flow.
-## Additional Langflow functionality
+## Custom flows and additional Langflow functionality
In addition to OpenRAG's built-in flows, all Langflow features are available through OpenRAG, including popular extensibility features such as the following:
diff --git a/docs/docs/core-components/chat.mdx b/docs/docs/core-components/chat.mdx
new file mode 100644
index 00000000..31827aa1
--- /dev/null
+++ b/docs/docs/core-components/chat.mdx
@@ -0,0 +1,62 @@
+---
+title: Chat in OpenRAG
+slug: /chat
+---
+
+import Icon from "@site/src/components/icon/icon";
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+After you [upload documents to your knowledge base](/knowledge), you can use the OpenRAG **Chat** feature to interact with your knowledge through natural language queries.
+
+
+
+## OpenRAG OpenSearch Agent flow {#flow}
+
+When you use the OpenRAG **Chat**, the **OpenRAG OpenSearch Agent** [flow](/agents) runs in the background to retrieve relevant information from your knowledge base and generate a response.
+
+If you [inspect the flow in Langflow](/agents#inspect-and-modify-flows), you'll see that it is comprised of eight components that work together to ingest chat messages, retrieve relevant information from your knowledge base, and then generate responses.
+
+
+
+* The [**Agent** component](https://docs.langflow.org/agents) orchestrates the entire flow by deciding when to search the knowledge base, how to formulate search queries, and how to combine retrieved information with the user's question to generate a comprehensive response.
+The **Agent** behaves according to the prompt in the **Agent Instructions** field.
+
+ The Agent component is the star of this flow because it powers decision making, tool calling, and an LLM-driven conversational experience.
+
+
+ How do agents work?
+
+ Agents extend Large Language Models (LLMs) by integrating tools, which are functions that provide additional context and enable autonomous task execution. These integrations make agents more specialized and powerful than standalone LLMs.
+
+ Whereas an LLM might generate acceptable, inert responses to general queries and tasks, an agent can leverage the integrated context and tools to provide more relevant responses and even take action. For example, you might create an agent that can access your company's documentation, repositories, and other resources to help your team with tasks that require knowledge of your specific products, customers, and code.
+
+ Agents use LLMs as a reasoning engine to process input, determine which actions to take to address the query, and then generate a response. The response could be a typical text-based LLM response, or it could involve an action, like editing a file, running a script, or calling an external API.
+
+ In an agentic context, tools are functions that the agent can run to perform tasks or access external resources. A function is wrapped as a Tool object with a common interface that the agent understands. Agents become aware of tools through tool registration, which is when the agent is provided a list of available tools typically at agent initialization. The Tool object's description tells the agent what the tool can do so that it can decide whether the tool is appropriate for a given request.
+
+
+
+* The [**Chat Input** component](https://docs.langflow.org/components-io) is connected to the Agent component's Input port. This allows to flow to be triggered by an incoming prompt from a user or application.
+
+* The [**OpenSearch** component](https://docs.langflow.org/bundles-elastic#opensearch) is connected to the Agent component's Tools port. The agent might not use this database for every request; the agent only uses this connection if it decides the knowledge can help respond to the prompt.
+
+* The [**Language Model** component](https://docs.langflow.org/components-models) is connected to the Agent component's Language Model port. The agent uses the connected LLM to reason through the request sent through Chat Input.
+
+* The [**Embedding Model** component](https://docs.langflow.org/components-embedding-models) is connected to the OpenSearch component's Embedding port. This component converts text queries into vector representations that are compared with document embeddings stored in OpenSearch for semantic similarity matching. This gives your Agent's queries context.
+
+* The [**Text Input** component](https://docs.langflow.org/components-io) is populated with the global variable `OPENRAG-QUERY-FILTER`.
+This filter is a [knowledge filter](/knowledge-filters) that limits the documents the Agent can access in the knowledge base if a global or chat filter is set.
+
+* The **Agent** component's Output port is connected to the [**Chat Output** component](https://docs.langflow.org/components-io), which returns the final response to the user or application.
+
+* An [**MCP Tools** component](https://docs.langflow.org/mcp-client) is connected to the Agent's **Tools** port. This component calls the [**OpenSearch URL Ingestion** flow](/knowledge#url-flow), which Langflow uses as a [Model Context Protocol (MCP) server](https://docs.langflow.org/mcp-server) to fetch content from URLs and store in OpenSearch.
+
+## Nudges {#nudges}
+
+When you use the OpenRAG **Chat**, the **OpenRAG OpenSearch Nudges** flow runs in the background to pull additional context from your knowledge base and chat history.
+
+Nudges appear as prompts in the chat.
+Click a nudge to accept it and provide the nudge's context to the OpenRAG **Chat** agent (the **OpenRAG OpenSearch Agent** flow).
+
+Like OpenRAG's other built-in flows, you can [inspect the flow in Langflow](#inspect-and-modify-flows), and you can customize it if you want to change the nudge behavior.
\ No newline at end of file
diff --git a/docs/docs/core-components/ingestion.mdx b/docs/docs/core-components/ingestion.mdx
index ef122138..b4378bed 100644
--- a/docs/docs/core-components/ingestion.mdx
+++ b/docs/docs/core-components/ingestion.mdx
@@ -7,116 +7,4 @@ import Icon from "@site/src/components/icon/icon";
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-OpenRAG uses [Docling](https://docling-project.github.io/docling/) for document ingestion.
-More specifically, OpenRAG uses [Docling Serve](https://github.com/docling-project/docling-serve), which starts a `docling serve` process on your local machine and runs Docling ingestion through an API service.
-
-Docling ingests documents from your local machine or OAuth connectors, splits them into chunks, and stores them as separate, structured documents in the OpenSearch `documents` index.
-
-OpenRAG chose Docling for its support for a wide variety of file formats, high performance, and advanced understanding of tables and images.
-
-To modify OpenRAG's ingestion settings, including the Docling settings and ingestion flows, **Settings**.
-
-## Knowledge ingestion settings
-
-These settings configure the Docling ingestion parameters.
-
-OpenRAG will warn you if `docling serve` is not running.
-To start or stop `docling serve` or any other native services, in the TUI main menu, click **Start Native Services** or **Stop Native Services**.
-
-**Embedding model** determines which AI model is used to create vector embeddings. The default is the OpenAI `text-embedding-3-small` model.
-
-**Chunk size** determines how large each text chunk is in number of characters.
-Larger chunks yield more context per chunk, but can include irrelevant information. Smaller chunks yield more precise semantic search, but can lack context.
-The default value of `1000` characters provides a good starting point that balances these considerations.
-
-**Chunk overlap** controls the number of characters that overlap over chunk boundaries.
-Use larger overlap values for documents where context is most important, and use smaller overlap values for simpler documents, or when optimization is most important.
-The default value of 200 characters of overlap with a chunk size of 1000 (20% overlap) is suitable for general use cases. Decrease the overlap to 10% for a more efficient pipeline, or increase to 40% for more complex documents.
-
-**Table Structure** enables Docling's [`DocumentConverter`](https://docling-project.github.io/docling/reference/document_converter/) tool for parsing tables. Instead of treating tables as plain text, tables are output as structured table data with preserved relationships and metadata. **Table Structure** is enabled by default.
-
-**OCR** enables or disabled OCR processing when extracting text from images and scanned documents.
-OCR is disabled by default. This setting is best suited for processing text-based documents as quickly as possible with Docling's [`DocumentConverter`](https://docling-project.github.io/docling/reference/document_converter/). Images are ignored and not processed.
-
-Enable OCR when you are processing documents containing images with text that requires extraction, or for scanned documents. Enabling OCR can slow ingestion performance.
-
-If OpenRAG detects that the local machine is running on macOS, OpenRAG uses the [ocrmac](https://www.piwheels.org/project/ocrmac/) OCR engine. Other platforms use [easyocr](https://www.jaided.ai/easyocr/).
-
-**Picture descriptions** adds image descriptions generated by the [SmolVLM-256M-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct) model to OCR processing. Enabling picture descriptions can slow ingestion performance.
-
-## Knowledge ingestion flows {#knowledge-ingestion-flows}
-
-[Flows](https://docs.langflow.org/concepts-overview) in Langflow are functional representations of application workflows, with multiple [component](https://docs.langflow.org/concepts-components) nodes connected as single steps in a workflow.
-
-### OpenSearch Ingestion flow
-
-The **OpenSearch Ingestion** flow is the default knowledge ingestion flow in OpenRAG. When you **Add Knowledge** in OpenRAG, the **OpenSearch Ingestion** flow runs in the background. The flow ingests documents using Docling Serve to import and process documents.
-
-If you [inspect the flow in Langflow](/agents#inspect-and-modify-flows), you'll see that it is comprised of ten components that work together to process and store documents in your knowledge base:
-
-* The [**Docling Serve** component](https://docs.langflow.org/bundles-docling) processes input documents by connecting to your instance of Docling Serve.
-* The [**Export DoclingDocument** component](https://docs.langflow.org/components-docling) exports the processed DoclingDocument to markdown format with image export mode set to placeholder. This conversion makes the structured document data into a standardized format for further processing.
-* Three [**DataFrame Operations** components](https://docs.langflow.org/components-processing#dataframe-operations) sequentially add metadata columns to the document data of `filename`, `file_size`, and `mimetype`.
-* The [**Split Text** component](https://docs.langflow.org/components-processing#split-text) splits the processed text into chunks with a chunk size of 1000 characters and an overlap of 200 characters.
-* Four **Secret Input** components provide secure access to configuration variables: `CONNECTOR_TYPE`, `OWNER`, `OWNER_EMAIL`, and `OWNER_NAME`. These are runtime variables populated from OAuth login.
-* The **Create Data** component combines the secret inputs into a structured data object that will be associated with the document embeddings.
-* The [**Embedding Model** component](https://docs.langflow.org/components-embedding-models) generates vector embeddings using OpenAI's `text-embedding-3-small` model. The embedding model is selected at [Application onboarding] and cannot be changed.
-* The [**OpenSearch** component](https://docs.langflow.org/bundles-elastic#opensearch) stores the processed documents and their embeddings in the `documents` index at `https://opensearch:9200`. By default, the component is authenticated with a JWT token, but you can also select `basic` auth mode, and enter your OpenSearch admin username and password.
-
-To customize this flow, see [Inspect and modify flows](/agents#inspect-and-modify-flows).
-
-### OpenSearch URL Ingestion flow {#url-flow}
-
-The **OpenSearch URL Ingestion** flow is used to ingest web content from URLs.
-This flow isn't directly accessible from the OpenRAG user interface.
-Instead, this flow is called by the [**OpenRAG OpenSearch Agent** flow](/agents#flow) as a Model Context Protocol (MCP) tool.
-The agent can call this component to fetch web content from a given URL, and then ingest that content into your OpenSearch knowledge base.
-
-For more information about MCP in Langflow, see the Langflow documentation on [MCP clients](https://docs.langflow.org/mcp-client) and [MCP servers](https://docs.langflow.org/mcp-tutorial).
-
-## Use OpenRAG default ingestion instead of Docling serve
-
-If you want to use OpenRAG's built-in pipeline instead of Docling serve, set `DISABLE_INGEST_WITH_LANGFLOW=true` in [Environment variables](/reference/configuration#document-processing).
-
-The built-in pipeline still uses the Docling processor, but uses it directly without the Docling Serve API.
-
-For more information, see [`processors.py` in the OpenRAG repository](https://github.com/langflow-ai/openrag/blob/main/src/models/processors.py#L58).
-
-## Performance expectations
-
-On a local VM with 7 vCPUs and 8 GiB RAM, OpenRAG ingested approximately 5.03 GB across 1,083 files in about 42 minutes.
-This equates to approximately 2.4 documents per second.
-
-You can generally expect equal or better performance on developer laptops and significantly faster on servers.
-Throughput scales with CPU cores, memory, storage speed, and configuration choices such as embedding model, chunk size and overlap, and concurrency.
-
-This test returned 12 errors (approximately 1.1%).
-All errors were file-specific, and they didn't stop the pipeline.
-
-Ingestion dataset:
-
-* Total files: 1,083 items mounted
-* Total size on disk: 5,026,474,862 bytes (approximately 5.03 GB)
-
-Hardware specifications:
-
-* Machine: Apple M4 Pro
-* Podman VM:
- * Name: `podman-machine-default`
- * Type: `applehv`
- * vCPUs: 7
- * Memory: 8 GiB
- * Disk size: 100 GiB
-
-Test results:
-
-```text
-2025-09-24T22:40:45.542190Z /app/src/main.py:231 Ingesting default documents when ready disable_langflow_ingest=False
-2025-09-24T22:40:45.546385Z /app/src/main.py:270 Using Langflow ingestion pipeline for default documents file_count=1082
-...
-2025-09-24T23:19:44.866365Z /app/src/main.py:351 Langflow ingestion completed success_count=1070 error_count=12 total_files=1082
-```
-
-Elapsed time: ~42 minutes 15 seconds (2,535 seconds)
-
-Throughput: ~2.4 documents/second
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/docs/core-components/knowledge-configure.mdx b/docs/docs/core-components/knowledge-configure.mdx
new file mode 100644
index 00000000..f3829d42
--- /dev/null
+++ b/docs/docs/core-components/knowledge-configure.mdx
@@ -0,0 +1,58 @@
+---
+title: Configure OpenSearch in OpenRAG
+slug: /knowledge-configure
+---
+
+import Icon from "@site/src/components/icon/icon";
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+OpenRAG includes a built-in [OpenSearch](https://docs.opensearch.org/latest/) instance that serves as the underlying datastore for your knowledge.
+This specialized database is used to store and retrieve your documents and the associated vector data (embeddings).
+
+The [OpenRAG **Chat**](/chat) runs [similarity searches](https://www.ibm.com/think/topics/vector-search) against your OpenSearch database to retrieve relevant information and generate context-aware responses.
+
+Additionally, OpenSearch provides powerful hybrid search capabilities with enterprise-grade security and multi-tenancy support.
+
+## OpenSearch authentication and document access {#auth}
+
+When you [install OpenRAG](/install), you can choose between two setup modes: **Basic Setup** and **Advanced Setup**.
+The mode you choose determines how OpenRAG authenticates with OpenSearch and controls access to documents:
+
+* **Basic Setup (no-auth mode)**: If you choose **Basic Setup**, then OpenRAG is installed in no-auth mode.
+This mode uses one, anonymous JWT token for OpenSearch authentication.
+There is no differentiation between users.
+All users that access your OpenRAG instance can access all documents uploaded to your OpenSearch `documents` index.
+
+* **Advanced Setup (OAuth mode)**: If you choose **Advanced Setup**, then OpenRAG is installed in OAuth mode.
+This mode uses a unique JWT token for each OpenRAG user, and each document is tagged with user ownership. Documents are filtered by user owner.
+This means users see only the documents that they uploaded or have access to.
+
+You can enable OAuth mode after installation.
+For more information, see [Ingest files through OAuth connectors](/knowledge#oauth-ingestion).
+
+## Set the embedding model and dimensions {#set-the-embedding-model-and-dimensions}
+
+When you [install OpenRAG](/install), you select an embedding model during **Application Onboarding**.
+OpenRAG automatically detects and configures the appropriate vector dimensions for your selected embedding model, ensuring optimal search performance and compatibility.
+
+In the OpenRAG repository, you can find the complete list of supported models in [`models_service.py`](https://github.com/langflow-ai/openrag/blob/main/src/services/models_service.py) and the corresponding vector dimensions in [`settings.py`](https://github.com/langflow-ai/openrag/blob/main/src/config/settings.py).
+
+The default embedding dimension is `1536` and the default model is `text-embedding-3-small`.
+
+You can use any supported or unsupported embedding model by specifying the model in your OpenRAG configuration during installation.
+
+If you use an unsupported embedding model that doesn't have defined dimensions in `settings.py`, then OpenRAG falls back to the default dimensions (1536) and logs a warning. OpenRAG's OpenSearch instance and flows continue to work, but [similarity search](https://www.ibm.com/think/topics/vector-search) quality can be affected if the actual model dimensions aren't 1536.
+
+The embedding model setting is immutable.
+To change the embedding model, you must [reinstall OpenRAG](/install#reinstall).
+
+## Set ingestion parameters
+
+For information about modifying ingestion parameters and flows, see [Knowledge ingestion settings](/knowledge#knowledge-ingestion-settings) and [Knowledge ingestion flows](/knowledge#knowledge-ingestion-flows).
+
+## See also
+
+* [Ingest knowledge](/knowledge)
+* [Filter knowledge](/knowledge-filters)
+* [Chat with knowledge](/chat)
\ No newline at end of file
diff --git a/docs/docs/core-components/knowledge-filters.mdx b/docs/docs/core-components/knowledge-filters.mdx
new file mode 100644
index 00000000..1b3a974b
--- /dev/null
+++ b/docs/docs/core-components/knowledge-filters.mdx
@@ -0,0 +1,53 @@
+---
+title: Filter knowledge
+slug: /knowledge-filters
+---
+
+import Icon from "@site/src/components/icon/icon";
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+OpenRAG's knowledge filters help you organize and manage your [knowledge base](/knowledge-configure) by creating pre-defined views of your documents.
+
+Each knowledge filter captures a specific subset of documents based on given a search query and filters.
+
+Knowledge filters can be used with different OpenRAG functionality.
+For example, knowledge filters can help agents access large knowledge bases efficiently by narrowing the scope of documents that you want the agent to use.
+
+## Create a filter
+
+To create a knowledge filter, do the following:
+
+1. Click **Knowledge**, and then click **Knowledge Filters**.
+
+2. Enter a **Name** and **Description**, and then click **Create Filter**.
+
+ By default, new filters match all documents in your knowledge base.
+ Modify the filter to customize it.
+
+3. To modify the filter, click **Knowledge**, and then click your new filter. You can edit the following settings:
+
+ * **Search Query**: Enter text for semantic search, such as `financial reports from Q4`.
+ * **Data Sources**: Select specific data sources or folders to include.
+ * **Document Types**: Filter by file type.
+ * **Owners**: Filter by the user that uploaded the documents.
+ * **Connectors**: Filter by [upload source](/knowledge), such as the local file system or a Google Drive OAuth connector.
+ * **Response Limit**: Set the maximum number of results to return from the knowledge base. The default is `10`.
+ * **Score Threshold**: Set the minimum relevance score for similarity search. The default score is `0`.
+
+4. To save your changes, click **Update Filter**.
+
+## Apply a filter {#apply-a-filter}
+
+* **Apply a global filter**: Click **Knowledge**, and then enable the toggle next to your preferred filter. Only one filter can be the global filter. The global filter applies to all chat sessions.
+
+* **Apply a chat filter**: In the **Chat** window, click **Filter**, and then select the filter to apply.
+Chat filters apply to one chat session only.
+
+## Delete a filter
+
+1. Click **Knowledge**.
+
+2. Click the filter that you want to delete.
+
+3. Click **Delete Filter**.
\ No newline at end of file
diff --git a/docs/docs/core-components/knowledge.mdx b/docs/docs/core-components/knowledge.mdx
index 05f1caec..cb8afaad 100644
--- a/docs/docs/core-components/knowledge.mdx
+++ b/docs/docs/core-components/knowledge.mdx
@@ -1,5 +1,5 @@
---
-title: OpenSearch in OpenRAG
+title: Ingest knowledge
slug: /knowledge
---
@@ -7,24 +7,22 @@ import Icon from "@site/src/components/icon/icon";
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-OpenRAG uses [OpenSearch](https://docs.opensearch.org/latest/) for its vector-backed knowledge store.
-This is a specialized database for storing and retrieving embeddings, which helps your Agent efficiently find relevant information.
-OpenSearch provides powerful hybrid search capabilities with enterprise-grade security and multi-tenancy support.
+The documents in your OpenRAG [OpenSearch knowledge base](/knowledge-configure) provide specialized context in addition to the general knowledge available to the language model that you select when you [install OpenRAG](/install).
+Upload documents to populate your knowledge base with unique content, such as your own company documents, research papers, or websites.
+Then, the [OpenRAG **Chat**](/chat) can retrieve relevant content from your knowledge base to provide context-aware responses.
-## Authentication and document access {#auth}
+OpenRAG can ingest knowledge from direct file uploads, URLs, and OAuth connectors.
-OpenRAG supports two authentication modes based on how you [install OpenRAG](/install), and which mode you choose affects document access.
+Knowledge ingestion is powered by OpenRAG's built-in [knowledge ingestion flows](/knowledge#knowledge-ingestion-flows) that use Docling Serve to process documents before storing the documents in your OpenSearch database.
-**No-auth mode (Basic Setup)**: This mode uses a single anonymous JWT token for OpenSearch authentication, so documents uploaded to the `documents` index by one user are visible to all other users on the OpenRAG server.
+During ingestion, documents are broken into smaller chunks of content that are then embedded using your selected [embedding model](/knowledge-configure#set-the-embedding-model-and-dimensions).
+The chunks, embeddings, and associated metadata (which connects chunks of the same document) are stored in your OpenSearch database.
-**OAuth mode (Advanced Setup)**: Each OpenRAG user is granted a JWT token, and each document is tagged with user ownership. Documents are filtered by user ownership, ensuring users only see documents they uploaded or have access to.
+Like all [OpenRAG flows](/agents), you can [inspect the flows in Langflow](/agents#inspect-and-modify-flows), and you can customize them if you want to change the [ingestion settings](/knowledge#knowledge-ingestion-settings).
-## Ingest knowledge
+## Ingest local files and folders {#knowledge-ingestion-flows}
-OpenRAG supports knowledge ingestion through direct file uploads and OAuth connectors.
-To configure the knowledge ingestion pipeline parameters, see [Docling Ingestion](/ingestion).
-
-### Direct file ingestion
+
The **OpenSearch Ingestion** flow uses Langflow's [**File** component](https://docs.langflow.org/components-data#file) to split and embed files loaded from your local machine into the OpenSearch database.
@@ -38,9 +36,35 @@ The files are loaded into your OpenSearch database, and appear in the Knowledge
To add files directly to a chat session, click in the chat input and select the files you want to include. Files added this way are processed and made available to the agent for the current conversation, and are not permanently added to the knowledge base.
-
+### OpenSearch Ingestion flow
-### Ingest files through OAuth connectors {#oauth-ingestion}
+
+
+The **OpenSearch Ingestion** flow is the default knowledge ingestion flow in OpenRAG. When you **Add Knowledge** in OpenRAG, the **OpenSearch Ingestion** flow runs in the background. The flow ingests documents using Docling Serve to import and process documents.
+
+If you [inspect the flow in Langflow](/agents#inspect-and-modify-flows), you'll see that it is comprised of ten components that work together to process and store documents in your knowledge base:
+
+* The [**Docling Serve** component](https://docs.langflow.org/bundles-docling) processes input documents by connecting to your instance of Docling Serve.
+* The [**Export DoclingDocument** component](https://docs.langflow.org/components-docling) exports the processed DoclingDocument to markdown format with image export mode set to placeholder. This conversion makes the structured document data into a standardized format for further processing.
+* Three [**DataFrame Operations** components](https://docs.langflow.org/components-processing#dataframe-operations) sequentially add metadata columns to the document data of `filename`, `file_size`, and `mimetype`.
+* The [**Split Text** component](https://docs.langflow.org/components-processing#split-text) splits the processed text into chunks with a chunk size of 1000 characters and an overlap of 200 characters.
+* Four **Secret Input** components provide secure access to configuration variables: `CONNECTOR_TYPE`, `OWNER`, `OWNER_EMAIL`, and `OWNER_NAME`. These are runtime variables populated from OAuth login.
+* The **Create Data** component combines the secret inputs into a structured data object that will be associated with the document embeddings.
+* The [**Embedding Model** component](https://docs.langflow.org/components-embedding-models) generates vector embeddings using OpenAI's `text-embedding-3-small` model. The embedding model is selected at [Application onboarding] and cannot be changed.
+* The [**OpenSearch** component](https://docs.langflow.org/bundles-elastic#opensearch) stores the processed documents and their embeddings in the `documents` index at `https://opensearch:9200`. By default, the component is authenticated with a JWT token, but you can also select `basic` auth mode, and enter your OpenSearch admin username and password.
+
+To customize this flow, see [Inspect and modify flows](/agents#inspect-and-modify-flows).
+
+## Ingest knowledge from URLs {#url-flow}
+
+The **OpenSearch URL Ingestion** flow is used to ingest web content from URLs.
+This flow isn't directly accessible from the OpenRAG user interface.
+Instead, this flow is called by the [**OpenRAG OpenSearch Agent** flow](/chat#flow) as a Model Context Protocol (MCP) tool.
+The agent can call this component to fetch web content from a given URL, and then ingest that content into your OpenSearch knowledge base.
+
+For more information about MCP in Langflow, see the Langflow documentation on [MCP clients](https://docs.langflow.org/mcp-client) and [MCP servers](https://docs.langflow.org/mcp-tutorial).
+
+## Ingest files through OAuth connectors {#oauth-ingestion}
OpenRAG supports Google Drive, OneDrive, and Sharepoint as OAuth connectors for seamless document synchronization.
@@ -89,75 +113,145 @@ The ingestion process can take some time depending on the size of your documents
If ingestion fails, click **Status** to view the logged error.
-### Monitor ingestion tasks
+## Monitor ingestion
-When you upload files, process folders, or sync documents, OpenRAG processes them as background tasks.
-A badge appears on the **Tasks** icon when there are active tasks running.
-To open the Tasks menu, click **Tasks**.
+Document ingestion tasks run in the background.
-**Active Tasks** shows tasks that are currently processing.
-A **Pending** task is queued and waiting to start, a **Running** task is actively processing files, and a **Processing** task is performing ingestion operations. For each active task, you can find the task ID, start time, duration, the number of files processed so far, and the total files.
+In the OpenRAG UI, a badge is shown on **Tasks** when OpenRAG tasks are active.
+Click **Tasks** to inspect and cancel tasks:
-You can cancel active tasks by clicking **Cancel**. Canceling a task stops processing immediately and marks the task as failed.
+* **Active Tasks**: All tasks that are **Pending**, **Running**, or **Processing**.
+For each active task, depending on its state, you can find the task ID, start time, duration, number of files processed, and the total files enqueued for processing.
-## Browse knowledge
+* **Pending**: The task is queued and waiting to start.
-The **Knowledge** page lists the documents OpenRAG has ingested into the OpenSearch vector database's `documents` index.
+* **Running**: The task is actively processing files.
+
+* **Processing**: The task is performing ingestion operations.
+
+* **Failed**: Something went wrong during ingestion, or the task was manually canceled.
+
+To stop an active task, click **Cancel**. Canceling a task stops processing immediately and marks the task as **Failed**.
+
+## Browse knowledge {#browse-knowledge}
+
+The **Knowledge** page lists the documents OpenRAG has ingested into your OpenSearch database, specifically in the `documents` index.
To explore the raw contents of your knowledge base, click **Knowledge** to get a list of all ingested documents.
-Click a document to view the chunks produced from splitting the documents during ingestion into the vector database.
+Click a document to view the chunks produced from splitting the document during ingestion.
-Documents are processed with the default **OpenSearch Ingestion** flow. If you want to split your documents differently, edit the **OpenSearch Ingestion** flow, as explained in [Inspect and modify flows](/agents#inspect-and-modify-flows).
+## Troubleshoot ingestion
-## Chat with knowledge
+If an ingestion task fails, do the following:
-
+* Make sure you are uploading supported file types.
+* Split excessively large files into smaller files before uploading.
+* Remove unusual embedded content, such as videos or animations, before uploading. Although Docling can replace some non-text content with placeholders during ingestion, some embedded content might cause errors.
-
+If the OpenRAG **Chat** doesn't seem to use your documents correctly, [browse your knowledge base](#browse-knowledge) to confirm that the documents are uploaded in full, and the chunks are correct.
-## Create knowledge filters
+If the documents are present and well-formed, check your [knowledge filters](/knowledge-filters).
+If a global filter is applied, make sure the expected documents are included in the global filter.
+If the global filter excludes any documents, the agent cannot access those documents unless you apply a chat-level filter or change the global filter.
-OpenRAG includes a knowledge filter system for organizing and managing document collections.
-Knowledge filters are saved search configurations that allow you to create custom views of your document collection. They store search queries, filter criteria, and display settings that can be reused across different parts of OpenRAG.
+If text is missing or incorrectly processed, you need to reupload the documents after modifying the ingestion parameters or the documents themselves.
+For example:
-Knowledge filters help agents work more efficiently with large document collections by focusing their context within relevant documents sets.
+* Break combined documents into separate files for better metadata context.
+* Make sure scanned documents are legible enough for extraction, and enable the **OCR** option. Poorly scanned documents might require additional preparation or rescanning before ingestion.
+* Adjust the **Chunk Size** and **Chunk Overlap** settings to better suit your documents. Larger chunks provide more context but can include irrelevant information, while smaller chunks yield more precise semantic search but can lack context.
-To create a knowledge filter, do the following:
+For more information about modifying ingestion parameters and flows, see [Docling Serve for knowledge ingestion](/knowledge#docling-serve-for-knowledge-ingestion).
-1. Click **Knowledge**, and then click **Knowledge Filters**.
- The **Knowledge Filter** pane appears.
-2. Enter a **Name** and **Description**, and then click **Create Filter**.
-A new filter is created with default settings that match all documents.
-3. To modify the filter, click **Knowledge**, and then click your new filter to edit it in the **Knowledge Filter** pane.
+## Docling Serve for knowledge ingestion {#docling-serve-for-knowledge-ingestion}
- The following filter options are configurable.
-
- * **Search Query**: Enter text for semantic search, such as "financial reports from Q4".
- * **Data Sources**: Select specific data sources or folders to include.
- * **Document Types**: Filter by file type.
- * **Owners**: Filter by who uploaded the documents.
- * **Connectors**: Filter by connector types, such as local upload or Google Drive.
- * **Response Limit**: Set maximum number of results. The default is `10`.
- * **Score Threshold**: Set minimum relevance score. The default score is `0`.
+
-4. When you're done editing the filter, click **Update Filter**.
+OpenRAG uses [Docling](https://docling-project.github.io/docling/) for document ingestion.
+More specifically, OpenRAG uses [Docling Serve](https://github.com/docling-project/docling-serve), which starts a `docling serve` process on your local machine and runs Docling ingestion through an API service.
-5. To apply the filter to OpenRAG globally, click **Knowledge**, and then select the filter to apply. One filter can be enabled at a time.
+Docling ingests documents from your local machine or OAuth connectors, splits them into chunks, and stores them as separate, structured documents in the OpenSearch `documents` index.
- To apply the filter to a single chat session, in the **Chat** window, click , and then select the filter to apply.
+OpenRAG chose Docling for its support for a wide variety of file formats, high performance, and advanced understanding of tables and images.
- To delete the filter, in the **Knowledge Filter** pane, click **Delete Filter**.
+### Knowledge ingestion settings {#knowledge-ingestion-settings}
-## OpenRAG default configuration
+To modify OpenRAG's ingestion settings, including the Docling settings and ingestion flows, **Settings**.
-OpenRAG automatically detects and configures the correct vector dimensions for embedding models, ensuring optimal search performance and compatibility.
+These settings configure the Docling ingestion parameters.
-The complete list of supported models is available at [`models_service.py` in the OpenRAG repository](https://github.com/langflow-ai/openrag/blob/main/src/services/models_service.py).
+OpenRAG will warn you if `docling serve` is not running.
+To start or stop `docling serve` or any other native services, in the TUI main menu, click **Start Native Services** or **Stop Native Services**.
-You can use custom embedding models by specifying them in your configuration.
+**Embedding model** determines which AI model is used to create vector embeddings. The default is the OpenAI `text-embedding-3-small` model.
-If you use an unknown embedding model, OpenRAG automatically falls back to `1536` dimensions and logs a warning. The system continues to work, but search quality can be affected if the actual model dimensions differ from `1536`.
+**Chunk size** determines how large each text chunk is in number of characters.
+Larger chunks yield more context per chunk, but can include irrelevant information. Smaller chunks yield more precise semantic search, but can lack context.
+The default value of `1000` characters provides a good starting point that balances these considerations.
-The default embedding dimension is `1536` and the default model is `text-embedding-3-small`.
+**Chunk overlap** controls the number of characters that overlap over chunk boundaries.
+Use larger overlap values for documents where context is most important, and use smaller overlap values for simpler documents, or when optimization is most important.
+The default value of 200 characters of overlap with a chunk size of 1000 (20% overlap) is suitable for general use cases. Decrease the overlap to 10% for a more efficient pipeline, or increase to 40% for more complex documents.
-For models with known vector dimensions, see [`settings.py` in the OpenRAG repository](https://github.com/langflow-ai/openrag/blob/main/src/config/settings.py).
\ No newline at end of file
+**Table Structure** enables Docling's [`DocumentConverter`](https://docling-project.github.io/docling/reference/document_converter/) tool for parsing tables. Instead of treating tables as plain text, tables are output as structured table data with preserved relationships and metadata. **Table Structure** is enabled by default.
+
+**OCR** enables or disabled OCR processing when extracting text from images and scanned documents.
+OCR is disabled by default. This setting is best suited for processing text-based documents as quickly as possible with Docling's [`DocumentConverter`](https://docling-project.github.io/docling/reference/document_converter/). Images are ignored and not processed.
+
+Enable OCR when you are processing documents containing images with text that requires extraction, or for scanned documents. Enabling OCR can slow ingestion performance.
+
+If OpenRAG detects that the local machine is running on macOS, OpenRAG uses the [ocrmac](https://www.piwheels.org/project/ocrmac/) OCR engine. Other platforms use [easyocr](https://www.jaided.ai/easyocr/).
+
+**Picture descriptions** adds image descriptions generated by the [SmolVLM-256M-Instruct](https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct) model to OCR processing. Enabling picture descriptions can slow ingestion performance.
+
+### Use OpenRAG default ingestion instead of Docling serve
+
+If you want to use OpenRAG's built-in pipeline instead of Docling serve, set `DISABLE_INGEST_WITH_LANGFLOW=true` in [Environment variables](/reference/configuration#document-processing).
+
+The built-in pipeline still uses the Docling processor, but uses it directly without the Docling Serve API.
+
+For more information, see [`processors.py` in the OpenRAG repository](https://github.com/langflow-ai/openrag/blob/main/src/models/processors.py#L58).
+
+## Ingestion performance expectations
+
+On a local VM with 7 vCPUs and 8 GiB RAM, OpenRAG ingested approximately 5.03 GB across 1,083 files in about 42 minutes.
+This equates to approximately 2.4 documents per second.
+
+You can generally expect equal or better performance on developer laptops and significantly faster on servers.
+Throughput scales with CPU cores, memory, storage speed, and configuration choices such as embedding model, chunk size and overlap, and concurrency.
+
+This test returned 12 errors (approximately 1.1 percent).
+All errors were file-specific, and they didn't stop the pipeline.
+
+* Ingestion dataset:
+
+ * Total files: 1,083 items mounted
+ * Total size on disk: 5,026,474,862 bytes (approximately 5.03 GB)
+
+* Hardware specifications:
+
+ * Machine: Apple M4 Pro
+ * Podman VM:
+ * Name: `podman-machine-default`
+ * Type: `applehv`
+ * vCPUs: 7
+ * Memory: 8 GiB
+ * Disk size: 100 GiB
+
+* Test results:
+
+ ```text
+ 2025-09-24T22:40:45.542190Z /app/src/main.py:231 Ingesting default documents when ready disable_langflow_ingest=False
+ 2025-09-24T22:40:45.546385Z /app/src/main.py:270 Using Langflow ingestion pipeline for default documents file_count=1082
+ ...
+ 2025-09-24T23:19:44.866365Z /app/src/main.py:351 Langflow ingestion completed success_count=1070 error_count=12 total_files=1082
+ ```
+
+* Elapsed time: Approximately 42 minutes 15 seconds (2,535 seconds)
+
+* Throughput: Approximately 2.4 documents/second
+
+## See also
+
+* [Configure OpenSearch in OpenRAG](/knowledge-configure)
+* [Filter knowledge](/knowledge-filters)
\ No newline at end of file
diff --git a/docs/docs/get-started/install.mdx b/docs/docs/get-started/install.mdx
index 3c1c4bcc..3d6e4b73 100644
--- a/docs/docs/get-started/install.mdx
+++ b/docs/docs/get-started/install.mdx
@@ -190,14 +190,14 @@ If the TUI detects OAuth credentials, it enforces the **Advanced Setup** path.
**Basic Setup** can generate all of the required values for OpenRAG. The OpenAI API key is optional and can be provided during onboarding.
**Basic Setup** does not set up OAuth connections for ingestion from cloud providers.
For OAuth setup, use **Advanced Setup**.
- For information about the difference between basic (no auth) and OAuth in OpenRAG, see [Authentication and document access](/knowledge#auth).
+ For information about the difference between basic (no auth) and OAuth in OpenRAG, see [OpenSearch authentication and document access](/knowledge-configure#auth).
- 1. To install OpenRAG with **Basic Setup**, click **Basic Setup** or press 1.
+ 1. To install OpenRAG with **Basic Setup**, click **Basic Setup** or press 1.
2. Click **Generate Passwords** to generate passwords for OpenSearch and Langflow.
-
+
The OpenSearch password is required. The Langflow admin password is optional.
If no Langflow admin password is generated, Langflow runs in [autologin mode](https://docs.langflow.org/api-keys-and-authentication#langflow-auto-login) with no password required.
-
+
3. Optional: Paste your OpenAI API key in the OpenAI API key field. You can also provide this during onboarding or choose a different model provider.
4. Click **Save Configuration**.
Your passwords are saved in the `.env` file used to start OpenRAG.
diff --git a/docs/docs/get-started/quickstart.mdx b/docs/docs/get-started/quickstart.mdx
index 4c4b7b1f..4256158c 100644
--- a/docs/docs/get-started/quickstart.mdx
+++ b/docs/docs/get-started/quickstart.mdx
@@ -84,7 +84,7 @@ You can complete this quickstart without going through the overview.
## Load and chat with documents {#chat-with-documents}
-OpenRAG's knowledge base chat is powered by the [OpenRAG OpenSearch Agent](/agents).
+Use the [OpenRAG **Chat**](/chat) to explore the documents in your OpenRAG database using natural language queries.
Some documents are included by default to get you started, and you can load your own documents.
1. In OpenRAG, click **Chat**.
@@ -116,7 +116,7 @@ You can click a document to view the chunks of the document as they are stored i
* Click **Settings** to modify the knowledge ingestion settings.
- For more information about knowledge bases, knowledge ingestion, and the OpenRAG **Chat**, see [OpenSearch in OpenRAG](/knowledge).
+ For more information, see [Configure OpenSearch in OpenRAG](/knowledge-configure) and [Ingest knowledge](/knowledge).
## Change the language model and chat settings {#change-components}
diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx
index 1dbc4198..a5d6e8d9 100644
--- a/docs/docs/reference/configuration.mdx
+++ b/docs/docs/reference/configuration.mdx
@@ -71,8 +71,7 @@ For more information, see [Application onboarding](/install#application-onboardi
### Document processing
-Control how OpenRAG processes and ingests documents into your knowledge base.
-For more information, see [Ingestion](/ingestion).
+Control how OpenRAG [processes and ingests documents](/knowledge) into your knowledge base.
| Variable | Default | Description |
|----------|---------|-------------|
diff --git a/docs/sidebars.js b/docs/sidebars.js
index 2381a125..c8a57c97 100644
--- a/docs/sidebars.js
+++ b/docs/sidebars.js
@@ -25,25 +25,23 @@ const sidebars = {
"get-started/install",
"get-started/docker",
{
- type: "category",
- label: "Langflow in OpenRAG",
- items: [
- "core-components/agents",
- ],
+ type: "doc",
+ id: "core-components/agents",
+ label: "Flows",
},
{
type: "category",
- label: "OpenSearch in OpenRAG",
+ label: "Knowledge",
items: [
+ "core-components/knowledge-configure",
"core-components/knowledge",
+ "core-components/knowledge-filters",
],
},
{
- type: "category",
- label: "Docling in OpenRAG",
- items: [
- "core-components/ingestion",
- ],
+ type: "doc",
+ id: "core-components/chat",
+ label: "Chat",
},
"reference/configuration",
"support/troubleshoot",