Merge branch 'main' into feature/websocket-streaming-api

This commit is contained in:
aka James4u 2025-12-09 01:07:40 -08:00 committed by GitHub
commit 327a933a2f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
67 changed files with 3268 additions and 1658 deletions

1
.github/copilot-instructions.md vendored Normal file
View file

@ -0,0 +1 @@
Refer to [AGENTS.MD](../AGENTS.md) for all repo instructions.

View file

@ -3,11 +3,12 @@ name: release
on:
schedule:
- cron: '0 13 * * *' # This schedule runs every 13:00:00Z(21:00:00+08:00)
# https://github.com/orgs/community/discussions/26286?utm_source=chatgpt.com#discussioncomment-3251208
# "The create event does not support branch filter and tag filter."
# The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags.
create:
push:
tags:
- "v*.*.*" # normal release
- "nightly" # the only one mutable tag
# https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
@ -21,9 +22,9 @@ jobs:
- name: Ensure workspace ownership
run: echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
# https://github.com/actions/checkout/blob/v3/README.md
# https://github.com/actions/checkout/blob/v6/README.md
- name: Check out code
uses: actions/checkout@v4
uses: actions/checkout@v6
with:
token: ${{ secrets.GITHUB_TOKEN }} # Use the secret as an environment variable
fetch-depth: 0
@ -31,12 +32,12 @@ jobs:
- name: Prepare release body
run: |
if [[ ${GITHUB_EVENT_NAME} == "create" ]]; then
if [[ ${GITHUB_EVENT_NAME} != "schedule" ]]; then
RELEASE_TAG=${GITHUB_REF#refs/tags/}
if [[ ${RELEASE_TAG} == "nightly" ]]; then
PRERELEASE=true
else
if [[ ${RELEASE_TAG} == v* ]]; then
PRERELEASE=false
else
PRERELEASE=true
fi
echo "Workflow triggered by create tag: ${RELEASE_TAG}"
else
@ -55,7 +56,7 @@ jobs:
git fetch --tags
if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
# Determine if a given tag exists and matches a specific Git commit.
# actions/checkout@v4 fetch-tags doesn't work when triggered by schedule
# actions/checkout@v6 fetch-tags doesn't work when triggered by schedule
if [ "$(git rev-parse -q --verify "refs/tags/${RELEASE_TAG}")" = "${GITHUB_SHA}" ]; then
echo "mutable tag ${RELEASE_TAG} exists and matches ${GITHUB_SHA}"
else
@ -88,7 +89,7 @@ jobs:
- name: Build and push image
run: |
sudo docker login --username infiniflow --password-stdin <<< ${{ secrets.DOCKERHUB_TOKEN }}
sudo docker build --build-arg NEED_MIRROR=1 -t infiniflow/ragflow:${RELEASE_TAG} -f Dockerfile .
sudo docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -t infiniflow/ragflow:${RELEASE_TAG} -f Dockerfile .
sudo docker tag infiniflow/ragflow:${RELEASE_TAG} infiniflow/ragflow:latest
sudo docker push infiniflow/ragflow:${RELEASE_TAG}
sudo docker push infiniflow/ragflow:latest

View file

@ -34,9 +34,6 @@ jobs:
if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci')) }}
runs-on: [ "self-hosted", "ragflow-test" ]
steps:
# https://github.com/hmarr/debug-action
#- uses: hmarr/debug-action@v2
- name: Ensure workspace ownership
run: |
echo "Workflow triggered by ${{ github.event_name }}"
@ -44,7 +41,7 @@ jobs:
# https://github.com/actions/checkout/issues/1781
- name: Check out code
uses: actions/checkout@v4
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
@ -129,7 +126,7 @@ jobs:
- name: Run unit test
run: |
uv sync --python 3.10 --group test --frozen
uv sync --python 3.11 --group test --frozen
source .venv/bin/activate
which pytest || echo "pytest not in PATH"
echo "Start to run unit test"
@ -141,7 +138,7 @@ jobs:
RAGFLOW_IMAGE=infiniflow/ragflow:${GITHUB_RUN_ID}
echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> ${GITHUB_ENV}
sudo docker pull ubuntu:22.04
sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 -f Dockerfile -t ${RAGFLOW_IMAGE} .
sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} .
if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
export HTTP_API_TEST_LEVEL=p3
else
@ -201,7 +198,7 @@ jobs:
echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV}
sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} up -d
uv sync --python 3.10 --only-group test --no-default-groups --frozen && uv pip install sdk/python --group test
uv sync --python 3.11 --only-group test --no-default-groups --frozen && uv pip install sdk/python --group test
- name: Run sdk tests against Elasticsearch
run: |

110
AGENTS.md Normal file
View file

@ -0,0 +1,110 @@
# RAGFlow Project Instructions for GitHub Copilot
This file provides context, build instructions, and coding standards for the RAGFlow project.
It is structured to follow GitHub Copilot's [customization guidelines](https://docs.github.com/en/copilot/concepts/prompting/response-customization).
## 1. Project Overview
RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It is a full-stack application with a Python backend and a React/TypeScript frontend.
- **Backend**: Python 3.10+ (Flask/Quart)
- **Frontend**: TypeScript, React, UmiJS
- **Architecture**: Microservices based on Docker.
- `api/`: Backend API server.
- `rag/`: Core RAG logic (indexing, retrieval).
- `deepdoc/`: Document parsing and OCR.
- `web/`: Frontend application.
## 2. Directory Structure
- `api/`: Backend API server (Flask/Quart).
- `apps/`: API Blueprints (Knowledge Base, Chat, etc.).
- `db/`: Database models and services.
- `rag/`: Core RAG logic.
- `llm/`: LLM, Embedding, and Rerank model abstractions.
- `deepdoc/`: Document parsing and OCR modules.
- `agent/`: Agentic reasoning components.
- `web/`: Frontend application (React + UmiJS).
- `docker/`: Docker deployment configurations.
- `sdk/`: Python SDK.
- `test/`: Backend tests.
## 3. Build Instructions
### Backend (Python)
The project uses **uv** for dependency management.
1. **Setup Environment**:
```bash
uv sync --python 3.11 --all-extras
uv run download_deps.py
```
2. **Run Server**:
- **Pre-requisite**: Start dependent services (MySQL, ES/Infinity, Redis, MinIO).
```bash
docker compose -f docker/docker-compose-base.yml up -d
```
- **Launch**:
```bash
source .venv/bin/activate
export PYTHONPATH=$(pwd)
bash docker/launch_backend_service.sh
```
### Frontend (TypeScript/React)
Located in `web/`.
1. **Install Dependencies**:
```bash
cd web
npm install
```
2. **Run Dev Server**:
```bash
npm run dev
```
Runs on port 8000 by default.
### Docker Deployment
To run the full stack using Docker:
```bash
cd docker
docker compose -f docker-compose.yml up -d
```
## 4. Testing Instructions
### Backend Tests
- **Run All Tests**:
```bash
uv run pytest
```
- **Run Specific Test**:
```bash
uv run pytest test/test_api.py
```
### Frontend Tests
- **Run Tests**:
```bash
cd web
npm run test
```
## 5. Coding Standards & Guidelines
- **Python Formatting**: Use `ruff` for linting and formatting.
```bash
ruff check
ruff format
```
- **Frontend Linting**:
```bash
cd web
npm run lint
```
- **Pre-commit**: Ensure pre-commit hooks are installed.
```bash
pre-commit install
pre-commit run --all-files
```

View file

@ -45,7 +45,7 @@ RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on d
### Backend Development
```bash
# Install Python dependencies
uv sync --python 3.10 --all-extras
uv sync --python 3.11 --all-extras
uv run download_deps.py
pre-commit install

View file

@ -49,20 +49,24 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
apt install -y libatk-bridge2.0-0 && \
apt install -y libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev && \
apt install -y libjemalloc-dev && \
apt install -y python3-pip pipx nginx unzip curl wget git vim less && \
apt install -y nginx unzip curl wget git vim less && \
apt install -y ghostscript && \
apt install -y pandoc && \
apt install -y texlive
RUN if [ "$NEED_MIRROR" == "1" ]; then \
pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \
pip3 config set global.trusted-host pypi.tuna.tsinghua.edu.cn; \
# Install uv
RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/deps \
if [ "$NEED_MIRROR" == "1" ]; then \
mkdir -p /etc/uv && \
echo "[[index]]" > /etc/uv/uv.toml && \
echo 'python-install-mirror = "https://registry.npmmirror.com/-/binary/python-build-standalone/"' > /etc/uv/uv.toml && \
echo '[[index]]' >> /etc/uv/uv.toml && \
echo 'url = "https://pypi.tuna.tsinghua.edu.cn/simple"' >> /etc/uv/uv.toml && \
echo "default = true" >> /etc/uv/uv.toml; \
echo 'default = true' >> /etc/uv/uv.toml; \
fi; \
pipx install uv
tar xzf /deps/uv-x86_64-unknown-linux-gnu.tar.gz \
&& cp uv-x86_64-unknown-linux-gnu/* /usr/local/bin/ \
&& rm -rf uv-x86_64-unknown-linux-gnu \
&& uv python install 3.11
ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
ENV PATH=/root/.local/bin:$PATH
@ -147,7 +151,7 @@ RUN --mount=type=cache,id=ragflow_uv,target=/root/.cache/uv,sharing=locked \
else \
sed -i 's|pypi.tuna.tsinghua.edu.cn|pypi.org|g' uv.lock; \
fi; \
uv sync --python 3.10 --frozen
uv sync --python 3.11 --frozen
COPY web web
COPY docs docs

View file

@ -3,7 +3,7 @@
FROM scratch
# Copy resources downloaded via download_deps.py
COPY chromedriver-linux64-121-0-6167-85 chrome-linux64-121-0-6167-85 cl100k_base.tiktoken libssl1.1_1.1.1f-1ubuntu2_amd64.deb libssl1.1_1.1.1f-1ubuntu2_arm64.deb tika-server-standard-3.0.0.jar tika-server-standard-3.0.0.jar.md5 libssl*.deb /
COPY chromedriver-linux64-121-0-6167-85 chrome-linux64-121-0-6167-85 cl100k_base.tiktoken libssl1.1_1.1.1f-1ubuntu2_amd64.deb libssl1.1_1.1.1f-1ubuntu2_arm64.deb tika-server-standard-3.0.0.jar tika-server-standard-3.0.0.jar.md5 libssl*.deb uv-x86_64-unknown-linux-gnu.tar.gz /
COPY nltk_data /nltk_data

View file

@ -316,7 +316,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
```bash
git clone https://github.com/infiniflow/ragflow.git
cd ragflow/
uv sync --python 3.10 # install RAGFlow dependent python modules
uv sync --python 3.11 # install RAGFlow dependent python modules
uv run download_deps.py
pre-commit install
```

View file

@ -288,7 +288,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
```bash
git clone https://github.com/infiniflow/ragflow.git
cd ragflow/
uv sync --python 3.10 # install RAGFlow dependent python modules
uv sync --python 3.11 # install RAGFlow dependent python modules
uv run download_deps.py
pre-commit install
```

View file

@ -288,7 +288,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
```bash
git clone https://github.com/infiniflow/ragflow.git
cd ragflow/
uv sync --python 3.10 # install RAGFlow dependent python modules
uv sync --python 3.11 # install RAGFlow dependent python modules
uv run download_deps.py
pre-commit install
```

View file

@ -283,7 +283,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
```bash
git clone https://github.com/infiniflow/ragflow.git
cd ragflow/
uv sync --python 3.10 # install RAGFlow dependent python modules
uv sync --python 3.11 # install RAGFlow dependent python modules
uv run download_deps.py
pre-commit install
```

View file

@ -305,7 +305,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
```bash
git clone https://github.com/infiniflow/ragflow.git
cd ragflow/
uv sync --python 3.10 # instala os módulos Python dependentes do RAGFlow
uv sync --python 3.11 # instala os módulos Python dependentes do RAGFlow
uv run download_deps.py
pre-commit install
```

View file

@ -315,7 +315,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
```bash
git clone https://github.com/infiniflow/ragflow.git
cd ragflow/
uv sync --python 3.10 # install RAGFlow dependent python modules
uv sync --python 3.11 # install RAGFlow dependent python modules
uv run download_deps.py
pre-commit install
```

View file

@ -315,7 +315,7 @@ docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly
```bash
git clone https://github.com/infiniflow/ragflow.git
cd ragflow/
uv sync --python 3.10 # install RAGFlow dependent python modules
uv sync --python 3.11 # install RAGFlow dependent python modules
uv run download_deps.py
pre-commit install
```

View file

@ -193,7 +193,7 @@
"presence_penalty": 0.4,
"prompts": [
{
"content": "Text Content:\n{Splitter:KindDingosJam@chunks}\n",
"content": "Text Content:\n{Splitter:NineTiesSin@chunks}\n",
"role": "user"
}
],
@ -226,7 +226,7 @@
"presence_penalty": 0.4,
"prompts": [
{
"content": "Text Content:\n\n{Splitter:KindDingosJam@chunks}\n",
"content": "Text Content:\n\n{Splitter:TastyPointsLay@chunks}\n",
"role": "user"
}
],
@ -259,7 +259,7 @@
"presence_penalty": 0.4,
"prompts": [
{
"content": "Content: \n\n{Splitter:KindDingosJam@chunks}",
"content": "Content: \n\n{Splitter:CuteBusesBet@chunks}",
"role": "user"
}
],
@ -485,7 +485,7 @@
"outputs": {},
"presencePenaltyEnabled": false,
"presence_penalty": 0.4,
"prompts": "Text Content:\n{Splitter:KindDingosJam@chunks}\n",
"prompts": "Text Content:\n{Splitter:NineTiesSin@chunks}\n",
"sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nExtract the most important keywords/phrases of a given piece of text content.\n\nRequirements\n- Summarize the text content, and give the top 5 important keywords/phrases.\n- The keywords MUST be in the same language as the given piece of text content.\n- The keywords are delimited by ENGLISH COMMA.\n- Output keywords ONLY.",
"temperature": 0.1,
"temperatureEnabled": false,
@ -522,7 +522,7 @@
"outputs": {},
"presencePenaltyEnabled": false,
"presence_penalty": 0.4,
"prompts": "Text Content:\n\n{Splitter:KindDingosJam@chunks}\n",
"prompts": "Text Content:\n\n{Splitter:TastyPointsLay@chunks}\n",
"sys_prompt": "Role\nYou are a text analyzer.\n\nTask\nPropose 3 questions about a given piece of text content.\n\nRequirements\n- Understand and summarize the text content, and propose the top 3 important questions.\n- The questions SHOULD NOT have overlapping meanings.\n- The questions SHOULD cover the main content of the text as much as possible.\n- The questions MUST be in the same language as the given piece of text content.\n- One question per line.\n- Output questions ONLY.",
"temperature": 0.1,
"temperatureEnabled": false,
@ -559,7 +559,7 @@
"outputs": {},
"presencePenaltyEnabled": false,
"presence_penalty": 0.4,
"prompts": "Content: \n\n{Splitter:KindDingosJam@chunks}",
"prompts": "Content: \n\n{Splitter:BlueResultsWink@chunks}",
"sys_prompt": "Extract important structured information from the given content. Output ONLY a valid JSON string with no additional text. If no important structured information is found, output an empty JSON object: {}.\n\nImportant structured information may include: names, dates, locations, events, key facts, numerical data, or other extractable entities.",
"temperature": 0.1,
"temperatureEnabled": false,

View file

@ -24,7 +24,9 @@ logger = logging.getLogger(__name__)
# Default knobs; keep conservative to avoid unexpected behavioural changes.
DEFAULT_TIMEOUT = float(os.environ.get("HTTP_CLIENT_TIMEOUT", "15"))
# Align with requests default: follow redirects with a max of 30 unless overridden.
DEFAULT_FOLLOW_REDIRECTS = bool(int(os.environ.get("HTTP_CLIENT_FOLLOW_REDIRECTS", "1")))
DEFAULT_FOLLOW_REDIRECTS = bool(
int(os.environ.get("HTTP_CLIENT_FOLLOW_REDIRECTS", "1"))
)
DEFAULT_MAX_REDIRECTS = int(os.environ.get("HTTP_CLIENT_MAX_REDIRECTS", "30"))
DEFAULT_MAX_RETRIES = int(os.environ.get("HTTP_CLIENT_MAX_RETRIES", "2"))
DEFAULT_BACKOFF_FACTOR = float(os.environ.get("HTTP_CLIENT_BACKOFF_FACTOR", "0.5"))
@ -32,7 +34,9 @@ DEFAULT_PROXY = os.environ.get("HTTP_CLIENT_PROXY")
DEFAULT_USER_AGENT = os.environ.get("HTTP_CLIENT_USER_AGENT", "ragflow-http-client")
def _clean_headers(headers: Optional[Dict[str, str]], auth_token: Optional[str] = None) -> Optional[Dict[str, str]]:
def _clean_headers(
headers: Optional[Dict[str, str]], auth_token: Optional[str] = None
) -> Optional[Dict[str, str]]:
merged_headers: Dict[str, str] = {}
if DEFAULT_USER_AGENT:
merged_headers["User-Agent"] = DEFAULT_USER_AGENT
@ -52,46 +56,58 @@ async def async_request(
method: str,
url: str,
*,
timeout: float | httpx.Timeout | None = None,
request_timeout: float | httpx.Timeout | None = None,
follow_redirects: bool | None = None,
max_redirects: Optional[int] = None,
headers: Optional[Dict[str, str]] = None,
auth_token: Optional[str] = None,
retries: Optional[int] = None,
backoff_factor: Optional[float] = None,
proxies: Any = None,
proxy: Any = None,
**kwargs: Any,
) -> httpx.Response:
"""Lightweight async HTTP wrapper using httpx.AsyncClient with safe defaults."""
timeout = timeout if timeout is not None else DEFAULT_TIMEOUT
follow_redirects = DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects
timeout = request_timeout if request_timeout is not None else DEFAULT_TIMEOUT
follow_redirects = (
DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects
)
max_redirects = DEFAULT_MAX_REDIRECTS if max_redirects is None else max_redirects
retries = DEFAULT_MAX_RETRIES if retries is None else max(retries, 0)
backoff_factor = DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor
backoff_factor = (
DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor
)
headers = _clean_headers(headers, auth_token=auth_token)
proxies = DEFAULT_PROXY if proxies is None else proxies
proxy = DEFAULT_PROXY if proxy is None else proxy
async with httpx.AsyncClient(
timeout=timeout,
follow_redirects=follow_redirects,
max_redirects=max_redirects,
proxies=proxies,
proxy=proxy,
) as client:
last_exc: Exception | None = None
for attempt in range(retries + 1):
try:
start = time.monotonic()
response = await client.request(method=method, url=url, headers=headers, **kwargs)
response = await client.request(
method=method, url=url, headers=headers, **kwargs
)
duration = time.monotonic() - start
logger.debug(f"async_request {method} {url} -> {response.status_code} in {duration:.3f}s")
logger.debug(
f"async_request {method} {url} -> {response.status_code} in {duration:.3f}s"
)
return response
except httpx.RequestError as exc:
last_exc = exc
if attempt >= retries:
logger.warning(f"async_request exhausted retries for {method} {url}: {exc}")
logger.warning(
f"async_request exhausted retries for {method} {url}: {exc}"
)
raise
delay = _get_delay(backoff_factor, attempt)
logger.warning(f"async_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s")
logger.warning(
f"async_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s"
)
await asyncio.sleep(delay)
raise last_exc # pragma: no cover
@ -107,39 +123,51 @@ def sync_request(
auth_token: Optional[str] = None,
retries: Optional[int] = None,
backoff_factor: Optional[float] = None,
proxies: Any = None,
proxy: Any = None,
**kwargs: Any,
) -> httpx.Response:
"""Synchronous counterpart to async_request, for CLI/tests or sync contexts."""
timeout = timeout if timeout is not None else DEFAULT_TIMEOUT
follow_redirects = DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects
follow_redirects = (
DEFAULT_FOLLOW_REDIRECTS if follow_redirects is None else follow_redirects
)
max_redirects = DEFAULT_MAX_REDIRECTS if max_redirects is None else max_redirects
retries = DEFAULT_MAX_RETRIES if retries is None else max(retries, 0)
backoff_factor = DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor
backoff_factor = (
DEFAULT_BACKOFF_FACTOR if backoff_factor is None else backoff_factor
)
headers = _clean_headers(headers, auth_token=auth_token)
proxies = DEFAULT_PROXY if proxies is None else proxies
proxy = DEFAULT_PROXY if proxy is None else proxy
with httpx.Client(
timeout=timeout,
follow_redirects=follow_redirects,
max_redirects=max_redirects,
proxies=proxies,
proxy=proxy,
) as client:
last_exc: Exception | None = None
for attempt in range(retries + 1):
try:
start = time.monotonic()
response = client.request(method=method, url=url, headers=headers, **kwargs)
response = client.request(
method=method, url=url, headers=headers, **kwargs
)
duration = time.monotonic() - start
logger.debug(f"sync_request {method} {url} -> {response.status_code} in {duration:.3f}s")
logger.debug(
f"sync_request {method} {url} -> {response.status_code} in {duration:.3f}s"
)
return response
except httpx.RequestError as exc:
last_exc = exc
if attempt >= retries:
logger.warning(f"sync_request exhausted retries for {method} {url}: {exc}")
logger.warning(
f"sync_request exhausted retries for {method} {url}: {exc}"
)
raise
delay = _get_delay(backoff_factor, attempt)
logger.warning(f"sync_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s")
logger.warning(
f"sync_request attempt {attempt + 1}/{retries + 1} failed for {method} {url}: {exc}; retrying in {delay:.2f}s"
)
time.sleep(delay)
raise last_exc # pragma: no cover

View file

@ -150,7 +150,7 @@ class MCPToolCallSession(ToolCallSession):
except asyncio.CancelledError:
break
async def _call_mcp_server(self, task_type: MCPTaskType, timeout: float | int = 8, **kwargs) -> Any:
async def _call_mcp_server(self, task_type: MCPTaskType, request_timeout: float | int = 8, **kwargs) -> Any:
if self._close:
raise ValueError("Session is closed")
@ -158,18 +158,18 @@ class MCPToolCallSession(ToolCallSession):
await self._queue.put((task_type, kwargs, results))
try:
result: CallToolResult | Exception = await asyncio.wait_for(results.get(), timeout=timeout)
result: CallToolResult | Exception = await asyncio.wait_for(results.get(), timeout=request_timeout)
if isinstance(result, Exception):
raise result
return result
except asyncio.TimeoutError:
raise asyncio.TimeoutError(f"MCP task '{task_type}' timeout after {timeout}s")
raise asyncio.TimeoutError(f"MCP task '{task_type}' timeout after {request_timeout}s")
except Exception:
raise
async def _call_mcp_tool(self, name: str, arguments: dict[str, Any], timeout: float | int = 10) -> str:
async def _call_mcp_tool(self, name: str, arguments: dict[str, Any], request_timeout: float | int = 10) -> str:
result: CallToolResult = await self._call_mcp_server("tool_call", name=name, arguments=arguments,
timeout=timeout)
request_timeout=request_timeout)
if result.isError:
return f"MCP server error: {result.content}"
@ -180,9 +180,9 @@ class MCPToolCallSession(ToolCallSession):
else:
return f"Unsupported content type {type(result.content)}"
async def _get_tools_from_mcp_server(self, timeout: float | int = 8) -> list[Tool]:
async def _get_tools_from_mcp_server(self, request_timeout: float | int = 8) -> list[Tool]:
try:
result: ListToolsResult = await self._call_mcp_server("list_tools", timeout=timeout)
result: ListToolsResult = await self._call_mcp_server("list_tools", request_timeout=request_timeout)
return result.tools
except Exception:
raise
@ -191,7 +191,7 @@ class MCPToolCallSession(ToolCallSession):
if self._close:
raise ValueError("Session is closed")
future = asyncio.run_coroutine_threadsafe(self._get_tools_from_mcp_server(timeout=timeout), self._event_loop)
future = asyncio.run_coroutine_threadsafe(self._get_tools_from_mcp_server(request_timeout=timeout), self._event_loop)
try:
return future.result(timeout=timeout)
except FuturesTimeoutError:

View file

@ -210,7 +210,10 @@ def init_settings():
IMAGE2TEXT_CFG = _resolve_per_model_config(image2text_entry, LLM_FACTORY, API_KEY, LLM_BASE_URL)
CHAT_MDL = CHAT_CFG.get("model", "") or ""
EMBEDDING_MDL = os.getenv("TEI_MODEL", "BAAI/bge-small-en-v1.5") if "tei-" in os.getenv("COMPOSE_PROFILES", "") else ""
EMBEDDING_MDL = EMBEDDING_CFG.get("model", "") or ""
compose_profiles = os.getenv("COMPOSE_PROFILES", "")
if "tei-" in compose_profiles:
EMBEDDING_MDL = os.getenv("TEI_MODEL", EMBEDDING_MDL or "BAAI/bge-small-en-v1.5")
RERANK_MDL = RERANK_CFG.get("model", "") or ""
ASR_MDL = ASR_CFG.get("model", "") or ""
IMAGE2TEXT_MDL = IMAGE2TEXT_CFG.get("model", "") or ""

View file

@ -2,6 +2,7 @@
"id": {"type": "varchar", "default": ""},
"doc_id": {"type": "varchar", "default": ""},
"kb_id": {"type": "varchar", "default": ""},
"mom_id": {"type": "varchar", "default": ""},
"create_time": {"type": "varchar", "default": ""},
"create_timestamp_flt": {"type": "float", "default": 0.0},
"img_id": {"type": "varchar", "default": ""},

View file

@ -72,7 +72,7 @@ services:
infinity:
profiles:
- infinity
image: infiniflow/infinity:v0.6.10
image: infiniflow/infinity:v0.6.11
volumes:
- infinity_data:/var/infinity
- ./infinity_conf.toml:/infinity_conf.toml

View file

@ -1,5 +1,5 @@
[general]
version = "0.6.10"
version = "0.6.11"
time_zone = "utc-8"
[network]

View file

@ -41,13 +41,19 @@ cd ragflow/
pipx install uv
```
2. Install Python dependencies:
2. Install RAGFlow service's Python dependencies:
```bash
uv sync --python 3.10 # install RAGFlow dependent python modules
uv sync --python 3.11 --frozen
```
*A virtual environment named `.venv` is created, and all Python dependencies are installed into the new environment.*
If you need to run tests against the RAGFlow service, install the test dependencies:
```bash
uv sync --python 3.11 --group test --frozen && uv pip install sdk/python --group test
```
### Launch third-party services
The following command launches the 'base' services (MinIO, Elasticsearch, Redis, and MySQL) using Docker Compose:

View file

@ -176,7 +176,7 @@ This section is contributed by our community contributor [yiminghub2024](https:/
iii. Copy [docker/entrypoint.sh](https://github.com/infiniflow/ragflow/blob/main/docker/entrypoint.sh) locally.
iv. Install the required dependencies using `uv`:
- Run `uv add mcp` or
- Copy [pyproject.toml](https://github.com/infiniflow/ragflow/blob/main/pyproject.toml) locally and run `uv sync --python 3.10`.
- Copy [pyproject.toml](https://github.com/infiniflow/ragflow/blob/main/pyproject.toml) locally and run `uv sync --python 3.11`.
2. Edit **docker-compose.yml** to enable MCP (disabled by default).
3. Launch the MCP server:

View file

@ -28,6 +28,7 @@ def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]:
"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
["https://registry.npmmirror.com/-/binary/chrome-for-testing/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"],
["https://registry.npmmirror.com/-/binary/chrome-for-testing/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"],
"https://github.com/astral-sh/uv/releases/download/0.9.16/uv-x86_64-unknown-linux-gnu.tar.gz",
]
else:
return [
@ -38,6 +39,7 @@ def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]:
"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"],
["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"],
"https://github.com/astral-sh/uv/releases/download/0.9.16/uv-x86_64-unknown-linux-gnu.tar.gz",
]

View file

@ -96,7 +96,7 @@ ragflow:
infinity:
image:
repository: infiniflow/infinity
tag: v0.6.10
tag: v0.6.11
pullPolicy: IfNotPresent
pullSecrets: []
storage:

View file

@ -5,7 +5,7 @@ description = "[RAGFlow](https://ragflow.io/) is an open-source RAG (Retrieval-A
authors = [{ name = "Zhichang Yu", email = "yuzhichang@gmail.com" }]
license-files = ["LICENSE"]
readme = "README.md"
requires-python = ">=3.10,<3.13"
requires-python = ">=3.11,<3.15"
dependencies = [
"datrie>=0.8.3,<0.9.0",
"akshare>=1.15.78,<2.0.0",
@ -49,7 +49,7 @@ dependencies = [
"html-text==0.6.2",
"httpx[socks]>=0.28.1,<0.29.0",
"huggingface-hub>=0.25.0,<0.26.0",
"infinity-sdk==0.6.10",
"infinity-sdk==0.6.11",
"infinity-emb>=0.0.66,<0.0.67",
"itsdangerous==2.1.2",
"json-repair==0.35.0",
@ -92,7 +92,7 @@ dependencies = [
"ranx==0.3.20",
"readability-lxml==0.8.1",
"valkey==6.0.2",
"requests==2.32.2",
"requests>=2.32.3,<3.0.0",
"replicate==0.31.0",
"roman-numbers==1.0.2",
"ruamel-base==1.0.0",
@ -101,7 +101,7 @@ dependencies = [
"scikit-learn==1.5.0",
"selenium==4.22.0",
"selenium-wire==5.1.0",
"setuptools>=75.2.0,<76.0.0",
"setuptools>=78.1.1,<81.0.0",
"shapely==2.0.5",
"six==1.16.0",
"slack-sdk==3.37.0",

View file

@ -14,6 +14,7 @@
# limitations under the License.
#
import asyncio
import io
import re
@ -50,7 +51,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs):
}
)
cv_mdl = LLMBundle(tenant_id, llm_type=LLMType.IMAGE2TEXT, lang=lang)
ans = cv_mdl.chat(system="", history=[], gen_conf={}, video_bytes=binary, filename=filename)
ans = asyncio.run(cv_mdl.async_chat(system="", history=[], gen_conf={}, video_bytes=binary, filename=filename))
callback(0.8, "CV LLM respond: %s ..." % ans[:32])
ans += "\n" + ans
tokenize(doc, ans, eng)

View file

@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import io
import json
import os
@ -634,7 +635,7 @@ class Parser(ProcessBase):
self.set_output("output_format", conf["output_format"])
cv_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.IMAGE2TEXT, llm_name=conf["llm_id"])
txt = cv_mdl.chat(system="", history=[], gen_conf={}, video_bytes=blob, filename=name)
txt = asyncio.run(cv_mdl.async_chat(system="", history=[], gen_conf={}, video_bytes=blob, filename=name))
self.set_output("text", txt)

View file

@ -28,7 +28,7 @@ import json_repair
import litellm
import openai
from openai import AsyncOpenAI, OpenAI
from openai.lib.azure import AzureOpenAI
from openai.lib.azure import AzureOpenAI, AsyncAzureOpenAI
from strenum import StrEnum
from common.token_utils import num_tokens_from_string, total_token_count_from_response
@ -535,6 +535,7 @@ class AzureChat(Base):
api_version = json.loads(key).get("api_version", "2024-02-01")
super().__init__(key, model_name, base_url, **kwargs)
self.client = AzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)
self.async_client = AsyncAzureOpenAI(api_key=key, base_url=base_url, api_version=api_version)
self.model_name = model_name
@property

View file

@ -14,6 +14,7 @@
# limitations under the License.
#
import asyncio
import base64
import json
import logging
@ -27,9 +28,8 @@ from pathlib import Path
from urllib.parse import urljoin
import requests
from openai import OpenAI
from openai.lib.azure import AzureOpenAI
from zhipuai import ZhipuAI
from openai import OpenAI, AsyncOpenAI
from openai.lib.azure import AzureOpenAI, AsyncAzureOpenAI
from common.token_utils import num_tokens_from_string, total_token_count_from_response
from rag.nlp import is_english
@ -76,9 +76,9 @@ class Base(ABC):
pmpt.append({"type": "image_url", "image_url": {"url": img if isinstance(img, str) and img.startswith("data:") else f"data:image/png;base64,{img}"}})
return pmpt
def chat(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat(self, system, history, gen_conf, images=None, **kwargs):
try:
response = self.client.chat.completions.create(
response = await self.async_client.chat.completions.create(
model=self.model_name,
messages=self._form_history(system, history, images),
extra_body=self.extra_body,
@ -87,17 +87,17 @@ class Base(ABC):
except Exception as e:
return "**ERROR**: " + str(e), 0
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
ans = ""
tk_count = 0
try:
response = self.client.chat.completions.create(
response = await self.async_client.chat.completions.create(
model=self.model_name,
messages=self._form_history(system, history, images),
stream=True,
extra_body=self.extra_body,
)
for resp in response:
async for resp in response:
if not resp.choices[0].delta.content:
continue
delta = resp.choices[0].delta.content
@ -191,6 +191,7 @@ class GptV4(Base):
base_url = "https://api.openai.com/v1"
self.api_key = key
self.client = OpenAI(api_key=key, base_url=base_url)
self.async_client = AsyncOpenAI(api_key=key, base_url=base_url)
self.model_name = model_name
self.lang = lang
super().__init__(**kwargs)
@ -221,6 +222,7 @@ class AzureGptV4(GptV4):
api_key = json.loads(key).get("api_key", "")
api_version = json.loads(key).get("api_version", "2024-02-01")
self.client = AzureOpenAI(api_key=api_key, azure_endpoint=kwargs["base_url"], api_version=api_version)
self.async_client = AsyncAzureOpenAI(api_key=api_key, azure_endpoint=kwargs["base_url"], api_version=api_version)
self.model_name = model_name
self.lang = lang
Base.__init__(self, **kwargs)
@ -243,7 +245,7 @@ class QWenCV(GptV4):
base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1"
super().__init__(key, model_name, lang=lang, base_url=base_url, **kwargs)
def chat(self, system, history, gen_conf, images=None, video_bytes=None, filename="", **kwargs):
async def async_chat(self, system, history, gen_conf, images=None, video_bytes=None, filename="", **kwargs):
if video_bytes:
try:
summary, summary_num_tokens = self._process_video(video_bytes, filename)
@ -313,7 +315,8 @@ class Zhipu4V(GptV4):
_FACTORY_NAME = "ZHIPU-AI"
def __init__(self, key, model_name="glm-4v", lang="Chinese", **kwargs):
self.client = ZhipuAI(api_key=key)
self.client = OpenAI(api_key=key, base_url="https://open.bigmodel.cn/api/paas/v4/")
self.async_client = AsyncOpenAI(api_key=key, base_url="https://open.bigmodel.cn/api/paas/v4/")
self.model_name = model_name
self.lang = lang
Base.__init__(self, **kwargs)
@ -342,20 +345,20 @@ class Zhipu4V(GptV4):
)
return response.json()
def chat(self, system, history, gen_conf, images=None, stream=False, **kwargs):
async def async_chat(self, system, history, gen_conf, images=None, **kwargs):
if system and history and history[0].get("role") != "system":
history.insert(0, {"role": "system", "content": system})
gen_conf = self._clean_conf(gen_conf)
logging.info(json.dumps(history, ensure_ascii=False, indent=2))
response = self.client.chat.completions.create(model=self.model_name, messages=self._form_history(system, history, images), stream=False, **gen_conf)
response = await self.async_client.chat.completions.create(model=self.model_name, messages=self._form_history(system, history, images), stream=False, **gen_conf)
content = response.choices[0].message.content.strip()
cleaned = re.sub(r"<\|(begin_of_box|end_of_box)\|>", "", content).strip()
return cleaned, total_token_count_from_response(response)
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
from rag.llm.chat_model import LENGTH_NOTIFICATION_CN, LENGTH_NOTIFICATION_EN
from rag.nlp import is_chinese
@ -366,8 +369,8 @@ class Zhipu4V(GptV4):
tk_count = 0
try:
logging.info(json.dumps(history, ensure_ascii=False, indent=2))
response = self.client.chat.completions.create(model=self.model_name, messages=self._form_history(system, history, images), stream=True, **gen_conf)
for resp in response:
response = await self.async_client.chat.completions.create(model=self.model_name, messages=self._form_history(system, history, images), stream=True, **gen_conf)
async for resp in response:
if not resp.choices[0].delta.content:
continue
delta = resp.choices[0].delta.content
@ -412,6 +415,7 @@ class StepFunCV(GptV4):
if not base_url:
base_url = "https://api.stepfun.com/v1"
self.client = OpenAI(api_key=key, base_url=base_url)
self.async_client = AsyncOpenAI(api_key=key, base_url=base_url)
self.model_name = model_name
self.lang = lang
Base.__init__(self, **kwargs)
@ -425,6 +429,7 @@ class VolcEngineCV(GptV4):
base_url = "https://ark.cn-beijing.volces.com/api/v3"
ark_api_key = json.loads(key).get("ark_api_key", "")
self.client = OpenAI(api_key=ark_api_key, base_url=base_url)
self.async_client = AsyncOpenAI(api_key=ark_api_key, base_url=base_url)
self.model_name = json.loads(key).get("ep_id", "") + json.loads(key).get("endpoint_id", "")
self.lang = lang
Base.__init__(self, **kwargs)
@ -438,6 +443,7 @@ class LmStudioCV(GptV4):
raise ValueError("Local llm url cannot be None")
base_url = urljoin(base_url, "v1")
self.client = OpenAI(api_key="lm-studio", base_url=base_url)
self.async_client = AsyncOpenAI(api_key="lm-studio", base_url=base_url)
self.model_name = model_name
self.lang = lang
Base.__init__(self, **kwargs)
@ -451,6 +457,7 @@ class OpenAI_APICV(GptV4):
raise ValueError("url cannot be None")
base_url = urljoin(base_url, "v1")
self.client = OpenAI(api_key=key, base_url=base_url)
self.async_client = AsyncOpenAI(api_key=key, base_url=base_url)
self.model_name = model_name.split("___")[0]
self.lang = lang
Base.__init__(self, **kwargs)
@ -491,6 +498,7 @@ class OpenRouterCV(GptV4):
base_url = "https://openrouter.ai/api/v1"
api_key = json.loads(key).get("api_key", "")
self.client = OpenAI(api_key=api_key, base_url=base_url)
self.async_client = AsyncOpenAI(api_key=api_key, base_url=base_url)
self.model_name = model_name
self.lang = lang
Base.__init__(self, **kwargs)
@ -522,6 +530,7 @@ class LocalAICV(GptV4):
raise ValueError("Local cv model url cannot be None")
base_url = urljoin(base_url, "v1")
self.client = OpenAI(api_key="empty", base_url=base_url)
self.async_client = AsyncOpenAI(api_key="empty", base_url=base_url)
self.model_name = model_name.split("___")[0]
self.lang = lang
Base.__init__(self, **kwargs)
@ -533,6 +542,7 @@ class XinferenceCV(GptV4):
def __init__(self, key, model_name="", lang="Chinese", base_url="", **kwargs):
base_url = urljoin(base_url, "v1")
self.client = OpenAI(api_key=key, base_url=base_url)
self.async_client = AsyncOpenAI(api_key=key, base_url=base_url)
self.model_name = model_name
self.lang = lang
Base.__init__(self, **kwargs)
@ -546,6 +556,7 @@ class GPUStackCV(GptV4):
raise ValueError("Local llm url cannot be None")
base_url = urljoin(base_url, "v1")
self.client = OpenAI(api_key=key, base_url=base_url)
self.async_client = AsyncOpenAI(api_key=key, base_url=base_url)
self.model_name = model_name
self.lang = lang
Base.__init__(self, **kwargs)
@ -635,19 +646,19 @@ class OllamaCV(Base):
except Exception as e:
return "**ERROR**: " + str(e), 0
def chat(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat(self, system, history, gen_conf, images=None, **kwargs):
try:
response = self.client.chat(model=self.model_name, messages=self._form_history(system, history, images), options=self._clean_conf(gen_conf), keep_alive=self.keep_alive)
response = await asyncio.to_thread(self.client.chat, model=self.model_name, messages=self._form_history(system, history, images), options=self._clean_conf(gen_conf), keep_alive=self.keep_alive)
ans = response["message"]["content"].strip()
return ans, response["eval_count"] + response.get("prompt_eval_count", 0)
except Exception as e:
return "**ERROR**: " + str(e), 0
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
ans = ""
try:
response = self.client.chat(model=self.model_name, messages=self._form_history(system, history, images), stream=True, options=self._clean_conf(gen_conf), keep_alive=self.keep_alive)
response = await asyncio.to_thread(self.client.chat, model=self.model_name, messages=self._form_history(system, history, images), stream=True, options=self._clean_conf(gen_conf), keep_alive=self.keep_alive)
for resp in response:
if resp["done"]:
yield resp.get("prompt_eval_count", 0) + resp.get("eval_count", 0)
@ -780,41 +791,41 @@ class GeminiCV(Base):
)
return res.text, total_token_count_from_response(res)
def chat(self, system, history, gen_conf, images=None, video_bytes=None, filename="", **kwargs):
async def async_chat(self, system, history, gen_conf, images=None, video_bytes=None, filename="", **kwargs):
if video_bytes:
try:
size = len(video_bytes) if video_bytes else 0
logging.info(f"[GeminiCV] chat called with video: filename={filename} size={size}")
summary, summary_num_tokens = self._process_video(video_bytes, filename)
logging.info(f"[GeminiCV] async_chat called with video: filename={filename} size={size}")
summary, summary_num_tokens = await asyncio.to_thread(self._process_video, video_bytes, filename)
return summary, summary_num_tokens
except Exception as e:
logging.info(f"[GeminiCV] chat video error: {e}")
logging.info(f"[GeminiCV] async_chat video error: {e}")
return "**ERROR**: " + str(e), 0
from google.genai import types
history_len = len(history) if history else 0
images_len = len(images) if images else 0
logging.info(f"[GeminiCV] chat called: history_len={history_len} images_len={images_len} gen_conf={gen_conf}")
logging.info(f"[GeminiCV] async_chat called: history_len={history_len} images_len={images_len} gen_conf={gen_conf}")
generation_config = types.GenerateContentConfig(
temperature=gen_conf.get("temperature", 0.3),
top_p=gen_conf.get("top_p", 0.7),
)
try:
response = self.client.models.generate_content(
response = await self.client.aio.models.generate_content(
model=self.model_name,
contents=self._form_history(system, history, images),
config=generation_config,
)
ans = response.text
logging.info("[GeminiCV] chat completed")
logging.info("[GeminiCV] async_chat completed")
return ans, total_token_count_from_response(response)
except Exception as e:
logging.warning(f"[GeminiCV] chat error: {e}")
logging.warning(f"[GeminiCV] async_chat error: {e}")
return "**ERROR**: " + str(e), 0
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
ans = ""
response = None
try:
@ -826,15 +837,15 @@ class GeminiCV(Base):
)
history_len = len(history) if history else 0
images_len = len(images) if images else 0
logging.info(f"[GeminiCV] chat_streamly called: history_len={history_len} images_len={images_len} gen_conf={gen_conf}")
logging.info(f"[GeminiCV] async_chat_streamly called: history_len={history_len} images_len={images_len} gen_conf={gen_conf}")
response_stream = self.client.models.generate_content_stream(
response_stream = await self.client.aio.models.generate_content_stream(
model=self.model_name,
contents=self._form_history(system, history, images),
config=generation_config,
)
for chunk in response_stream:
async for chunk in response_stream:
if chunk.text:
ans += chunk.text
yield chunk.text
@ -939,17 +950,17 @@ class NvidiaCV(Base):
response = self._request(vision_prompt)
return (response["choices"][0]["message"]["content"].strip(), total_token_count_from_response(response))
def chat(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat(self, system, history, gen_conf, images=None, **kwargs):
try:
response = self._request(self._form_history(system, history, images), gen_conf)
response = await asyncio.to_thread(self._request, self._form_history(system, history, images), gen_conf)
return (response["choices"][0]["message"]["content"].strip(), total_token_count_from_response(response))
except Exception as e:
return "**ERROR**: " + str(e), 0
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
total_tokens = 0
try:
response = self._request(self._form_history(system, history, images), gen_conf)
response = await asyncio.to_thread(self._request, self._form_history(system, history, images), gen_conf)
cnt = response["choices"][0]["message"]["content"]
total_tokens += total_token_count_from_response(response)
for resp in cnt:
@ -967,6 +978,7 @@ class AnthropicCV(Base):
import anthropic
self.client = anthropic.Anthropic(api_key=key)
self.async_client = anthropic.AsyncAnthropic(api_key=key)
self.model_name = model_name
self.system = ""
self.max_tokens = 8192
@ -1012,17 +1024,18 @@ class AnthropicCV(Base):
gen_conf["max_tokens"] = self.max_tokens
return gen_conf
def chat(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat(self, system, history, gen_conf, images=None, **kwargs):
gen_conf = self._clean_conf(gen_conf)
ans = ""
try:
response = self.client.messages.create(
response = await self.async_client.messages.create(
model=self.model_name,
messages=self._form_history(system, history, images),
system=system,
stream=False,
**gen_conf,
).to_dict()
)
response = response.to_dict()
ans = response["content"][0]["text"]
if response["stop_reason"] == "max_tokens":
ans += "...\nFor the content length reason, it stopped, continue?" if is_english([ans]) else "······\n由于长度的原因,回答被截断了,要继续吗?"
@ -1033,11 +1046,11 @@ class AnthropicCV(Base):
except Exception as e:
return ans + "\n**ERROR**: " + str(e), 0
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
gen_conf = self._clean_conf(gen_conf)
total_tokens = 0
try:
response = self.client.messages.create(
response = self.async_client.messages.create(
model=self.model_name,
messages=self._form_history(system, history, images),
system=system,
@ -1045,7 +1058,7 @@ class AnthropicCV(Base):
**gen_conf,
)
think = False
for res in response:
async for res in response:
if res.type == "content_block_delta":
if res.delta.type == "thinking_delta" and res.delta.thinking:
if not think:
@ -1117,18 +1130,18 @@ class GoogleCV(AnthropicCV, GeminiCV):
else:
return GeminiCV.describe_with_prompt(self, image, prompt)
def chat(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat(self, system, history, gen_conf, images=None, **kwargs):
if "claude" in self.model_name:
return AnthropicCV.chat(self, system, history, gen_conf, images)
return await AnthropicCV.async_chat(self, system, history, gen_conf, images)
else:
return GeminiCV.chat(self, system, history, gen_conf, images)
return await GeminiCV.async_chat(self, system, history, gen_conf, images)
def chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
async def async_chat_streamly(self, system, history, gen_conf, images=None, **kwargs):
if "claude" in self.model_name:
for ans in AnthropicCV.chat_streamly(self, system, history, gen_conf, images):
async for ans in AnthropicCV.async_chat_streamly(self, system, history, gen_conf, images):
yield ans
else:
for ans in GeminiCV.chat_streamly(self, system, history, gen_conf, images):
async for ans in GeminiCV.async_chat_streamly(self, system, history, gen_conf, images):
yield ans

View file

@ -91,7 +91,7 @@ class Dealer:
["docnm_kwd", "content_ltks", "kb_id", "img_id", "title_tks", "important_kwd", "position_int",
"doc_id", "page_num_int", "top_int", "create_timestamp_flt", "knowledge_graph_kwd",
"question_kwd", "question_tks", "doc_type_kwd",
"available_int", "content_with_weight", PAGERANK_FLD, TAG_FLD])
"available_int", "content_with_weight", "mom_id", PAGERANK_FLD, TAG_FLD])
kwds = set([])
qst = req.get("question", "")
@ -469,6 +469,7 @@ class Dealer:
"vector": chunk.get(vector_column, zero_vector),
"positions": position_int,
"doc_type_kwd": chunk.get("doc_type_kwd", ""),
"mom_id": chunk.get("mom_id", ""),
}
if highlight and sres.highlight:
if id in sres.highlight:
@ -650,7 +651,8 @@ class Dealer:
i = 0
while i < len(chunks):
ck = chunks[i]
if not ck.get("mom_id"):
mom_id = ck.get("mom_id")
if not isinstance(mom_id, str) or not mom_id.strip():
i += 1
continue
mom_chunks[ck["mom_id"]].append(chunks.pop(i))

View file

@ -781,7 +781,10 @@ async def run_toc_from_text(chunks, chat_mdl, callback=None):
# Merge structure and content (by index)
prune = len(toc_with_levels) > 512
max_lvl = sorted([t.get("level", "0") for t in toc_with_levels if isinstance(t, dict)])[-1]
max_lvl = "0"
sorted_list = sorted([t.get("level", "0") for t in toc_with_levels if isinstance(t, dict)])
if sorted_list:
max_lvl = sorted_list[-1]
merged = []
for _ , (toc_item, src_item) in enumerate(zip(toc_with_levels, filtered)):
if prune and toc_item.get("level", "0") >= max_lvl:

View file

@ -727,17 +727,17 @@ async def insert_es(task_id, task_tenant_id, task_dataset_id, chunks, progress_c
if not mom:
continue
id = xxhash.xxh64(mom.encode("utf-8")).hexdigest()
ck["mom_id"] = id
if id in mother_ids:
continue
mother_ids.add(id)
ck["mom_id"] = id
mom_ck = copy.deepcopy(ck)
mom_ck["id"] = id
mom_ck["content_with_weight"] = mom
mom_ck["available_int"] = 0
flds = list(mom_ck.keys())
for fld in flds:
if fld not in ["id", "content_with_weight", "doc_id", "kb_id", "available_int", "position_int"]:
if fld not in ["id", "content_with_weight", "doc_id", "docnm_kwd", "kb_id", "available_int", "position_int"]:
del mom_ck[fld]
mothers.append(mom_ck)

View file

@ -443,6 +443,9 @@ class InfinityConnection(DocStoreConnection):
del matchExpr.extra_options["similarity"]
logger.debug(f"INFINITY search MatchDenseExpr: {json.dumps(matchExpr.__dict__)}")
elif isinstance(matchExpr, FusionExpr):
if matchExpr.method == "weighted_sum":
# The default is "minmax" which gives a zero score for the last doc.
matchExpr.fusion_params["normalize"] = "atan"
logger.debug(f"INFINITY search FusionExpr: {json.dumps(matchExpr.__dict__)}")
order_by_expr_list = list()

2808
uv.lock generated

File diff suppressed because it is too large Load diff

View file

@ -45,7 +45,7 @@ export function ConfirmDeleteDialog({
const { t } = useTranslation();
if (hidden) {
return children;
return children || <></>;
}
return (
@ -54,7 +54,7 @@ export function ConfirmDeleteDialog({
open={open}
defaultOpen={defaultOpen}
>
<AlertDialogTrigger asChild>{children}</AlertDialogTrigger>
{children && <AlertDialogTrigger asChild>{children}</AlertDialogTrigger>}
<AlertDialogOverlay
onClick={(e) => {
e.stopPropagation();
@ -109,23 +109,28 @@ export function ConfirmDeleteDialog({
export const ConfirmDeleteDialogNode = ({
avatar,
name,
warnText,
children,
}: {
avatar?: { avatar?: string; name?: string; isPerson?: boolean };
name?: string;
warnText?: string;
children?: React.ReactNode;
}) => {
return (
<div className="flex items-center border-0.5 text-text-secondary border-border-button rounded-lg px-3 py-4">
{avatar && (
<RAGFlowAvatar
className="w-8 h-8"
avatar={avatar.avatar}
isPerson={avatar.isPerson}
name={avatar.name}
/>
)}
{name && <div className="ml-3">{name}</div>}
<div className="flex flex-col gap-2.5">
<div className="flex items-center border-0.5 text-text-secondary border-border-button rounded-lg px-3 py-4">
{avatar && (
<RAGFlowAvatar
className="w-8 h-8"
avatar={avatar.avatar}
isPerson={avatar.isPerson}
name={avatar.name}
/>
)}
{name && <div className="ml-3">{name}</div>}
</div>
{warnText && <div className="text-state-error text-xs">{warnText}</div>}
{children}
</div>
);

View file

@ -110,7 +110,7 @@ export interface DynamicFormRef {
}
// Generate Zod validation schema based on field configurations
const generateSchema = (fields: FormFieldConfig[]): ZodSchema<any> => {
export const generateSchema = (fields: FormFieldConfig[]): ZodSchema<any> => {
const schema: Record<string, ZodSchema> = {};
const nestedSchemas: Record<string, Record<string, ZodSchema>> = {};
@ -311,6 +311,271 @@ const generateDefaultValues = <T extends FieldValues>(
return defaultValues as DefaultValues<T>;
};
// Render form fields
export const RenderField = ({
field,
labelClassName,
}: {
field: FormFieldConfig;
labelClassName?: string;
}) => {
const form = useFormContext();
if (field.render) {
if (field.type === FormFieldType.Custom && field.hideLabel) {
return <div className="w-full">{field.render({})}</div>;
}
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (e: any) => {
fieldProps.onChange(e);
field.onChange?.(e.target?.value ?? e);
},
}
: fieldProps;
return field.render?.(finalFieldProps);
}}
</RAGFlowFormItem>
);
}
switch (field.type) {
case FormFieldType.Textarea:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (e: any) => {
fieldProps.onChange(e);
field.onChange?.(e.target.value);
},
}
: fieldProps;
return (
<Textarea
{...finalFieldProps}
placeholder={field.placeholder}
// className="resize-none"
/>
);
}}
</RAGFlowFormItem>
);
case FormFieldType.Select:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (value: string) => {
console.log('select value', value);
if (fieldProps.onChange) {
fieldProps.onChange(value);
}
field.onChange?.(value);
},
}
: fieldProps;
return (
<SelectWithSearch
triggerClassName="!shrink"
{...finalFieldProps}
options={field.options}
/>
);
}}
</RAGFlowFormItem>
);
case FormFieldType.MultiSelect:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
console.log('multi select value', fieldProps);
const finalFieldProps = {
...fieldProps,
onValueChange: (value: string[]) => {
if (fieldProps.onChange) {
fieldProps.onChange(value);
}
field.onChange?.(value);
},
};
return (
<MultiSelect
variant="inverted"
maxCount={100}
{...finalFieldProps}
// onValueChange={(data) => {
// console.log(data);
// field.onChange?.(data);
// }}
options={field.options as MultiSelectOptionType[]}
/>
);
}}
</RAGFlowFormItem>
);
case FormFieldType.Checkbox:
return (
<FormField
control={form.control}
name={field.name as any}
render={({ field: formField }) => (
<FormItem
className={cn('flex items-center w-full', {
'flex-row items-center space-x-3 space-y-0': !field.horizontal,
})}
>
{field.label && !field.horizontal && (
<div className="space-y-1 leading-none">
<FormLabel
className={cn(
'font-medium',
labelClassName || field.labelClassName,
)}
tooltip={field.tooltip}
>
{field.label}{' '}
{field.required && (
<span className="text-destructive">*</span>
)}
</FormLabel>
</div>
)}
{field.label && field.horizontal && (
<div className="space-y-1 leading-none w-1/4">
<FormLabel
className={cn(
'font-medium',
labelClassName || field.labelClassName,
)}
tooltip={field.tooltip}
>
{field.label}{' '}
{field.required && (
<span className="text-destructive">*</span>
)}
</FormLabel>
</div>
)}
<FormControl>
<div className={cn({ 'w-full': field.horizontal })}>
<Checkbox
checked={formField.value}
onCheckedChange={(checked) => {
formField.onChange(checked);
field.onChange?.(checked);
}}
/>
</div>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
);
case FormFieldType.Tag:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (value: string[]) => {
fieldProps.onChange(value);
field.onChange?.(value);
},
}
: fieldProps;
return (
// <TagInput {...fieldProps} placeholder={field.placeholder} />
<div className="w-full">
<EditTag {...finalFieldProps}></EditTag>
</div>
);
}}
</RAGFlowFormItem>
);
default:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (e: any) => {
fieldProps.onChange(e);
field.onChange?.(e.target.value);
},
}
: fieldProps;
return (
<div className="w-full">
<Input
{...finalFieldProps}
type={field.type}
placeholder={field.placeholder}
/>
</div>
);
}}
</RAGFlowFormItem>
);
}
};
// Dynamic form component
const DynamicForm = {
@ -497,266 +762,6 @@ const DynamicForm = {
// Submit handler
// const handleSubmit = form.handleSubmit(onSubmit);
// Render form fields
const renderField = (field: FormFieldConfig) => {
if (field.render) {
if (field.type === FormFieldType.Custom && field.hideLabel) {
return <div className="w-full">{field.render({})}</div>;
}
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (e: any) => {
fieldProps.onChange(e);
field.onChange?.(e.target?.value ?? e);
},
}
: fieldProps;
return field.render?.(finalFieldProps);
}}
</RAGFlowFormItem>
);
}
switch (field.type) {
case FormFieldType.Textarea:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (e: any) => {
fieldProps.onChange(e);
field.onChange?.(e.target.value);
},
}
: fieldProps;
return (
<Textarea
{...finalFieldProps}
placeholder={field.placeholder}
// className="resize-none"
/>
);
}}
</RAGFlowFormItem>
);
case FormFieldType.Select:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (value: string) => {
console.log('select value', value);
if (fieldProps.onChange) {
fieldProps.onChange(value);
}
field.onChange?.(value);
},
}
: fieldProps;
return (
<SelectWithSearch
triggerClassName="!shrink"
{...finalFieldProps}
options={field.options}
/>
);
}}
</RAGFlowFormItem>
);
case FormFieldType.MultiSelect:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
console.log('multi select value', fieldProps);
const finalFieldProps = {
...fieldProps,
onValueChange: (value: string[]) => {
if (fieldProps.onChange) {
fieldProps.onChange(value);
}
field.onChange?.(value);
},
};
return (
<MultiSelect
variant="inverted"
maxCount={100}
{...finalFieldProps}
// onValueChange={(data) => {
// console.log(data);
// field.onChange?.(data);
// }}
options={field.options as MultiSelectOptionType[]}
/>
);
}}
</RAGFlowFormItem>
);
case FormFieldType.Checkbox:
return (
<FormField
control={form.control}
name={field.name as any}
render={({ field: formField }) => (
<FormItem
className={cn('flex items-center w-full', {
'flex-row items-center space-x-3 space-y-0':
!field.horizontal,
})}
>
{field.label && !field.horizontal && (
<div className="space-y-1 leading-none">
<FormLabel
className={cn(
'font-medium',
labelClassName || field.labelClassName,
)}
tooltip={field.tooltip}
>
{field.label}{' '}
{field.required && (
<span className="text-destructive">*</span>
)}
</FormLabel>
</div>
)}
{field.label && field.horizontal && (
<div className="space-y-1 leading-none w-1/4">
<FormLabel
className={cn(
'font-medium',
labelClassName || field.labelClassName,
)}
tooltip={field.tooltip}
>
{field.label}{' '}
{field.required && (
<span className="text-destructive">*</span>
)}
</FormLabel>
</div>
)}
<FormControl>
<div className={cn({ 'w-full': field.horizontal })}>
<Checkbox
checked={formField.value}
onCheckedChange={(checked) => {
formField.onChange(checked);
field.onChange?.(checked);
}}
/>
</div>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
);
case FormFieldType.Tag:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (value: string[]) => {
fieldProps.onChange(value);
field.onChange?.(value);
},
}
: fieldProps;
return (
// <TagInput {...fieldProps} placeholder={field.placeholder} />
<div className="w-full">
<EditTag {...finalFieldProps}></EditTag>
</div>
);
}}
</RAGFlowFormItem>
);
default:
return (
<RAGFlowFormItem
name={field.name}
label={field.label}
required={field.required}
horizontal={field.horizontal}
tooltip={field.tooltip}
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
const finalFieldProps = field.onChange
? {
...fieldProps,
onChange: (e: any) => {
fieldProps.onChange(e);
field.onChange?.(e.target.value);
},
}
: fieldProps;
return (
<div className="w-full">
<Input
{...finalFieldProps}
type={field.type}
placeholder={field.placeholder}
/>
</div>
);
}}
</RAGFlowFormItem>
);
}
};
// Watch all form values to re-render when they change (for shouldRender checks)
const formValues = form.watch();
@ -779,7 +784,10 @@ const DynamicForm = {
key={field.name}
className={cn({ hidden: field.hidden || !shouldShow })}
>
{renderField(field)}
<RenderField
field={field}
labelClassName={labelClassName}
/>
</div>
);
})}
@ -798,7 +806,7 @@ const DynamicForm = {
buttonText,
submitFunc,
}: {
submitLoading: boolean;
submitLoading?: boolean;
buttonText?: string;
submitFunc?: (values: FieldValues) => void;
}) => {

View file

@ -53,7 +53,12 @@ export function RAGFlowFormItem({
{label}
</FormLabel>
)}
<div className="w-full flex flex-col">
<div
className={cn('flex flex-col', {
'w-3/4': horizontal,
'w-full': !horizontal,
})}
>
<FormControl>
{typeof children === 'function'
? children(field)

View file

@ -70,6 +70,7 @@ export function Header() {
{ path: Routes.Chats, name: t('header.chat'), icon: MessageSquareText },
{ path: Routes.Searches, name: t('header.search'), icon: Search },
{ path: Routes.Agents, name: t('header.flow'), icon: Cpu },
// { path: Routes.Memories, name: t('header.Memories'), icon: Cpu },
{ path: Routes.Files, name: t('header.fileManager'), icon: File },
],
[t],

View file

@ -101,7 +101,7 @@ export default {
dataset: 'Dataset',
Memories: 'Memory',
},
memory: {
memories: {
memory: 'Memory',
createMemory: 'Create Memory',
name: 'Name',
@ -110,9 +110,15 @@ export default {
embeddingModel: 'Embedding model',
selectModel: 'Select model',
llm: 'LLM',
delMemoryWarn: `After deletion, all messages in this memory will be deleted and cannot be retrieved by agents.`,
},
memoryDetail: {
memory: {
messages: {
copied: 'Copied!',
contentEmbed: 'Content embed',
content: 'Content',
delMessageWarn: `After forgetting, this message will not be retrieved by agents.`,
forgetMessage: 'Forget message',
sessionId: 'Session ID',
agent: 'Agent',
type: 'Type',
@ -122,6 +128,27 @@ export default {
enable: 'Enable',
action: 'Action',
},
config: {
avatar: 'Avatar',
description: 'Description',
memorySize: 'Memory size',
advancedSettings: 'Advanced Settings',
permission: 'Permission',
onlyMe: 'Only Me',
team: 'Team',
storageType: 'Storage Type',
storageTypePlaceholder: 'Please select storage type',
forgetPolicy: 'Forget Policy',
temperature: 'Temperature',
systemPrompt: 'System Prompt',
systemPromptPlaceholder: 'Please enter system prompt',
userPrompt: 'User Prompt',
userPromptPlaceholder: 'Please enter user prompt',
},
sideBar: {
messages: 'Messages',
configuration: 'Configuration',
},
},
knowledgeList: {
welcome: 'Welcome back',

View file

@ -1,4 +1,5 @@
import { SelectWithSearch } from '@/components/originui/select-with-search';
import { SliderInputFormField } from '@/components/slider-input-form-field';
import {
FormControl,
FormField,
@ -11,6 +12,7 @@ import { Spin } from '@/components/ui/spin';
import { Switch } from '@/components/ui/switch';
import { useTranslate } from '@/hooks/common-hooks';
import { cn } from '@/lib/utils';
import { t } from 'i18next';
import { useMemo, useState } from 'react';
import { FieldValues, useFormContext } from 'react-hook-form';
import {
@ -285,3 +287,14 @@ export function EnableTocToggle() {
/>
);
}
export function OverlappedPercent() {
return (
<SliderInputFormField
name="parser_config.overlapped_percent"
label={t('flow.filenameEmbeddingWeight')}
max={0.5}
step={0.01}
></SliderInputFormField>
);
}

View file

@ -10,7 +10,7 @@ import {
ConfigurationFormContainer,
MainContainer,
} from '../configuration-form-container';
import { EnableTocToggle } from './common-item';
import { EnableTocToggle, OverlappedPercent } from './common-item';
export function NaiveConfiguration() {
return (
@ -20,6 +20,7 @@ export function NaiveConfiguration() {
<MaxTokenNumberFormField initialValue={512}></MaxTokenNumberFormField>
<DelimiterFormField></DelimiterFormField>
<EnableTocToggle />
<OverlappedPercent />
</ConfigurationFormContainer>
<ConfigurationFormContainer>
<AutoKeywordsFormField></AutoKeywordsFormField>

View file

@ -29,6 +29,7 @@ export const formSchema = z
tag_kb_ids: z.array(z.string()).nullish(),
topn_tags: z.number().optional(),
toc_extraction: z.boolean().optional(),
overlapped_percent: z.number().optional(),
raptor: z
.object({
use_raptor: z.boolean().optional(),

View file

@ -67,6 +67,7 @@ export default function DatasetSettings() {
html4excel: false,
topn_tags: 3,
toc_extraction: false,
overlapped_percent: 0,
raptor: {
use_raptor: true,
max_token: 256,

View file

@ -14,16 +14,18 @@ export function PermissionFormField() {
}, [t]);
return (
<RAGFlowFormItem
name="permission"
label={t('knowledgeConfiguration.permissions')}
tooltip={t('knowledgeConfiguration.permissionsTip')}
horizontal
>
<SelectWithSearch
options={teamOptions}
triggerClassName="w-3/4"
></SelectWithSearch>
</RAGFlowFormItem>
<div className="items-center">
<RAGFlowFormItem
name="permission"
label={t('knowledgeConfiguration.permissions')}
tooltip={t('knowledgeConfiguration.permissionsTip')}
horizontal={true}
>
<SelectWithSearch
options={teamOptions}
triggerClassName="w-full"
></SelectWithSearch>
</RAGFlowFormItem>
</div>
);
}

View file

@ -3,7 +3,7 @@ import { useModelOptions } from '@/components/llm-setting-items/llm-form-field';
import { HomeIcon } from '@/components/svg-icon';
import { Modal } from '@/components/ui/modal/modal';
import { t } from 'i18next';
import { useCallback, useEffect, useState } from 'react';
import { memo, useCallback, useMemo, useState } from 'react';
import { createMemoryFields } from './constants';
import { IMemory } from './interface';
@ -13,11 +13,10 @@ type IProps = {
onSubmit?: (data: any) => void;
initialMemory: IMemory;
loading?: boolean;
isCreate?: boolean;
};
export const AddOrEditModal = (props: IProps) => {
const { open, onClose, onSubmit, initialMemory } = props;
// const [fields, setFields] = useState<FormFieldConfig[]>(createMemoryFields);
// const formRef = useRef<DynamicFormRef>(null);
export const AddOrEditModal = memo((props: IProps) => {
const { open, onClose, onSubmit, initialMemory, isCreate } = props;
const [formInstance, setFormInstance] = useState<DynamicFormRef | null>(null);
const formCallbackRef = useCallback((node: DynamicFormRef | null) => {
@ -28,15 +27,25 @@ export const AddOrEditModal = (props: IProps) => {
}, []);
const { modelOptions } = useModelOptions();
useEffect(() => {
if (initialMemory && initialMemory.id) {
formInstance?.onFieldUpdate('memory_type', { hidden: true });
formInstance?.onFieldUpdate('embedding', { hidden: true });
formInstance?.onFieldUpdate('llm', { hidden: true });
const fields = useMemo(() => {
if (!isCreate) {
return createMemoryFields.filter((field: any) => field.name === 'name');
} else {
formInstance?.onFieldUpdate('llm', { options: modelOptions as any });
const tempFields = createMemoryFields.map((field: any) => {
if (field.name === 'llm_id') {
return {
...field,
options: modelOptions,
};
} else {
return {
...field,
};
}
});
return tempFields;
}
}, [modelOptions, formInstance, initialMemory]);
}, [modelOptions, isCreate]);
return (
<Modal
@ -48,7 +57,7 @@ export const AddOrEditModal = (props: IProps) => {
<div>
<HomeIcon name="memory" width={'24'} />
</div>
{t('memory.createMemory')}
{t('memories.createMemory')}
</div>
}
showfooter={false}
@ -56,7 +65,7 @@ export const AddOrEditModal = (props: IProps) => {
>
<DynamicForm.Root
ref={formCallbackRef}
fields={createMemoryFields}
fields={fields}
onSubmit={() => {}}
defaultValues={initialMemory}
>
@ -72,4 +81,4 @@ export const AddOrEditModal = (props: IProps) => {
</DynamicForm.Root>
</Modal>
);
};
});

View file

@ -4,16 +4,16 @@ import { t } from 'i18next';
export const createMemoryFields = [
{
name: 'memory_name',
label: t('memory.name'),
placeholder: t('memory.memoryNamePlaceholder'),
name: 'name',
label: t('memories.name'),
placeholder: t('memories.memoryNamePlaceholder'),
required: true,
},
{
name: 'memory_type',
label: t('memory.memoryType'),
label: t('memories.memoryType'),
type: FormFieldType.MultiSelect,
placeholder: t('memory.descriptionPlaceholder'),
placeholder: t('memories.descriptionPlaceholder'),
options: [
{ label: 'Raw', value: 'raw' },
{ label: 'Semantic', value: 'semantic' },
@ -23,18 +23,18 @@ export const createMemoryFields = [
required: true,
},
{
name: 'embedding',
label: t('memory.embeddingModel'),
placeholder: t('memory.selectModel'),
name: 'embd_id',
label: t('memories.embeddingModel'),
placeholder: t('memories.selectModel'),
required: true,
// hideLabel: true,
// type: 'custom',
render: (field) => <EmbeddingSelect field={field} isEdit={false} />,
},
{
name: 'llm',
label: t('memory.llm'),
placeholder: t('memory.selectModel'),
name: 'llm_id',
label: t('memories.llm'),
placeholder: t('memories.selectModel'),
required: true,
type: FormFieldType.Select,
},

View file

@ -7,6 +7,7 @@ import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
import memoryService, { updateMemoryById } from '@/services/memory-service';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { useDebounce } from 'ahooks';
import { omit } from 'lodash';
import { useCallback, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useParams, useSearchParams } from 'umi';
@ -73,12 +74,12 @@ export const useFetchMemoryList = () => {
queryFn: async () => {
const { data: response } = await memoryService.getMemoryList(
{
params: {
data: {
keywords: debouncedSearchString,
page_size: pagination.pageSize,
page: pagination.current,
},
data: {},
params: {},
},
true,
);
@ -153,7 +154,9 @@ export const useDeleteMemory = () => {
} = useMutation<DeleteMemoryResponse, Error, DeleteMemoryProps>({
mutationKey: ['deleteMemory'],
mutationFn: async (props) => {
const { data: response } = await memoryService.deleteMemory(props);
const { data: response } = await memoryService.deleteMemory(
props.memory_id,
);
if (response.code !== 0) {
throw new Error(response.message || 'Failed to delete memory');
}
@ -189,7 +192,8 @@ export const useUpdateMemory = () => {
} = useMutation<any, Error, IMemoryAppDetailProps>({
mutationKey: ['updateMemory'],
mutationFn: async (formData) => {
const { data: response } = await updateMemoryById(formData.id, formData);
const param = omit(formData, ['id']);
const { data: response } = await updateMemoryById(formData.id, param);
if (response.code !== 0) {
throw new Error(response.message || 'Failed to update memory');
}
@ -259,7 +263,7 @@ export const useRenameMemory = () => {
// const { id, created_by, update_time, ...memoryDataTemp } = detail;
res = await updateMemory({
// ...memoryDataTemp,
name: data.memory_name,
name: data.name,
id: memory?.id,
} as unknown as IMemoryAppDetailProps);
} catch (e) {
@ -268,9 +272,9 @@ export const useRenameMemory = () => {
} else {
res = await createMemory(data);
}
if (res && !memory?.id) {
navigateToMemory(res?.id)();
}
// if (res && !memory?.id) {
// navigateToMemory(res?.id)();
// }
callBack?.();
setLoading(false);
handleHideModal();

View file

@ -7,7 +7,7 @@ import { RAGFlowPagination } from '@/components/ui/ragflow-pagination';
import { useTranslate } from '@/hooks/common-hooks';
import { pick } from 'lodash';
import { Plus } from 'lucide-react';
import { useCallback, useEffect } from 'react';
import { useCallback, useEffect, useState } from 'react';
import { useSearchParams } from 'umi';
import { AddOrEditModal } from './add-or-edit-modal';
import { useFetchMemoryList, useRenameMemory } from './hooks';
@ -16,7 +16,8 @@ import { MemoryCard } from './memory-card';
export default function MemoryList() {
// const { data } = useFetchFlowList();
const { t } = useTranslate('memory');
const { t } = useTranslate('memories');
const [addOrEditType, setAddOrEditType] = useState<'add' | 'edit'>('add');
// const [isEdit, setIsEdit] = useState(false);
const {
data: list,
@ -43,6 +44,7 @@ export default function MemoryList() {
};
const openCreateModalFun = useCallback(() => {
// setIsEdit(false);
setAddOrEditType('add');
showMemoryRenameModal();
}, [showMemoryRenameModal]);
const handlePageChange = useCallback(
@ -121,6 +123,7 @@ export default function MemoryList() {
key={x.id}
data={x}
showMemoryRenameModal={() => {
setAddOrEditType('edit');
showMemoryRenameModal(x);
}}
></MemoryCard>
@ -152,6 +155,7 @@ export default function MemoryList() {
{openCreateModal && (
<AddOrEditModal
initialMemory={initialMemory}
isCreate={addOrEditType === 'add'}
open={openCreateModal}
loading={searchRenameLoading}
onClose={hideMemoryModal}

View file

@ -1,10 +1,3 @@
export interface ICreateMemoryProps {
memory_name: string;
memory_type: Array<string>;
embedding: string;
llm: string;
}
export interface CreateMemoryResponse {
id: string;
name: string;
@ -24,17 +17,18 @@ export type MemoryType = 'raw' | 'semantic' | 'episodic' | 'procedural';
export type StorageType = 'table' | 'graph';
export type Permissions = 'me' | 'team';
export type ForgettingPolicy = 'fifo' | 'lru';
export interface IMemory {
id: string;
export interface ICreateMemoryProps {
name: string;
memory_type: MemoryType[];
embd_id: string;
llm_id: string;
}
export interface IMemory extends ICreateMemoryProps {
id: string;
avatar: string;
tenant_id: string;
owner_name: string;
memory_type: MemoryType[];
storage_type: StorageType;
embedding: string;
llm: string;
permissions: Permissions;
description: string;
memory_size: number;

View file

@ -20,7 +20,7 @@ export function MemoryCard({ data, showMemoryRenameModal }: IProps) {
}}
moreDropdown={
<MemoryDropdown
dataset={data}
memory={data}
showMemoryRenameModal={showMemoryRenameModal}
>
<MoreButton></MoreButton>

View file

@ -12,15 +12,16 @@ import {
import { PenLine, Trash2 } from 'lucide-react';
import { MouseEventHandler, PropsWithChildren, useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { IMemoryAppProps, useDeleteMemory } from './hooks';
import { useDeleteMemory } from './hooks';
import { IMemory } from './interface';
export function MemoryDropdown({
children,
dataset,
memory,
showMemoryRenameModal,
}: PropsWithChildren & {
dataset: IMemoryAppProps;
showMemoryRenameModal: (dataset: IMemoryAppProps) => void;
memory: IMemory;
showMemoryRenameModal: (memory: IMemory) => void;
}) {
const { t } = useTranslation();
const { deleteMemory } = useDeleteMemory();
@ -28,13 +29,13 @@ export function MemoryDropdown({
useCallback(
(e) => {
e.stopPropagation();
showMemoryRenameModal(dataset);
showMemoryRenameModal(memory);
},
[dataset, showMemoryRenameModal],
[memory, showMemoryRenameModal],
);
const handleDelete: MouseEventHandler<HTMLDivElement> = useCallback(() => {
deleteMemory({ search_id: dataset.id });
}, [dataset.id, deleteMemory]);
deleteMemory({ memory_id: memory.id });
}, [memory, deleteMemory]);
return (
<DropdownMenu>
@ -50,8 +51,9 @@ export function MemoryDropdown({
content={{
node: (
<ConfirmDeleteDialogNode
avatar={{ avatar: dataset.avatar, name: dataset.name }}
name={dataset.name}
avatar={{ avatar: memory.avatar, name: memory.name }}
name={memory.name}
warnText={t('memories.delMemoryWarn')}
/>
),
}}

View file

@ -1,3 +1,5 @@
export enum MemoryApiAction {
FetchMemoryDetail = 'fetchMemoryDetail',
FetchMemoryMessage = 'fetchMemoryMessage',
FetchMessageContent = 'fetchMessageContent',
}

View file

@ -1,59 +0,0 @@
import { useHandleSearchChange } from '@/hooks/logic-hooks';
import { getMemoryDetailById } from '@/services/memory-service';
import { useQuery } from '@tanstack/react-query';
import { useParams, useSearchParams } from 'umi';
import { MemoryApiAction } from '../constant';
import { IMessageTableProps } from '../memory-message/interface';
export const useFetchMemoryMessageList = (props?: {
refreshCount?: number;
}) => {
const { refreshCount } = props || {};
const { id } = useParams();
const [searchParams] = useSearchParams();
const memoryBaseId = searchParams.get('id') || id;
const { handleInputChange, searchString, pagination, setPagination } =
useHandleSearchChange();
let queryKey: (MemoryApiAction | number)[] = [
MemoryApiAction.FetchMemoryDetail,
];
if (typeof refreshCount === 'number') {
queryKey = [MemoryApiAction.FetchMemoryDetail, refreshCount];
}
const { data, isFetching: loading } = useQuery<IMessageTableProps>({
queryKey: [...queryKey, searchString, pagination],
initialData: {} as IMessageTableProps,
gcTime: 0,
queryFn: async () => {
if (memoryBaseId) {
const { data } = await getMemoryDetailById(memoryBaseId as string, {
// filter: {
// agent_id: '',
// },
keyword: searchString,
page: pagination.current,
page_size: pagination.pageSize,
});
// setPagination({
// page: data?.page ?? 1,
// pageSize: data?.page_size ?? 10,
// total: data?.total ?? 0,
// });
return data?.data ?? {};
} else {
return {};
}
},
});
return {
data,
loading,
handleInputChange,
searchString,
pagination,
setPagination,
};
};

View file

@ -1,14 +1,11 @@
import { useHandleSearchChange } from '@/hooks/logic-hooks';
import { IMemory } from '@/pages/memories/interface';
import { getMemoryDetailById } from '@/services/memory-service';
import memoryService from '@/services/memory-service';
import { useQuery } from '@tanstack/react-query';
import { useParams, useSearchParams } from 'umi';
import { MemoryApiAction } from '../constant';
export const useFetchMemoryBaseConfiguration = (props?: {
refreshCount?: number;
}) => {
const { refreshCount } = props || {};
export const useFetchMemoryBaseConfiguration = () => {
const { id } = useParams();
const [searchParams] = useSearchParams();
const memoryBaseId = searchParams.get('id') || id;
@ -18,9 +15,6 @@ export const useFetchMemoryBaseConfiguration = (props?: {
let queryKey: (MemoryApiAction | number)[] = [
MemoryApiAction.FetchMemoryDetail,
];
if (typeof refreshCount === 'number') {
queryKey = [MemoryApiAction.FetchMemoryDetail, refreshCount];
}
const { data, isFetching: loading } = useQuery<IMemory>({
queryKey: [...queryKey, searchString, pagination],
@ -28,19 +22,9 @@ export const useFetchMemoryBaseConfiguration = (props?: {
gcTime: 0,
queryFn: async () => {
if (memoryBaseId) {
const { data } = await getMemoryDetailById(memoryBaseId as string, {
// filter: {
// agent_id: '',
// },
keyword: searchString,
page: pagination.current,
page_size: pagination.size,
});
// setPagination({
// page: data?.page ?? 1,
// pageSize: data?.page_size ?? 10,
// total: data?.total ?? 0,
// });
const { data } = await memoryService.getMemoryConfig(
memoryBaseId as string,
);
return data?.data ?? {};
} else {
return {};

View file

@ -0,0 +1,128 @@
import message from '@/components/ui/message';
import { useHandleSearchChange } from '@/hooks/logic-hooks';
import memoryService, { getMemoryDetailById } from '@/services/memory-service';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { t } from 'i18next';
import { useCallback, useState } from 'react';
import { useParams, useSearchParams } from 'umi';
import { MemoryApiAction } from '../constant';
import {
IMessageContentProps,
IMessageTableProps,
} from '../memory-message/interface';
import { IMessageInfo } from './interface';
export const useFetchMemoryMessageList = () => {
const { id } = useParams();
const [searchParams] = useSearchParams();
const memoryBaseId = searchParams.get('id') || id;
const { handleInputChange, searchString, pagination, setPagination } =
useHandleSearchChange();
let queryKey: (MemoryApiAction | number)[] = [
MemoryApiAction.FetchMemoryMessage,
];
const { data, isFetching: loading } = useQuery<IMessageTableProps>({
queryKey: [...queryKey, searchString, pagination],
initialData: {} as IMessageTableProps,
gcTime: 0,
queryFn: async () => {
if (memoryBaseId) {
const { data } = await getMemoryDetailById(memoryBaseId as string, {
keyword: searchString,
page: pagination.current,
page_size: pagination.pageSize,
});
return data?.data ?? {};
} else {
return {};
}
},
});
return {
data,
loading,
handleInputChange,
searchString,
pagination,
setPagination,
};
};
export const useMessageAction = () => {
const queryClient = useQueryClient();
const [selectedMessage, setSelectedMessage] = useState<IMessageInfo>(
{} as IMessageInfo,
);
const [showDeleteDialog, setShowDeleteDialog] = useState(false);
const handleClickDeleteMessage = useCallback((message: IMessageInfo) => {
console.log('handleClickDeleteMessage', message);
setSelectedMessage(message);
setShowDeleteDialog(true);
}, []);
const handleDeleteMessage = useCallback(() => {
// delete message
memoryService.deleteMemoryMessage(selectedMessage.message_id).then(() => {
message.success(t('message.deleted'));
queryClient.invalidateQueries({
queryKey: [MemoryApiAction.FetchMemoryMessage],
});
});
setShowDeleteDialog(false);
}, [selectedMessage.message_id, queryClient]);
const [showMessageContentDialog, setShowMessageContentDialog] =
useState(false);
const [selectedMessageContent, setSelectedMessageContent] =
useState<IMessageContentProps>({} as IMessageContentProps);
const {
data: messageContent,
isPending: fetchMessageContentLoading,
mutateAsync: fetchMessageContent,
} = useMutation<IMessageContentProps>({
mutationKey: [
MemoryApiAction.FetchMessageContent,
selectedMessage.message_id,
],
mutationFn: async () => {
setShowMessageContentDialog(true);
const res = await memoryService.getMessageContent(
selectedMessage.message_id,
);
if (res.data.code === 0) {
setSelectedMessageContent(res.data.data);
} else {
message.error(res.data.message);
}
return res.data.data;
},
});
const handleClickMessageContentDialog = useCallback(
(message: IMessageInfo) => {
setSelectedMessage(message);
fetchMessageContent();
},
[fetchMessageContent],
);
return {
selectedMessage,
setSelectedMessage,
showDeleteDialog,
setShowDeleteDialog,
handleClickDeleteMessage,
handleDeleteMessage,
messageContent,
fetchMessageContentLoading,
fetchMessageContent,
selectedMessageContent,
showMessageContentDialog,
setShowMessageContentDialog,
handleClickMessageContentDialog,
};
};

View file

@ -1,6 +1,6 @@
import ListFilterBar from '@/components/list-filter-bar';
import { t } from 'i18next';
import { useFetchMemoryMessageList } from '../hooks/use-memory-messages';
import { useFetchMemoryMessageList } from './hook';
import { MemoryTable } from './message-table';
export default function MemoryMessage() {

View file

@ -17,3 +17,8 @@ export interface IMessageTableProps {
messages: { message_list: Array<IMessageInfo>; total: number };
storage_type: string;
}
export interface IMessageContentProps {
content: string;
content_embed: string;
}

View file

@ -1,3 +1,23 @@
import {
ConfirmDeleteDialog,
ConfirmDeleteDialogNode,
} from '@/components/confirm-delete-dialog';
import { EmptyType } from '@/components/empty/constant';
import Empty from '@/components/empty/empty';
import { Button } from '@/components/ui/button';
import { Modal } from '@/components/ui/modal/modal';
import { RAGFlowPagination } from '@/components/ui/ragflow-pagination';
import { Switch } from '@/components/ui/switch';
import {
Table,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
} from '@/components/ui/table';
import { Pagination } from '@/interfaces/common';
import { replaceText } from '@/pages/dataset/process-log-modal';
import {
ColumnDef,
ColumnFiltersState,
@ -10,26 +30,13 @@ import {
getSortedRowModel,
useReactTable,
} from '@tanstack/react-table';
import * as React from 'react';
import { EmptyType } from '@/components/empty/constant';
import Empty from '@/components/empty/empty';
import { Button } from '@/components/ui/button';
import { RAGFlowPagination } from '@/components/ui/ragflow-pagination';
import { Switch } from '@/components/ui/switch';
import {
Table,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
} from '@/components/ui/table';
import { Pagination } from '@/interfaces/common';
import { t } from 'i18next';
import { pick } from 'lodash';
import { Eraser, TextSelect } from 'lucide-react';
import { useMemo } from 'react';
import { Copy, Eraser, TextSelect } from 'lucide-react';
import * as React from 'react';
import { useMemo, useState } from 'react';
import { CopyToClipboard } from 'react-copy-to-clipboard';
import { useMessageAction } from './hook';
import { IMessageInfo } from './interface';
export type MemoryTableProps = {
@ -51,13 +58,27 @@ export function MemoryTable({
);
const [columnVisibility, setColumnVisibility] =
React.useState<VisibilityState>({});
const [copied, setCopied] = useState(false);
const {
showDeleteDialog,
setShowDeleteDialog,
handleClickDeleteMessage,
selectedMessage,
handleDeleteMessage,
fetchMessageContent,
selectedMessageContent,
showMessageContentDialog,
setShowMessageContentDialog,
handleClickMessageContentDialog,
} = useMessageAction();
// Define columns for the memory table
const columns: ColumnDef<IMessageInfo>[] = useMemo(
() => [
{
accessorKey: 'session_id',
header: () => <span>{t('memoryDetail.messages.sessionId')}</span>,
header: () => <span>{t('memory.messages.sessionId')}</span>,
cell: ({ row }) => (
<div className="text-sm font-medium ">
{row.getValue('session_id')}
@ -66,7 +87,7 @@ export function MemoryTable({
},
{
accessorKey: 'agent_name',
header: () => <span>{t('memoryDetail.messages.agent')}</span>,
header: () => <span>{t('memory.messages.agent')}</span>,
cell: ({ row }) => (
<div className="text-sm font-medium ">
{row.getValue('agent_name')}
@ -75,7 +96,7 @@ export function MemoryTable({
},
{
accessorKey: 'message_type',
header: () => <span>{t('memoryDetail.messages.type')}</span>,
header: () => <span>{t('memory.messages.type')}</span>,
cell: ({ row }) => (
<div className="text-sm font-medium capitalize">
{row.getValue('message_type')}
@ -84,28 +105,28 @@ export function MemoryTable({
},
{
accessorKey: 'valid_at',
header: () => <span>{t('memoryDetail.messages.validDate')}</span>,
header: () => <span>{t('memory.messages.validDate')}</span>,
cell: ({ row }) => (
<div className="text-sm ">{row.getValue('valid_at')}</div>
),
},
{
accessorKey: 'forget_at',
header: () => <span>{t('memoryDetail.messages.forgetAt')}</span>,
header: () => <span>{t('memory.messages.forgetAt')}</span>,
cell: ({ row }) => (
<div className="text-sm ">{row.getValue('forget_at')}</div>
),
},
{
accessorKey: 'source_id',
header: () => <span>{t('memoryDetail.messages.source')}</span>,
header: () => <span>{t('memory.messages.source')}</span>,
cell: ({ row }) => (
<div className="text-sm ">{row.getValue('source_id')}</div>
),
},
{
accessorKey: 'status',
header: () => <span>{t('memoryDetail.messages.enable')}</span>,
header: () => <span>{t('memory.messages.enable')}</span>,
cell: ({ row }) => {
const isEnabled = row.getValue('status') as boolean;
return (
@ -117,19 +138,28 @@ export function MemoryTable({
},
{
accessorKey: 'action',
header: () => <span>{t('memoryDetail.messages.action')}</span>,
header: () => <span>{t('memory.messages.action')}</span>,
meta: {
cellClassName: 'w-12',
},
cell: () => (
cell: ({ row }) => (
<div className=" flex opacity-0 group-hover:opacity-100">
<Button variant={'ghost'} className="bg-transparent">
<Button
variant={'ghost'}
className="bg-transparent"
onClick={() => {
handleClickMessageContentDialog(row.original);
}}
>
<TextSelect />
</Button>
<Button
variant={'delete'}
className="bg-transparent"
aria-label="Edit"
onClick={() => {
handleClickDeleteMessage(row.original);
}}
>
<Eraser />
</Button>
@ -137,7 +167,7 @@ export function MemoryTable({
),
},
],
[],
[handleClickDeleteMessage],
);
const currentPagination = useMemo(() => {
@ -210,6 +240,85 @@ export function MemoryTable({
)}
</TableBody>
</Table>
{showDeleteDialog && (
<ConfirmDeleteDialog
onOk={handleDeleteMessage}
title={t('memory.messages.forgetMessage')}
open={showDeleteDialog}
onOpenChange={setShowDeleteDialog}
content={{
node: (
<ConfirmDeleteDialogNode
// avatar={{ avatar: selectedMessage.avatar, name: selectedMessage.name }}
name={
t('memory.messages.sessionId') +
': ' +
selectedMessage.session_id
}
warnText={t('memory.messages.delMessageWarn')}
/>
),
}}
/>
)}
{showMessageContentDialog && (
<Modal
title={t('memory.messages.content')}
open={showMessageContentDialog}
onOpenChange={setShowMessageContentDialog}
className="!w-[640px]"
footer={
<div className="flex justify-end gap-2">
<button
type="button"
onClick={() => setShowMessageContentDialog(false)}
className={
'px-2 py-1 border border-border-button rounded-md hover:bg-bg-card hover:text-text-primary '
}
>
{t('common.close')}
</button>
</div>
}
>
<div className="flex flex-col gap-2.5">
<div className="text-text-secondary text-sm">
{t('memory.messages.sessionId')}:&nbsp;&nbsp;
{selectedMessage.session_id}
</div>
{selectedMessageContent?.content && (
<div className="w-full bg-accent-primary-5 whitespace-pre-line text-wrap rounded-lg h-fit max-h-[350px] overflow-y-auto scrollbar-auto px-2.5 py-1">
{replaceText(selectedMessageContent?.content || '')}
</div>
)}
{selectedMessageContent?.content_embed && (
<div className="flex gap-2 items-center">
<CopyToClipboard
text={selectedMessageContent?.content_embed}
onCopy={() => {
setCopied(true);
setTimeout(() => setCopied(false), 1000);
}}
>
<Button
variant={'ghost'}
className="border border-border-button "
>
{t('memory.messages.contentEmbed')}
<Copy />
</Button>
</CopyToClipboard>
{copied && (
<span className="text-xs text-text-secondary">
{t('memory.messages.copied')}
</span>
)}
</div>
)}
</div>
</Modal>
)}
<div className="flex items-center justify-end py-4 absolute bottom-3 right-3">
<RAGFlowPagination

View file

@ -0,0 +1,159 @@
import { FormFieldType, RenderField } from '@/components/dynamic-form';
import { SingleFormSlider } from '@/components/ui/dual-range-slider';
import { NumberInput } from '@/components/ui/input';
import { Label } from '@/components/ui/label';
import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group';
import { cn } from '@/lib/utils';
import { t } from 'i18next';
import { ListChevronsDownUp, ListChevronsUpDown } from 'lucide-react';
import { useState } from 'react';
import { z } from 'zod';
export const advancedSettingsFormSchema = {
permission: z.string().optional(),
storage_type: z.enum(['table', 'graph']).optional(),
forget_policy: z.enum(['lru', 'fifo']).optional(),
temperature: z.number().optional(),
system_prompt: z.string().optional(),
user_prompt: z.string().optional(),
};
export const defaultAdvancedSettingsForm = {
permission: 'me',
storage_type: 'table',
forget_policy: 'fifo',
temperature: 0.7,
system_prompt: '',
user_prompt: '',
};
export const AdvancedSettingsForm = () => {
const [showAdvancedSettings, setShowAdvancedSettings] = useState(false);
return (
<>
<div
className="flex items-center gap-1 w-full cursor-pointer"
onClick={() => setShowAdvancedSettings(!showAdvancedSettings)}
>
{showAdvancedSettings ? (
<ListChevronsDownUp size={14} />
) : (
<ListChevronsUpDown size={14} />
)}
{t('memory.config.advancedSettings')}
</div>
{/* {showAdvancedSettings && ( */}
<>
<RenderField
field={{
name: 'permission',
label: t('memory.config.permission'),
required: false,
horizontal: true,
// hideLabel: true,
type: FormFieldType.Custom,
render: (field) => (
<RadioGroup
defaultValue="me"
className="flex"
{...field}
onValueChange={(value) => {
console.log(value);
field.onChange(value);
}}
>
<div className="flex items-center gap-3">
<RadioGroupItem value="me" id="r1" />
<Label htmlFor="r1">{t('memory.config.onlyMe')}</Label>
</div>
<div className="flex items-center gap-3">
<RadioGroupItem value="team" id="r2" />
<Label htmlFor="r2">{t('memory.config.team')}</Label>
</div>
</RadioGroup>
),
}}
/>
<RenderField
field={{
name: 'storage_type',
label: t('memory.config.storageType'),
type: FormFieldType.Select,
horizontal: true,
placeholder: t('memory.config.storageTypePlaceholder'),
options: [
{ label: 'table', value: 'table' },
// { label: 'graph', value: 'graph' },
],
required: false,
}}
/>
<RenderField
field={{
name: 'forget_policy',
label: t('memory.config.forgetPolicy'),
type: FormFieldType.Select,
horizontal: true,
// placeholder: t('memory.config.storageTypePlaceholder'),
options: [
{ label: 'lru', value: 'lru' },
{ label: 'fifo', value: 'fifo' },
],
required: false,
}}
/>
<RenderField
field={{
name: 'temperature',
label: t('memory.config.temperature'),
type: FormFieldType.Custom,
horizontal: true,
required: false,
render: (field) => (
<div className="flex gap-2 items-center">
<SingleFormSlider
{...field}
onChange={(value: number) => {
field.onChange(value);
}}
max={1}
step={0.01}
min={0}
disabled={false}
></SingleFormSlider>
<NumberInput
className={cn(
'h-6 w-10 p-1 border border-border-button rounded-sm',
)}
max={1}
step={0.01}
min={0}
{...field}
></NumberInput>
</div>
),
}}
/>
<RenderField
field={{
name: 'system_prompt',
label: t('memory.config.systemPrompt'),
type: FormFieldType.Textarea,
horizontal: true,
placeholder: t('memory.config.systemPromptPlaceholder'),
required: false,
}}
/>
<RenderField
field={{
name: 'user_prompt',
label: t('memory.config.userPrompt'),
type: FormFieldType.Text,
horizontal: true,
placeholder: t('memory.config.userPromptPlaceholder'),
required: false,
}}
/>
</>
{/* )} */}
</>
);
};

View file

@ -0,0 +1,53 @@
import { AvatarUpload } from '@/components/avatar-upload';
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { Input } from '@/components/ui/input';
import { t } from 'i18next';
import { z } from 'zod';
export const basicInfoSchema = {
name: z.string().min(1, { message: t('setting.nameRequired') }),
avatar: z.string().optional(),
description: z.string().optional(),
};
export const defaultBasicInfo = { name: '', avatar: '', description: '' };
export const BasicInfo = () => {
return (
<>
<RAGFlowFormItem
name={'name'}
label={t('memories.name')}
required={true}
horizontal={true}
// tooltip={field.tooltip}
// labelClassName={labelClassName || field.labelClassName}
>
{(field) => {
return <Input {...field}></Input>;
}}
</RAGFlowFormItem>
<RAGFlowFormItem
name={'avatar'}
label={t('memory.config.avatar')}
required={false}
horizontal={true}
// tooltip={field.tooltip}
// labelClassName={labelClassName || field.labelClassName}
>
{(field) => {
return <AvatarUpload {...field}></AvatarUpload>;
}}
</RAGFlowFormItem>
<RAGFlowFormItem
name={'description'}
label={t('memory.config.description')}
required={false}
horizontal={true}
// tooltip={field.tooltip}
// labelClassName={labelClassName || field.labelClassName}
>
{(field) => {
return <Input {...field}></Input>;
}}
</RAGFlowFormItem>
</>
);
};

View file

@ -0,0 +1,42 @@
import { useUpdateMemory } from '@/pages/memories/hooks';
import { IMemory, IMemoryAppDetailProps } from '@/pages/memories/interface';
import { omit } from 'lodash';
import { useCallback, useState } from 'react';
export const useUpdateMemoryConfig = () => {
const { updateMemory } = useUpdateMemory();
const [loading, setLoading] = useState(false);
const onMemoryRenameOk = useCallback(
async (data: IMemory) => {
let res;
setLoading(true);
if (data?.id) {
// console.log('memory-->', memory, data);
try {
const params = omit(data, [
'id',
'memory_type',
'embd_id',
'storage_type',
]);
res = await updateMemory({
// ...memoryDataTemp,
// data: data,
id: data.id,
...params,
} as unknown as IMemoryAppDetailProps);
// if (res && res.data.code === 0) {
// message.success(t('message.update_success'));
// } else {
// message.error(t('message.update_fail'));
// }
} catch (e) {
console.error('error', e);
}
}
setLoading(false);
},
[updateMemory],
);
return { onMemoryRenameOk, loading };
};

View file

@ -1,13 +1,110 @@
import { DynamicForm } from '@/components/dynamic-form';
import { Button } from '@/components/ui/button';
import Divider from '@/components/ui/divider';
import { Form } from '@/components/ui/form';
import { MainContainer } from '@/pages/dataset/dataset-setting/configuration-form-container';
import { TopTitle } from '@/pages/dataset/dataset-title';
import { IMemory } from '@/pages/memories/interface';
import { zodResolver } from '@hookform/resolvers/zod';
import { t } from 'i18next';
import { useEffect } from 'react';
import { useForm } from 'react-hook-form';
import { z } from 'zod';
import { useFetchMemoryBaseConfiguration } from '../hooks/use-memory-setting';
import {
AdvancedSettingsForm,
advancedSettingsFormSchema,
defaultAdvancedSettingsForm,
} from './advanced-settings-form';
import { BasicInfo, basicInfoSchema, defaultBasicInfo } from './basic-form';
import { useUpdateMemoryConfig } from './hook';
import {
MemoryModelForm,
defaultMemoryModelForm,
memoryModelFormSchema,
} from './memory-model-form';
const MemoryMessageSchema = z.object({
id: z.string(),
...basicInfoSchema,
...memoryModelFormSchema,
...advancedSettingsFormSchema,
});
// type MemoryMessageForm = z.infer<typeof MemoryMessageSchema>;
export default function MemoryMessage() {
const form = useForm<IMemory>({
resolver: zodResolver(MemoryMessageSchema),
defaultValues: {
id: '',
...defaultBasicInfo,
...defaultMemoryModelForm,
...defaultAdvancedSettingsForm,
} as unknown as IMemory,
});
const { data } = useFetchMemoryBaseConfiguration();
const { onMemoryRenameOk, loading } = useUpdateMemoryConfig();
useEffect(() => {
form.reset({
id: data?.id,
embd_id: data?.embd_id,
llm_id: data?.llm_id,
name: data?.name || '',
description: data?.description || '',
avatar: data?.avatar || '',
memory_size: data?.memory_size,
memory_type: data?.memory_type,
temperature: data?.temperature,
system_prompt: data?.system_prompt || '',
user_prompt: data?.user_prompt || '',
forgetting_policy: data?.forgetting_policy || 'fifo',
permissions: data?.permissions || 'me',
});
}, [data, form]);
const onSubmit = (data: IMemory) => {
onMemoryRenameOk(data);
};
return (
<div className="flex flex-col gap-2">
<div className="flex items-center gap-2">
<div className="h-4 w-4 rounded-full bg-text-secondary">11</div>
<div className="h-4 w-4 rounded-full bg-text-secondary">11</div>
<section className="h-full flex flex-col">
<TopTitle
title={t('knowledgeDetails.configuration')}
description={t('knowledgeConfiguration.titleDescription')}
></TopTitle>
<div className="flex gap-14 flex-1 min-h-0">
<Form {...form}>
<form onSubmit={form.handleSubmit(() => {})} className="space-y-6 ">
<div className="w-[768px] h-[calc(100vh-300px)] pr-1 overflow-y-auto scrollbar-auto">
<MainContainer className="text-text-secondary !space-y-10">
<div className="text-base font-medium text-text-primary">
{t('knowledgeConfiguration.baseInfo')}
</div>
<BasicInfo></BasicInfo>
<Divider />
<MemoryModelForm />
<AdvancedSettingsForm />
</MainContainer>
</div>
<div className="text-right items-center flex justify-end gap-3 w-[768px]">
<Button
type="reset"
className="bg-transparent text-color-white hover:bg-transparent border-border-button border"
onClick={() => {
form.reset();
}}
>
{t('knowledgeConfiguration.cancel')}
</Button>
<DynamicForm.SavingButton
submitLoading={loading}
submitFunc={(value) => {
console.log('form-value', value);
onSubmit(value as IMemory);
}}
></DynamicForm.SavingButton>
</div>
</form>
</Form>
</div>
<div className="flex items-center gap-2">
<div className="h-4 w-4 rounded-full bg-text ">setting</div>
</div>
</div>
</section>
);
}

View file

@ -0,0 +1,74 @@
import { FormFieldType, RenderField } from '@/components/dynamic-form';
import { useModelOptions } from '@/components/llm-setting-items/llm-form-field';
import { EmbeddingSelect } from '@/pages/dataset/dataset-setting/configuration/common-item';
import { t } from 'i18next';
import { z } from 'zod';
export const memoryModelFormSchema = {
embd_id: z.string(),
llm_id: z.string(),
memory_type: z.array(z.string()).optional(),
memory_size: z.number().optional(),
};
export const defaultMemoryModelForm = {
embd_id: '',
llm_id: '',
memory_type: [],
memory_size: 0,
};
export const MemoryModelForm = () => {
const { modelOptions } = useModelOptions();
return (
<>
<RenderField
field={{
name: 'embd_id',
label: t('memories.embeddingModel'),
placeholder: t('memories.selectModel'),
required: true,
horizontal: true,
// hideLabel: true,
type: FormFieldType.Custom,
render: (field) => <EmbeddingSelect field={field} isEdit={false} />,
}}
/>
<RenderField
field={{
name: 'llm_id',
label: t('memories.llm'),
placeholder: t('memories.selectModel'),
required: true,
horizontal: true,
type: FormFieldType.Select,
options: modelOptions as { value: string; label: string }[],
}}
/>
<RenderField
field={{
name: 'memory_type',
label: t('memories.memoryType'),
type: FormFieldType.MultiSelect,
horizontal: true,
placeholder: t('memories.memoryTypePlaceholder'),
options: [
{ label: 'Raw', value: 'raw' },
{ label: 'Semantic', value: 'semantic' },
{ label: 'Episodic', value: 'episodic' },
{ label: 'Procedural', value: 'procedural' },
],
required: true,
}}
/>
<RenderField
field={{
name: 'memory_size',
label: t('memory.config.memorySize'),
type: FormFieldType.Number,
horizontal: true,
// placeholder: t('memory.config.memorySizePlaceholder'),
required: false,
}}
/>
</>
);
};

View file

@ -1,36 +1,31 @@
import { RAGFlowAvatar } from '@/components/ragflow-avatar';
import { Button } from '@/components/ui/button';
import { useSecondPathName } from '@/hooks/route-hook';
import { cn, formatBytes } from '@/lib/utils';
import { cn } from '@/lib/utils';
import { Routes } from '@/routes';
import { formatPureDate } from '@/utils/date';
import { Banknote, Logs } from 'lucide-react';
import { useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { useFetchMemoryBaseConfiguration } from '../hooks/use-memory-setting';
import { useHandleMenuClick } from './hooks';
type PropType = {
refreshCount?: number;
};
export function SideBar({ refreshCount }: PropType) {
export function SideBar() {
const pathName = useSecondPathName();
const { handleMenuClick } = useHandleMenuClick();
// refreshCount: be for avatar img sync update on top left
const { data } = useFetchMemoryBaseConfiguration({ refreshCount });
const { data } = useFetchMemoryBaseConfiguration();
const { t } = useTranslation();
const items = useMemo(() => {
const list = [
{
icon: <Logs className="size-4" />,
label: t(`knowledgeDetails.overview`),
label: t(`memory.sideBar.messages`),
key: Routes.MemoryMessage,
},
{
icon: <Banknote className="size-4" />,
label: t(`knowledgeDetails.configuration`),
label: t(`memory.sideBar.configuration`),
key: Routes.MemorySetting,
},
];
@ -49,15 +44,15 @@ export function SideBar({ refreshCount }: PropType) {
<h3 className="text-lg font-semibold line-clamp-1 text-text-primary text-ellipsis overflow-hidden">
{data.name}
</h3>
<div className="flex justify-between">
{/* <div className="flex justify-between">
<span>
{data.doc_num} {t('knowledgeDetails.files')}
</span>
<span>{formatBytes(data.size)}</span>
</div>
<div>
{t('knowledgeDetails.created')} {formatPureDate(data.create_time)}
</div>
{t('knowledgeDetails.created')} {formatPureDate(data.)}
</div> */}
</div>
</div>

View file

@ -8,6 +8,9 @@ const {
deleteMemory,
getMemoryDetail,
updateMemorySetting,
getMemoryConfig,
deleteMemoryMessage,
getMessageContent,
// getMemoryDetailShare,
} = api;
const methods = {
@ -17,27 +20,21 @@ const methods = {
},
getMemoryList: {
url: getMemoryList,
method: 'post',
method: 'get',
},
deleteMemory: { url: deleteMemory, method: 'post' },
// getMemoryDetail: {
// url: getMemoryDetail,
// method: 'get',
// },
// updateMemorySetting: {
// url: updateMemorySetting,
// method: 'post',
// },
// getMemoryDetailShare: {
// url: getMemoryDetailShare,
// method: 'get',
// },
deleteMemory: { url: deleteMemory, method: 'delete' },
getMemoryConfig: {
url: getMemoryConfig,
method: 'get',
},
deleteMemoryMessage: { url: deleteMemoryMessage, method: 'delete' },
getMessageContent: { url: getMessageContent, method: 'get' },
} as const;
const memoryService = registerNextServer<keyof typeof methods>(methods);
export const updateMemoryById = (id: string, data: any) => {
return request.post(updateMemorySetting(id), { data });
return request.put(updateMemorySetting(id), { data });
};
export const getMemoryDetailById = (id: string, data: any) => {
return request.post(getMemoryDetail(id), { data });
return request.get(getMemoryDetail(id), { params: data });
};
export default memoryService;

View file

@ -227,11 +227,15 @@ export default {
retrievalTestShare: `${ExternalApi}${api_host}/searchbots/retrieval_test`,
// memory
createMemory: `${api_host}/memory/create`,
getMemoryList: `${api_host}/memory/list`,
createMemory: `${api_host}/memories`,
getMemoryList: `${api_host}/memories`,
getMemoryConfig: (id: string) => `${api_host}/memories/${id}/config`,
deleteMemory: (id: string) => `${api_host}/memory/rm/${id}`,
getMemoryDetail: (id: string) => `${api_host}/memory/detail/${id}`,
updateMemorySetting: (id: string) => `${api_host}/memory/update/${id}`,
getMemoryDetail: (id: string) => `${api_host}/memories/${id}`,
updateMemorySetting: (id: string) => `${api_host}/memories/${id}`,
deleteMemoryMessage: (id: string) => `${api_host}/message/rm/${id}`,
getMessageContent: (message_id: string) =>
`${api_host}/messages/${message_id}/content`,
// data pipeline
fetchDataflow: (id: string) => `${api_host}/dataflow/get/${id}`,