Merge pull request #2280 from danielaskdd/fix-exit-handler

Refact: Graceful shutdown and signal handling in Gunicorn Mode

lightrag-api

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-source /home/netman/lightrag-xyj/venv/bin/activate
-lightrag-server

lightrag.service.example

@@ -1,5 +1,5 @@
 [Unit]
-Description=LightRAG XYJ Ollama Service
+Description=LightRAG XYJ Service
 After=network.target
 
 [Service]
@@ -8,10 +8,20 @@ User=netman
 # Memory settings
 MemoryHigh=8G
 MemoryMax=12G
+
+# Using virtual enviroment created by miniconda
+Environment="PATH=/home/netman/miniconda3/bin:/home/netman/lightrag-xyj/venv/bin"
 WorkingDirectory=/home/netman/lightrag-xyj
-ExecStart=/home/netman/lightrag-xyj/lightrag-api
+# ExecStart=/home/netman/lightrag-xyj/venv/bin/lightrag-server
+ExecStart=/home/netman/lightrag-xyj/venv/bin/lightrag-gunicorn
+
+# Kill mode require ExecStart must be gunicorn or unvicorn main process
+KillMode=process
+ExecStop=/bin/kill -s TERM $MAINPID
+TimeoutStopSec=60
+
 Restart=always
-RestartSec=10
+RestartSec=30
 
 [Install]
 WantedBy=multi-user.target

lightrag/api/README-zh.md

@@ -184,24 +184,16 @@ MAX_ASYNC=4
 
 ### 将 Lightrag 安装为 Linux 服务
 
-从示例文件 `lightrag.service.example` 创建您的服务文件 `lightrag.service`。修改服务文件中的 WorkingDirectory 和 ExecStart：
+从示例文件 `lightrag.service.example` 创建您的服务文件 `lightrag.service`。修改服务文件中的服务启动定义：
 
 ```text
-Description=LightRAG Ollama Service
+# Set Enviroment to your Python virtual enviroment
-WorkingDirectory=<lightrag 安装目录>
+Environment="PATH=/home/netman/lightrag-xyj/venv/bin"
-ExecStart=<lightrag 安装目录>/lightrag/api/lightrag-api
+WorkingDirectory=/home/netman/lightrag-xyj
-```
+# ExecStart=/home/netman/lightrag-xyj/venv/bin/lightrag-server
-
+ExecStart=/home/netman/lightrag-xyj/venv/bin/lightrag-gunicorn
-修改您的服务启动脚本：`lightrag-api`。根据需要更改 python 虚拟环境激活命令：
-
-```shell
-#!/bin/bash
-
-# 您的 python 虚拟环境激活命令
-source /home/netman/lightrag-xyj/venv/bin/activate
-# 启动 lightrag api 服务器
-lightrag-server
 ```
+> ExecStart命令必须是 lightrag-gunicorn 或 lightrag-server 中的一个，不能使用其它脚本包裹它们。因为停止服务必须要求主进程必须是这两个进程。
 
 安装 LightRAG 服务。如果您的系统是 Ubuntu，以下命令将生效：
 

lightrag/api/README.md

@@ -188,24 +188,18 @@ MAX_ASYNC=4
 
 ### Install LightRAG as a Linux Service
 
-Create your service file `lightrag.service` from the sample file: `lightrag.service.example`. Modify the `WorkingDirectory` and `ExecStart` in the service file:
+Create your service file `lightrag.service` from the sample file: `lightrag.service.example`. Modify the start options the service file:
 
 ```text
-Description=LightRAG Ollama Service
+# Set Enviroment to your Python virtual enviroment
-WorkingDirectory=<lightrag installed directory>
+Environment="PATH=/home/netman/lightrag-xyj/venv/bin"
-ExecStart=<lightrag installed directory>/lightrag/api/lightrag-api
+WorkingDirectory=/home/netman/lightrag-xyj
+# ExecStart=/home/netman/lightrag-xyj/venv/bin/lightrag-server
+ExecStart=/home/netman/lightrag-xyj/venv/bin/lightrag-gunicorn
+
 ```
 
-Modify your service startup script: `lightrag-api`. Change your Python virtual environment activation command as needed:
+> The ExecStart command must be either `lightrag-gunicorn` or `lightrag-server`; no wrapper scripts are allowed. This is because service termination requires the main process to be one of these two executables.
-
-```shell
-#!/bin/bash
-
-# your python virtual environment activation
-source /home/netman/lightrag-xyj/venv/bin/activate
-# start lightrag api server
-lightrag-server
-```
 
 Install LightRAG service. If your system is Ubuntu, the following commands will work:
 

lightrag/api/gunicorn_config.py

@@ -129,11 +129,13 @@ def on_exit(server):
     print("=" * 80)
     print("GUNICORN MASTER PROCESS: Shutting down")
     print(f"Process ID: {os.getpid()}")
-    print("=" * 80)
 
-    # Release shared resources
+    print("Finalizing shared storage...")
     finalize_share_data()
 
+    print("Gunicorn shutdown complete")
+    print("=" * 80)
+
     print("=" * 80)
     print("Gunicorn shutdown complete")
     print("=" * 80)

lightrag/api/lightrag_server.py

@@ -12,7 +12,6 @@ from fastapi.openapi.docs import (
 import os
 import logging
 import logging.config
-import signal
 import sys
 import uvicorn
 import pipmaster as pm
@@ -82,24 +81,6 @@ config.read("config.ini")
 auth_configured = bool(auth_handler.accounts)
 
 
-def setup_signal_handlers():
-    """Setup signal handlers for graceful shutdown"""
-
-    def signal_handler(sig, frame):
-        print(f"\n\nReceived signal {sig}, shutting down gracefully...")
-        print(f"Process ID: {os.getpid()}")
-
-        # Release shared resources
-        finalize_share_data()
-
-        # Exit with success status
-        sys.exit(0)
-
-    # Register signal handlers
-    signal.signal(signal.SIGINT, signal_handler)  # Ctrl+C
-    signal.signal(signal.SIGTERM, signal_handler)  # kill command
-
-
 class LLMConfigCache:
     """Smart LLM and Embedding configuration cache class"""
 
@@ -345,8 +326,15 @@ def create_app(args):
             # Clean up database connections
             await rag.finalize_storages()
 
-            # Clean up shared data
+            if "LIGHTRAG_GUNICORN_MODE" not in os.environ:
-            finalize_share_data()
+                # Only perform cleanup in Uvicorn single-process mode
+                logger.debug("Unvicorn Mode: finalizing shared storage...")
+                finalize_share_data()
+            else:
+                # In Gunicorn mode with preload_app=True, cleanup is handled by on_exit hooks
+                logger.debug(
+                    "Gunicorn Mode: postpone shared storage finalization to master process"
+                )
 
     # Initialize FastAPI
     base_description = (
@@ -1108,8 +1096,10 @@ def main():
     update_uvicorn_mode_config()
     display_splash_screen(global_args)
 
-    # Setup signal handlers for graceful shutdown
+    # Note: Signal handlers are NOT registered here because:
-    setup_signal_handlers()
+    # - Uvicorn has built-in signal handling that properly calls lifespan shutdown
+    # - Custom signal handlers can interfere with uvicorn's graceful shutdown
+    # - Cleanup is handled by the lifespan context manager's finally block
 
     # Create application instance directly instead of using factory function
     app = create_app(global_args)

lightrag/api/run_with_gunicorn.py

@@ -5,12 +5,11 @@ Start LightRAG server with Gunicorn
 
 import os
 import sys
-import signal
 import pipmaster as pm
 from lightrag.api.utils_api import display_splash_screen, check_env_file
 from lightrag.api.config import global_args
 from lightrag.utils import get_env_value
-from lightrag.kg.shared_storage import initialize_share_data, finalize_share_data
+from lightrag.kg.shared_storage import initialize_share_data
 
 from lightrag.constants import (
     DEFAULT_WOKERS,
@@ -34,21 +33,10 @@ def check_and_install_dependencies():
             print(f"{package} installed successfully")
 
 
-# Signal handler for graceful shutdown
-def signal_handler(sig, frame):
-    print("\n\n" + "=" * 80)
-    print("RECEIVED TERMINATION SIGNAL")
-    print(f"Process ID: {os.getpid()}")
-    print("=" * 80 + "\n")
-
-    # Release shared resources
-    finalize_share_data()
-
-    # Exit with success status
-    sys.exit(0)
-
-
 def main():
+    # Set Gunicorn mode flag for lifespan cleanup detection
+    os.environ["LIGHTRAG_GUNICORN_MODE"] = "1"
+
     # Check .env file
     if not check_env_file():
         sys.exit(1)
@@ -56,9 +44,8 @@ def main():
     # Check and install dependencies
     check_and_install_dependencies()
 
-    # Register signal handlers for graceful shutdown
+    # Note: Signal handlers are NOT registered here because:
-    signal.signal(signal.SIGINT, signal_handler)  # Ctrl+C
+    # - Master cleanup already handled by gunicorn_config.on_exit()
-    signal.signal(signal.SIGTERM, signal_handler)  # kill command
 
     # Display startup information
     display_splash_screen(global_args)

lightrag/kg/shared_storage.py

@@ -10,6 +10,8 @@ from typing import Any, Dict, List, Optional, Union, TypeVar, Generic
 
 from lightrag.exceptions import PipelineNotInitializedError
 
+DEBUG_LOCKS = False
+
 
 # Define a direct print function for critical logs that must be visible in all processes
 def direct_log(message, enable_output: bool = True, level: str = "DEBUG"):
@@ -90,7 +92,6 @@ _storage_keyed_lock: Optional["KeyedUnifiedLock"] = None
 # async locks for coroutine synchronization in multiprocess mode
 _async_locks: Optional[Dict[str, asyncio.Lock]] = None
 
-DEBUG_LOCKS = False
 _debug_n_locks_acquired: int = 0