diff --git a/lightrag/api/gunicorn_config.py b/lightrag/api/gunicorn_config.py index 111e7e26..aa6886f1 100644 --- a/lightrag/api/gunicorn_config.py +++ b/lightrag/api/gunicorn_config.py @@ -163,3 +163,24 @@ def post_fork(server, worker): uvicorn_error_logger.handlers = [] uvicorn_error_logger.setLevel(logging.CRITICAL) uvicorn_error_logger.propagate = False + + +def worker_exit(server, worker): + """ + Executed when a worker is about to exit. + + This is called for each worker process when it exits. We should only + clean up worker-local resources here, NOT the shared Manager. + The Manager should only be shut down by the main process in on_exit(). + """ + print("=" * 80) + print(f"GUNICORN WORKER PROCESS: Shutting down worker {worker.pid}") + print(f"Process ID: {os.getpid()}") + print("=" * 80) + + # Clean up worker-local resources without shutting down the Manager + # Pass shutdown_manager=False to prevent Manager shutdown + finalize_share_data(shutdown_manager=False) + + print(f"Worker {worker.pid} cleanup complete") + print("=" * 80) diff --git a/lightrag/api/run_with_gunicorn.py b/lightrag/api/run_with_gunicorn.py index 5460daa3..c5f7cb5c 100644 --- a/lightrag/api/run_with_gunicorn.py +++ b/lightrag/api/run_with_gunicorn.py @@ -5,12 +5,11 @@ Start LightRAG server with Gunicorn import os import sys -import signal import pipmaster as pm from lightrag.api.utils_api import display_splash_screen, check_env_file from lightrag.api.config import global_args from lightrag.utils import get_env_value -from lightrag.kg.shared_storage import initialize_share_data, finalize_share_data +from lightrag.kg.shared_storage import initialize_share_data from lightrag.constants import ( DEFAULT_WOKERS, @@ -34,20 +33,6 @@ def check_and_install_dependencies(): print(f"{package} installed successfully") -# Signal handler for graceful shutdown -def signal_handler(sig, frame): - print("\n\n" + "=" * 80) - print("RECEIVED TERMINATION SIGNAL") - print(f"Process ID: {os.getpid()}") - print("=" * 80 + "\n") - - # Release shared resources - finalize_share_data() - - # Exit with success status - sys.exit(0) - - def main(): # Set Gunicorn mode flag for lifespan cleanup detection os.environ["LIGHTRAG_GUNICORN_MODE"] = "1" @@ -59,9 +44,10 @@ def main(): # Check and install dependencies check_and_install_dependencies() - # Register signal handlers for graceful shutdown - signal.signal(signal.SIGINT, signal_handler) # Ctrl+C - signal.signal(signal.SIGTERM, signal_handler) # kill command + # Note: Signal handlers are NOT registered here because: + # - Worker cleanup is handled by gunicorn_config.worker_exit() + # - Master cleanup is handled by gunicorn_config.on_exit() + # This prevents race conditions when multiple processes try to finalize shared data # Display startup information display_splash_screen(global_args) diff --git a/lightrag/kg/shared_storage.py b/lightrag/kg/shared_storage.py index ef0f61e2..3d47eab8 100644 --- a/lightrag/kg/shared_storage.py +++ b/lightrag/kg/shared_storage.py @@ -1565,7 +1565,7 @@ def get_namespace_lock( return NamespaceLock(namespace, workspace, enable_logging) -def finalize_share_data(): +def finalize_share_data(shutdown_manager: bool = True): """ Release shared resources and clean up. @@ -1574,6 +1574,10 @@ def finalize_share_data(): In multi-process mode, it shuts down the Manager and releases all shared objects. In single-process mode, it simply resets the global variables. + + Args: + shutdown_manager: If True, shut down the multiprocessing Manager. + Should be True only for the main process, False for worker processes. """ global \ _manager, \ @@ -1598,8 +1602,8 @@ def finalize_share_data(): f"Process {os.getpid()} finalizing storage data (multiprocess={_is_multiprocess})" ) - # In multi-process mode, shut down the Manager - if _is_multiprocess and _manager is not None: + # In multi-process mode, shut down the Manager only if requested + if _is_multiprocess and _manager is not None and shutdown_manager: try: # Clear shared resources before shutting down Manager if _shared_dicts is not None: