Add offline deployment support with cache management and layered deps
• Add tiktoken cache downloader CLI
• Add layered offline dependencies
• Add offline requirements files
• Add offline deployment guide
(cherry picked from commit a5c05f1b92)
This commit is contained in:
yangdx
Raphaël MANSUY
parent
e19a4be0af
commit
6bd5b2d95b
4 changed files with 68 additions and 110 deletions
116
pyproject.toml
116
pyproject.toml
|
|
@ -23,13 +23,13 @@ classifiers = [
|
|||
dependencies = [
|
||||
"aiohttp",
|
||||
"configparser",
|
||||
"google-api-core>=2.0.0,<3.0.0",
|
||||
"google-genai>=1.0.0,<2.0.0",
|
||||
"dotenv",
|
||||
"future",
|
||||
"json_repair",
|
||||
"nano-vectordb",
|
||||
"networkx",
|
||||
"numpy>=1.24.0,<2.0.0",
|
||||
"pandas>=2.0.0,<2.4.0",
|
||||
"numpy",
|
||||
"pandas>=2.0.0",
|
||||
"pipmaster",
|
||||
"pydantic",
|
||||
"pypinyin",
|
||||
|
|
@ -41,24 +41,18 @@ dependencies = [
|
|||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
# Test framework dependencies (for CI/CD and testing)
|
||||
pytest = [
|
||||
"pytest>=8.4.2",
|
||||
"pytest-asyncio>=1.2.0",
|
||||
"pre-commit",
|
||||
"ruff",
|
||||
]
|
||||
|
||||
api = [
|
||||
# Core dependencies
|
||||
"aiohttp",
|
||||
"configparser",
|
||||
"dotenv",
|
||||
"future",
|
||||
"json_repair",
|
||||
"nano-vectordb",
|
||||
"networkx",
|
||||
"numpy>=1.24.0,<2.0.0",
|
||||
"openai>=2.0.0,<3.0.0",
|
||||
"pandas>=2.0.0,<2.4.0",
|
||||
"numpy",
|
||||
"openai",
|
||||
"pandas>=2.0.0",
|
||||
"pipmaster",
|
||||
"pydantic",
|
||||
"pypinyin",
|
||||
|
|
@ -67,8 +61,6 @@ api = [
|
|||
"tenacity",
|
||||
"tiktoken",
|
||||
"xlsxwriter>=3.1.0",
|
||||
"google-api-core>=2.0.0,<3.0.0",
|
||||
"google-genai>=1.0.0,<2.0.0",
|
||||
# API-specific dependencies
|
||||
"aiofiles",
|
||||
"ascii_colors",
|
||||
|
|
@ -78,80 +70,57 @@ api = [
|
|||
"httpcore",
|
||||
"httpx",
|
||||
"jiter",
|
||||
"bcrypt>=4.0.0",
|
||||
"passlib[bcrypt]",
|
||||
"psutil",
|
||||
"PyJWT>=2.8.0,<3.0.0",
|
||||
"PyJWT",
|
||||
"python-jose[cryptography]",
|
||||
"python-multipart",
|
||||
"pytz",
|
||||
"uvicorn",
|
||||
"gunicorn",
|
||||
# Document processing dependencies (required for API document upload functionality)
|
||||
"openpyxl>=3.0.0,<4.0.0", # XLSX processing
|
||||
"pycryptodome>=3.0.0,<4.0.0", # PDF encryption support
|
||||
"pypdf>=6.1.0", # PDF processing
|
||||
"python-docx>=0.8.11,<2.0.0", # DOCX processing
|
||||
"python-pptx>=0.6.21,<2.0.0", # PPTX processing
|
||||
]
|
||||
|
||||
# Advanced document processing engine (optional)
|
||||
docling = [
|
||||
# On macOS, pytorch and frameworks use Objective-C are not fork-safe,
|
||||
# and not compatible to gunicorn multi-worker mode
|
||||
"docling>=2.0.0,<3.0.0; sys_platform != 'darwin'",
|
||||
]
|
||||
|
||||
# Offline deployment dependencies (layered design for flexibility)
|
||||
offline-docs = [
|
||||
# Document processing dependencies
|
||||
"docling>=1.0.0",
|
||||
"pypdf2>=3.0.0",
|
||||
"python-docx>=0.8.11",
|
||||
"python-pptx>=0.6.21",
|
||||
"openpyxl>=3.0.0",
|
||||
]
|
||||
|
||||
offline-storage = [
|
||||
# Storage backend dependencies
|
||||
"redis>=5.0.0,<8.0.0",
|
||||
"neo4j>=5.0.0,<7.0.0",
|
||||
"pymilvus>=2.6.2,<3.0.0",
|
||||
"pymongo>=4.0.0,<5.0.0",
|
||||
"asyncpg>=0.29.0,<1.0.0",
|
||||
"qdrant-client>=1.11.0,<2.0.0",
|
||||
"redis>=5.0.0",
|
||||
"neo4j>=5.0.0",
|
||||
"pymilvus==2.5.2",
|
||||
"pymongo>=4.0.0",
|
||||
"asyncpg>=0.29.0",
|
||||
"qdrant-client>=1.7.0",
|
||||
]
|
||||
|
||||
offline-llm = [
|
||||
# LLM provider dependencies
|
||||
"openai>=2.0.0,<3.0.0",
|
||||
"anthropic>=0.18.0,<1.0.0",
|
||||
"ollama>=0.1.0,<1.0.0",
|
||||
"zhipuai>=2.0.0,<3.0.0",
|
||||
"aioboto3>=12.0.0,<16.0.0",
|
||||
"voyageai>=0.2.0,<1.0.0",
|
||||
"llama-index>=0.9.0,<1.0.0",
|
||||
"google-api-core>=2.0.0,<3.0.0",
|
||||
"google-genai>=1.0.0,<2.0.0",
|
||||
"openai>=1.0.0",
|
||||
"anthropic>=0.18.0",
|
||||
"ollama>=0.1.0",
|
||||
"zhipuai>=2.0.0",
|
||||
"aioboto3>=12.0.0",
|
||||
"voyageai>=0.2.0",
|
||||
"llama-index>=0.9.0",
|
||||
"transformers>=4.30.0",
|
||||
"torch>=2.0.0",
|
||||
]
|
||||
|
||||
offline = [
|
||||
# Complete offline package (includes api for document processing, plus storage and LLM)
|
||||
"lightrag-hku[api,offline-storage,offline-llm]",
|
||||
]
|
||||
|
||||
evaluation = [
|
||||
# Test framework dependencies
|
||||
"pytest>=8.4.2",
|
||||
"pytest-asyncio>=1.2.0",
|
||||
"pre-commit",
|
||||
"ruff",
|
||||
# RAG evaluation dependencies (RAGAS framework)
|
||||
"ragas>=0.3.7",
|
||||
"datasets>=4.3.0",
|
||||
"httpx>=0.28.1",
|
||||
]
|
||||
|
||||
observability = [
|
||||
# LLM observability and tracing dependencies
|
||||
"langfuse>=3.8.1",
|
||||
# Complete offline package (includes all offline dependencies)
|
||||
"lightrag-hku[offline-docs,offline-storage,offline-llm]",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
lightrag-server = "lightrag.api.lightrag_server:main"
|
||||
lightrag-gunicorn = "lightrag.api.run_with_gunicorn:main"
|
||||
lightrag-download-cache = "lightrag.tools.download_cache:main"
|
||||
lightrag-clean-llmqc = "lightrag.tools.clean_llm_query_cache:main"
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/HKUDS/LightRAG"
|
||||
|
|
@ -161,7 +130,6 @@ Repository = "https://github.com/HKUDS/LightRAG"
|
|||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["lightrag*"]
|
||||
exclude = ["data*", "tests*", "scripts*", "examples*", "dickens*", "reproduce*", "output_complete*", "rag_storage*", "inputs*"]
|
||||
|
||||
[tool.setuptools]
|
||||
include-package-data = true
|
||||
|
|
@ -170,15 +138,7 @@ include-package-data = true
|
|||
version = {attr = "lightrag.__version__"}
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
lightrag = ["api/webui/**/*", "api/static/**/*"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
python_classes = ["Test*"]
|
||||
python_functions = ["test_*"]
|
||||
lightrag = ["api/webui/**/*"]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py310"
|
||||
|
|
|
|||
|
|
@ -3,12 +3,10 @@
|
|||
# For offline installation:
|
||||
# pip download -r requirements-offline-docs.txt -d ./packages
|
||||
# pip install --no-index --find-links=./packages -r requirements-offline-docs.txt
|
||||
#
|
||||
# Recommended: Use pip install lightrag-hku[offline-docs] for the same effect
|
||||
# Or use constraints: pip install --constraint constraints-offline.txt -r requirements-offline-docs.txt
|
||||
|
||||
# Document processing dependencies (with version constraints matching pyproject.toml)
|
||||
openpyxl>=3.0.0,<4.0.0
|
||||
# Document processing dependencies
|
||||
docling>=1.0.0
|
||||
openpyxl>=3.0.0
|
||||
pypdf2>=3.0.0
|
||||
python-docx>=0.8.11,<2.0.0
|
||||
python-pptx>=0.6.21,<2.0.0
|
||||
python-docx>=0.8.11
|
||||
python-pptx>=0.6.21
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
asyncpg>=0.29.0
|
||||
neo4j>=5.0.0
|
||||
pymilvus>=2.6.2
|
||||
pymilvus==2.5.2
|
||||
pymongo>=4.0.0
|
||||
qdrant-client>=1.7.0
|
||||
# Storage backend dependencies
|
||||
|
|
|
|||
|
|
@ -4,30 +4,30 @@
|
|||
# pip download -r requirements-offline.txt -d ./packages
|
||||
# pip install --no-index --find-links=./packages -r requirements-offline.txt
|
||||
#
|
||||
# Recommended: Use pip install lightrag-hku[offline] for the same effect
|
||||
# Or use constraints: pip install --constraint constraints-offline.txt -r requirements-offline.txt
|
||||
# Or use pip install lightrag-hku[offline] for the same effect
|
||||
|
||||
# LLM provider dependencies (with version constraints matching pyproject.toml)
|
||||
aioboto3>=12.0.0,<16.0.0
|
||||
anthropic>=0.18.0,<1.0.0
|
||||
aioboto3>=12.0.0
|
||||
anthropic>=0.18.0
|
||||
asyncpg>=0.29.0
|
||||
# Document processing dependencies
|
||||
docling>=1.0.0
|
||||
llama-index>=0.9.0
|
||||
neo4j>=5.0.0
|
||||
ollama>=0.1.0
|
||||
|
||||
# LLM provider dependencies
|
||||
openai>=1.0.0
|
||||
openpyxl>=3.0.0
|
||||
pymilvus==2.5.2
|
||||
pymongo>=4.0.0
|
||||
pypdf2>=3.0.0
|
||||
python-docx>=0.8.11
|
||||
python-pptx>=0.6.21
|
||||
qdrant-client>=1.7.0
|
||||
|
||||
# Storage backend dependencies
|
||||
asyncpg>=0.29.0,<1.0.0
|
||||
|
||||
# Document processing dependencies
|
||||
llama-index>=0.9.0,<1.0.0
|
||||
neo4j>=5.0.0,<7.0.0
|
||||
ollama>=0.1.0,<1.0.0
|
||||
openai>=1.0.0,<2.0.0
|
||||
openpyxl>=3.0.0,<4.0.0
|
||||
pymilvus>=2.6.2,<3.0.0
|
||||
pymongo>=4.0.0,<5.0.0
|
||||
pypdf2>=3.0.0
|
||||
python-docx>=0.8.11,<2.0.0
|
||||
python-pptx>=0.6.21,<2.0.0
|
||||
qdrant-client>=1.7.0,<2.0.0
|
||||
redis>=5.0.0,<7.0.0
|
||||
torch>=2.0.0,<2.3.0
|
||||
transformers>=4.30.0,<5.0.0
|
||||
voyageai>=0.2.0,<1.0.0
|
||||
zhipuai>=2.0.0,<3.0.0
|
||||
redis>=5.0.0
|
||||
torch>=2.0.0
|
||||
transformers>=4.30.0
|
||||
voyageai>=0.2.0
|
||||
zhipuai>=2.0.0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue