48c7732edc
Implement automatic entity resolution to prevent duplicate nodes in the knowledge graph. The system uses a 3-layer approach: 1. Case-insensitive exact matching (free, instant) 2. Fuzzy string matching >85% threshold (free, instant) 3. Vector similarity + LLM verification (for acronyms/synonyms) Key features: - Pre-resolution phase prevents race conditions in parallel processing - Numeric suffix detection blocks false matches (IL-4 ≠ IL-13) - PostgreSQL alias cache for fast lookups on subsequent ingestion - Configurable thresholds via environment variables Bug fixes included: - Fix fuzzy matching false positives for numbered entities - Fix alias cache not being populated (missing db parameter) - Skip entity_aliases table from generic id index creation New files: - lightrag/entity_resolution/ - Core resolution module - tests/test_entity_resolution/ - Unit tests - docker/postgres-age-vector/ - Custom PG image with pgvector + AGE - docker-compose.test.yml - Integration test environment Configuration (env.example): - ENTITY_RESOLUTION_ENABLED=true - ENTITY_RESOLUTION_FUZZY_THRESHOLD=0.85 - ENTITY_RESOLUTION_VECTOR_THRESHOLD=0.5 - ENTITY_RESOLUTION_MAX_CANDIDATES=3
84 lines
2.3 KiB
YAML
84 lines
2.3 KiB
YAML
name: lightrag-entity-resolution-test
|
|
|
|
services:
|
|
postgres:
|
|
container_name: lightrag-postgres
|
|
build:
|
|
context: ./docker/postgres-age-vector
|
|
dockerfile: Dockerfile
|
|
environment:
|
|
POSTGRES_DB: lightrag
|
|
POSTGRES_USER: lightrag
|
|
POSTGRES_PASSWORD: lightrag_pass
|
|
ports:
|
|
- "5433:5432" # Use 5433 to avoid conflict with agent-sdk postgres
|
|
volumes:
|
|
- pgdata_test:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U lightrag -d lightrag"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 5
|
|
|
|
lightrag:
|
|
container_name: lightrag-test
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
ports:
|
|
- "9622:9621" # Use 9622 to avoid conflict
|
|
volumes:
|
|
- ./data/rag_storage_test:/app/data/rag_storage
|
|
- ./data/inputs_test:/app/data/inputs
|
|
environment:
|
|
# Server
|
|
- HOST=0.0.0.0
|
|
- PORT=9621
|
|
- LOG_LEVEL=DEBUG
|
|
|
|
# LLM (OpenAI)
|
|
- LLM_BINDING=openai
|
|
- LLM_MODEL=gpt-4o-mini
|
|
- LLM_BINDING_HOST=https://api.openai.com/v1
|
|
- LLM_BINDING_API_KEY=${OPENAI_API_KEY}
|
|
|
|
# Embedding
|
|
- EMBEDDING_BINDING=openai
|
|
- EMBEDDING_MODEL=text-embedding-3-small
|
|
- EMBEDDING_DIM=1536
|
|
- EMBEDDING_BINDING_HOST=https://api.openai.com/v1
|
|
- EMBEDDING_BINDING_API_KEY=${OPENAI_API_KEY}
|
|
|
|
# Storage Configuration - Full PostgreSQL!
|
|
# Custom postgres image has pgvector + Apache AGE
|
|
- LIGHTRAG_KV_STORAGE=PGKVStorage
|
|
- LIGHTRAG_VECTOR_STORAGE=PGVectorStorage
|
|
- LIGHTRAG_GRAPH_STORAGE=PGGraphStorage
|
|
- LIGHTRAG_DOC_STATUS_STORAGE=PGDocStatusStorage
|
|
- POSTGRES_HOST=postgres
|
|
- POSTGRES_PORT=5432
|
|
- POSTGRES_USER=lightrag
|
|
- POSTGRES_PASSWORD=lightrag_pass
|
|
- POSTGRES_DATABASE=lightrag
|
|
|
|
# Entity Resolution - ENABLED!
|
|
- ENTITY_RESOLUTION_ENABLED=true
|
|
- ENTITY_RESOLUTION_FUZZY_THRESHOLD=0.85
|
|
- ENTITY_RESOLUTION_VECTOR_THRESHOLD=0.5
|
|
- ENTITY_RESOLUTION_MAX_CANDIDATES=3
|
|
|
|
# Processing
|
|
- MAX_ASYNC=4
|
|
- CHUNK_SIZE=1200
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "curl -f http://localhost:9621/health || exit 1"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 10
|
|
start_period: 30s
|
|
|
|
volumes:
|
|
pgdata_test:
|