gmakstutis/LightRAG
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improved parallele

Browse source
This commit is contained in:
Yannick Stephan 2025-02-09 21:17:09 +01:00
parent 068924b59f
commit f37f2c7555
1 changed files with 1 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
lightrag/lightrag.py
View file

@ -494,10 +494,8 @@ class LightRAG:
# 3. Filter out already processed documents # 3. Filter out already processed documents
# Get docs ids # Get docs ids
all_new_doc_ids = set(new_docs.keys()) all_new_doc_ids = set(new_docs.keys())
# Retrieve IDs that are already being processed
existing_ids = await self.doc_status.filter_keys(all_new_doc_ids)
# Exclude IDs of documents that are already in progress # Exclude IDs of documents that are already in progress
unique_new_doc_ids = all_new_doc_ids - existing_ids unique_new_doc_ids = await self.doc_status.filter_keys(all_new_doc_ids)
# Filter new_docs to only include documents with unique IDs # Filter new_docs to only include documents with unique IDs
new_docs = {doc_id: new_docs[doc_id] for doc_id in unique_new_doc_ids} new_docs = {doc_id: new_docs[doc_id] for doc_id in unique_new_doc_ids}