gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Resolve chunking issue for notebook

Browse source 
Add cleaning of local data to resolve chunking issue with repeated notebook use

Fix
This commit is contained in:
Igor Ilic 2024-10-27 22:33:20 +01:00
parent 99f50523e9
commit 6555f4e88e
1 changed files with 9 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
notebooks/cognee_demo.ipynb
View file

@ -577,6 +577,7 @@
"\n",
"import cognee\n",
"\n",
"await cognee.prune.prune_data()\n",
"await cognee.prune.prune_system(metadata=True)"
]
},
@ -639,7 +640,8 @@
" chunks_into_graph, \\\n",
" source_documents_to_chunks, \\\n",
" check_permissions_on_documents, \\\n",
" classify_documents\n",
" classify_documents, \\\n",
" chunk_naive_llm_classifier\n",
"from cognee.tasks.summarization import summarize_text\n",
"\n",
"async def run_cognify_pipeline(dataset: Dataset, user: User = None):\n",
@ -667,6 +669,10 @@
" summarization_model = cognee_config.summarization_model,\n",
" collection_name = \"summaries\",\n",
" ),\n",
" Task(\n",
" chunk_naive_llm_classifier,\n",
" classification_model = cognee_config.classification_model,\n",
" ),\n",
" Task(chunk_remove_disconnected), # Remove the obsolete document chunks.\n",
" ]\n",
"\n",
@ -876,7 +882,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "cognee-bGi0WgSG-py3.9",
"display_name": ".venv",
"language": "python",
"name": "python3"
},
@ -890,7 +896,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.5"
"version": "3.9.6"
}
},
"nbformat": 4,