gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Resolve chunking issue for notebook

Browse source 
Add cleaning of local data to resolve chunking issue with repeated notebook use

Fix
This commit is contained in:
Igor Ilic 2024-10-27 22:33:20 +01:00
parent 99f50523e9
commit 6555f4e88e
1 changed files with 9 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
notebooks/cognee_demo.ipynb
View file

@ -577,6 +577,7 @@
"\n", "\n",
"import cognee\n", "import cognee\n",
"\n", "\n",
"await cognee.prune.prune_data()\n",
"await cognee.prune.prune_system(metadata=True)" "await cognee.prune.prune_system(metadata=True)"
] ]
}, },
@ -639,7 +640,8 @@
" chunks_into_graph, \\\n", " chunks_into_graph, \\\n",
" source_documents_to_chunks, \\\n", " source_documents_to_chunks, \\\n",
" check_permissions_on_documents, \\\n", " check_permissions_on_documents, \\\n",
" classify_documents\n", " classify_documents, \\\n",
" chunk_naive_llm_classifier\n",
"from cognee.tasks.summarization import summarize_text\n", "from cognee.tasks.summarization import summarize_text\n",
"\n", "\n",
"async def run_cognify_pipeline(dataset: Dataset, user: User = None):\n", "async def run_cognify_pipeline(dataset: Dataset, user: User = None):\n",
@ -667,6 +669,10 @@
" summarization_model = cognee_config.summarization_model,\n", " summarization_model = cognee_config.summarization_model,\n",
" collection_name = \"summaries\",\n", " collection_name = \"summaries\",\n",
" ),\n", " ),\n",
" Task(\n",
" chunk_naive_llm_classifier,\n",
" classification_model = cognee_config.classification_model,\n",
" ),\n",
" Task(chunk_remove_disconnected), # Remove the obsolete document chunks.\n", " Task(chunk_remove_disconnected), # Remove the obsolete document chunks.\n",
" ]\n", " ]\n",
"\n", "\n",
@ -876,7 +882,7 @@
], ],
"metadata": { "metadata": {
"kernelspec": { "kernelspec": {
"display_name": "cognee-bGi0WgSG-py3.9", "display_name": ".venv",
"language": "python", "language": "python",
"name": "python3" "name": "python3"
}, },
@ -890,7 +896,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.9.5" "version": "3.9.6"
} }
}, },
"nbformat": 4, "nbformat": 4,