From df6d1a0f072dad67da5ac918db9a46a1e0725cd5 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:50:51 +0100 Subject: [PATCH] Add utils for graph visualization + classification nodes --- Demo_graph.ipynb | 1580 +++++++++-------- .../api/v1/cognify/cognify.py | 73 +- .../databases/vector/qdrant/adapter.py | 10 + .../databases/vector/vector_db_interface.py | 15 +- .../cognify/graph/add_node_connections.py | 14 +- .../graph/add_semantic_search_connection.py | 101 -- .../llm/add_node_connection_embeddings.py | 4 +- .../modules/cognify/vector/batch_search.py | 4 +- .../cognify/vector/load_propositions.py | 25 +- 9 files changed, 892 insertions(+), 934 deletions(-) diff --git a/Demo_graph.ipynb b/Demo_graph.ipynb index 5cea71c62..1b24fc86b 100644 --- a/Demo_graph.ipynb +++ b/Demo_graph.ipynb @@ -1222,6 +1222,56 @@ "# descriptions.append({'node_id': node_id, 'description': attributes['description'], 'layer_uuid': attributes['layer_uuid'], 'layer_decomposition_uuid': attributes['layer_decomposition_uuid']})\n" ] }, + { + "cell_type": "code", + "execution_count": 122, + "id": "951588c2-e96d-4025-bcb3-6bd46baac78a", + "metadata": {}, + "outputs": [], + "source": [ + "bb =[{'node_id': '32b11173-ab64-4741-9a36-c58300525efb', 'description': 'People of Britain', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'}, {'node_id': 'cf603ed2-917e-4519-82cf-4481cffd0a16', 'description': 'Non-human living beings', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'}, {'node_id': 'c67b6aaa-bc74-4f13-ada4-308b954bfd16', 'description': 'Animals kept for companionship', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'}, {'node_id': '3c24b71a-9bff-40be-bcc2-a9ac4e4038d7', 'description': 'A type of pet, often considered as humans best friend', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'}, {'node_id': '80dee8b8-c131-4dfd-983b-7018ca37f0ac', 'description': 'Anthropologist who wrote Watching the English, nearly 20 years ago', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'}, {'node_id': '0ba68c23-1d72-4547-8c77-775ef1736f19', 'description': 'Global health crisis that increased pet dog ownership in the UK', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'}, {'node_id': '63dabc5d-746b-4762-bb8d-3a2e81cacdc2', 'description': 'Charity that coined the slogan A dog is for life, not just for Christmas in 1978', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'}, {'node_id': '0ee75618-4bfb-42cb-8de6-1b3efbef9402', 'description': 'Time period between 2019 and 2022', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'}, {'node_id': 'f5a9f247-1816-4e55-a89b-b9b516e60dca', 'description': 'Britons have always been a bit silly about animals', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'b47fe3d3-c84e-4fdb-b90d-746c4d42dc04'}, {'node_id': '43a44379-a5ae-4832-8543-e3c862f32e07', 'description': 'In the UK, keeping pets is considered an entire way of life', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'b47fe3d3-c84e-4fdb-b90d-746c4d42dc04'}, {'node_id': 'aa0a62c0-780c-4b4c-bf69-1478a418a229', 'description': 'Dogs serve as an acceptable outlet for emotions and impulses in the UK', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'b47fe3d3-c84e-4fdb-b90d-746c4d42dc04'}, {'node_id': '5f9eb7c3-9c5e-4dd6-b158-6765d2fb0835', 'description': 'In the UK, dogs are often encouraged on public transport', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'b47fe3d3-c84e-4fdb-b90d-746c4d42dc04'}, {'node_id': '6ba46f17-a801-4b62-8b50-f82a46a7a97a', 'description': 'Many pubs and shops in the UK welcome dogs', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'b47fe3d3-c84e-4fdb-b90d-746c4d42dc04'}, {'node_id': '87bc41a4-181b-4c79-9563-ea33440ddd4d', 'description': 'Pet dog ownership in the UK rose from nine million to 13 million between 2019 and 2022 due to the pandemic', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'b47fe3d3-c84e-4fdb-b90d-746c4d42dc04'}, {'node_id': 'd3b36591-9f41-4d64-9ce1-435f707ec35a', 'description': 'A dog is for life, not just for Christmas - Dogs Trust slogan', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'b47fe3d3-c84e-4fdb-b90d-746c4d42dc04'}, {'node_id': 'c30d9605-b1a9-4794-972c-6581e07ad94c', 'description': 'Britons have always been passionate about animals, considering keeping pets as an entire way of life.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '03c1865f-86ef-40e8-9dfd-809aec4f247a'}, {'node_id': 'd3992373-b3ad-4269-a35f-8dbb1233d9c4', 'description': 'Dogs serve as an acceptable outlet for emotions and impulses such as affection and social interaction among Britons.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '03c1865f-86ef-40e8-9dfd-809aec4f247a'}, {'node_id': '1ae29ee0-8d58-45cc-9cb2-b508ad245cfd', 'description': 'The COVID-19 pandemic led to a significant increase in the number of pet dogs in the UK, from about nine million to 13 million between 2019 and 2022.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '03c1865f-86ef-40e8-9dfd-809aec4f247a'}, {'node_id': '88fc2e46-22ae-445f-bc8a-87da750d4ae1', 'description': 'A famous slogan coined by the Dogs Trust charity in 1978, emphasizing that dogs are lifelong commitments.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '03c1865f-86ef-40e8-9dfd-809aec4f247a'}, {'node_id': 'dd576909-760d-4ba3-8e92-be04acf9bba9', 'description': 'Britons have a notable attachment to animals, particularly considering them an integral part of their lifestyle and a means to express emotions.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'f0e301a6-7e87-4d7b-8bbd-b684e67049f1'}, {'node_id': '88b22148-5f0f-435b-87ff-bef93d016335', 'description': 'Kate Fox is an anthropologist who wrote about the importance of pets in English culture in her book Watching the English nearly 20 years ago.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'f0e301a6-7e87-4d7b-8bbd-b684e67049f1'}, {'node_id': 'f091de12-9e95-4ebf-ae8d-cefb589faf56', 'description': 'In British culture, dogs serve as outlets for emotions and interactions, including affection and communication with strangers.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'f0e301a6-7e87-4d7b-8bbd-b684e67049f1'}, {'node_id': 'e1bf4f50-12be-47ae-9b43-cde85ba568e7', 'description': 'In the UK, unlike Australia or New Zealand, dogs are not only allowed but encouraged on public transport.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'f0e301a6-7e87-4d7b-8bbd-b684e67049f1'}, {'node_id': 'c102ac94-5634-400f-a171-4a75c20a652a', 'description': 'Between 2019 and 2022, pet dog ownership in the UK rose from about 9 million to 13 million due to the pandemic.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'f0e301a6-7e87-4d7b-8bbd-b684e67049f1'}, {'node_id': '9f1e7f7a-fb83-4697-a528-74912f364b13', 'description': 'Dogs Trust is a charity that coined the slogan \"A dog is for life, not just for Christmas\" back in 1978.', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'f0e301a6-7e87-4d7b-8bbd-b684e67049f1'}, {'node_id': '6385c542-da46-4c29-8cc5-41316e942766', 'description': 'Britons', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '0af274b4-96e2-44cf-876a-c02db53299ab'}, {'node_id': '21f5ae47-df34-4608-b0d4-2823a389c8b4', 'description': 'animals', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '0af274b4-96e2-44cf-876a-c02db53299ab'}, {'node_id': '4ad23648-2795-4aec-8077-32618a03e53e', 'description': 'Kate Fox, an anthropologist', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '0af274b4-96e2-44cf-876a-c02db53299ab'}, {'node_id': 'c48bc4c4-c3e7-478a-bb78-e0c810ba0c42', 'description': 'Watching the English, a book by Kate Fox', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '0af274b4-96e2-44cf-876a-c02db53299ab'}, {'node_id': 'd3f2fcbc-f2b5-4171-a850-c340b9f8b763', 'description': 'dogs', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '0af274b4-96e2-44cf-876a-c02db53299ab'}, {'node_id': 'a49d4f4c-062e-49ac-9d34-8e241d8ef02a', 'description': 'the pandemic', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '0af274b4-96e2-44cf-876a-c02db53299ab'}, {'node_id': '558ec89b-6a93-4e93-bee2-073ca17308b0', 'description': 'Dogs Trust, a charity', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '0af274b4-96e2-44cf-876a-c02db53299ab'}, {'node_id': 'afd9ac56-aa54-43b7-a46e-0d1487000102', 'description': '1978', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': '0af274b4-96e2-44cf-876a-c02db53299ab'}, {'node_id': 'd926bebb-aa9d-4e36-9e33-b5cf5016bd62', 'description': 'Britons', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}, {'node_id': '93b3c5a9-aa85-4c70-86b1-debd73a58933', 'description': 'animals', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}, {'node_id': '0599111c-1467-46bc-9535-ce0826a5948b', 'description': 'pets', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}, {'node_id': '4e3e3c37-b1e4-4231-b93b-624496243c84', 'description': 'English lifestyle', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}, {'node_id': '48a3c236-a4a1-44c8-be7c-73e67040e40b', 'description': 'dogs', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}, {'node_id': 'bef44708-caea-4b13-b17f-5738998ba4c8', 'description': 'emotions', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}, {'node_id': '262c2a38-c973-4df8-a5b5-09453acd7561', 'description': 'public transport in the UK', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}, {'node_id': '022a0489-3db7-4ffb-8ffb-98ddafe9c339', 'description': 'pandemic', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}, {'node_id': 'c0f62d77-ecb5-4e27-aae7-5fdb3ced39b4', 'description': 'pet dogs in the UK 2019-2022', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}, {'node_id': 'd997cbe9-e27e-4033-aa8e-58d3644bedeb', 'description': 'Dogs Trust charity', 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8', 'layer_decomposition_uuid': 'ec947375-7086-416a-9884-dd7565b5f4de'}]" + ] + }, + { + "cell_type": "code", + "execution_count": 123, + "id": "68651a26-248c-492a-890e-f939260eb744", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'node_id': '32b11173-ab64-4741-9a36-c58300525efb',\n", + " 'description': 'People of Britain',\n", + " 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8',\n", + " 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'},\n", + " {'node_id': 'cf603ed2-917e-4519-82cf-4481cffd0a16',\n", + " 'description': 'Non-human living beings',\n", + " 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8',\n", + " 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'},\n", + " {'node_id': 'c67b6aaa-bc74-4f13-ada4-308b954bfd16',\n", + " 'description': 'Animals kept for companionship',\n", + " 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8',\n", + " 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'},\n", + " {'node_id': '3c24b71a-9bff-40be-bcc2-a9ac4e4038d7',\n", + " 'description': 'A type of pet, often considered as humans best friend',\n", + " 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8',\n", + " 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'},\n", + " {'node_id': '80dee8b8-c131-4dfd-983b-7018ca37f0ac',\n", + " 'description': 'Anthropologist who wrote Watching the English, nearly 20 years ago',\n", + " 'layer_uuid': '8822b6ef-0b0d-4ba7-bec4-99d80d5e41e8',\n", + " 'layer_decomposition_uuid': 'cd154db9-1a63-4c75-a632-fee11b0cbab2'}]" + ] + }, + "execution_count": 123, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bb[:5]" + ] + }, { "cell_type": "code", "execution_count": 38, @@ -1390,25 +1440,23 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 143, "id": "adbee010-3f05-4ee7-8ff4-2072158467fe", "metadata": {}, "outputs": [], "source": [ - "qdrant = AsyncQdrantClient(\n", + "qdrant = QdrantClient(\n", " url = os.getenv('QDRANT_URL'),\n", " api_key = os.getenv('QDRANT_API_KEY'))" ] }, { "cell_type": "code", - "execution_count": 61, + "execution_count": null, "id": "c35342d7-a3ce-491b-971d-142b52110bca", "metadata": {}, "outputs": [], - "source": [ - "qdrant = QdrantClient(\":memory:\")" - ] + "source": [] }, { "cell_type": "code", @@ -1470,14 +1518,6 @@ "# await db.create_collection(\"blabla\",collection_config)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "3c34e888-e325-4f33-a73f-7e2d4ee923bd", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 90, @@ -1489,39 +1529,6 @@ " await db.create_collection(layer,collection_config)" ] }, - { - "cell_type": "code", - "execution_count": 41, - "id": "3c53de05-65d2-4232-bb23-f1b50fbd36c8", - "metadata": {}, - "outputs": [], - "source": [ - "# for item in node_descriptions[:5]: # Adjust the slice as needed for display\n", - "# print(item)\n", - "# print(item['description'])" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "id": "b810075a-3c1a-4e2e-98d6-150427951491", - "metadata": {}, - "outputs": [], - "source": [ - "# collection_schema = qdrant.get_collection(collection_name=\"97597864-a11a-4058-b854-8f21864c7e06\")\n", - "\n", - "# # Print the schema to inspect it\n", - "# print(collection_schema)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "313a893d-31b5-4599-8f6f-2a2cd0759fa0", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 94, @@ -1545,14 +1552,6 @@ " )" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "317036c5-0d03-46f2-b5c9-26f112b07f0d", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 44, @@ -1595,47 +1594,47 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "fb0235a8-7e91-4d34-9058-0cdd1dc7cfe7", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4cd43233-533a-4929-b3f2-7b8a310669a8", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 91, + "execution_count": 121, "id": "28bab5ad-cfcc-45e0-9ad9-b8736b907b39", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'node_id': '1377f8b9-9af1-49ad-a29b-ca456a5006b6',\n", - " 'description': 'Britons',\n", - " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", - " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'}" + "[{'node_id': '1377f8b9-9af1-49ad-a29b-ca456a5006b6',\n", + " 'description': 'Britons',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '98329542-0508-4077-87e4-c0fe19f89b49',\n", + " 'description': 'animals',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '0c2f31b3-290b-4bdd-9da2-73eb2bfd1807',\n", + " 'description': 'Kate Fox',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '3c4bf5e9-d95e-4d3c-9204-1d8919ff36c3',\n", + " 'description': 'Watching the English',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '993368e9-4af4-4225-b737-89cbc72acef2',\n", + " 'description': 'dogs',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'}]" ] }, - "execution_count": 91, + "execution_count": 121, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "node_descriptions[:5][0]" + "node_descriptions[:5]" ] }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 126, "id": "68f5a029-b1d3-4476-896d-711462099d52", "metadata": {}, "outputs": [ @@ -1643,126 +1642,65 @@ "name": "stdout", "output_type": "stream", "text": [ - "Britons\n", + "People of Britain\n", + "People of Britain\n", "text-embedding-3-large\n", - "animals\n", + "Non-human living beings\n", + "Non-human living beings\n", "text-embedding-3-large\n", - "Kate Fox\n", + "Animals kept for companionship\n", + "Animals kept for companionship\n", "text-embedding-3-large\n", - "Watching the English\n", + "A type of pet, often considered as humans best friend\n", + "A type of pet, often considered as humans best friend\n", "text-embedding-3-large\n", - "dogs\n", - "text-embedding-3-large\n", - "United Kingdom\n", - "text-embedding-3-large\n", - "Australia\n", - "text-embedding-3-large\n", - "New Zealand\n", - "text-embedding-3-large\n", - "Dogs Trust\n", - "text-embedding-3-large\n", - "the pandemic\n", - "text-embedding-3-large\n", - "Britons have always had a special relationship with animals, viewing pet-keeping not just as a leisure activity but as an entire way of life. This is particularly true for dogs, which serve as an acceptable outlet for emotions and impulses that are otherwise kept controlled.\n", - "text-embedding-3-large\n", - "British society is accommodating to dogs, evident from dogs being encouraged on public transport and welcome signs in many establishments.\n", - "text-embedding-3-large\n", - "The number of pet dogs in the UK increased from about 9 million to 13 million between 2019 and 2022, indicating a pet ownership boom during the pandemic.\n", - "text-embedding-3-large\n", - "In the nicest possible way, Britons have always been a bit silly about animals\n", - "text-embedding-3-large\n", - "Dogs serve as an acceptable outlet for Britons to be affectionate, to be silly, and to chat with strangers\n", - "text-embedding-3-large\n", - "Kate Fox, anthropologist\n", - "text-embedding-3-large\n", - "Watching the English, book by Kate Fox written nearly 20 years ago\n", - "text-embedding-3-large\n", - "Between 2019 and 2022, the number of pet dogs in the UK rose from about nine million to 13 million\n", - "text-embedding-3-large\n", - "A dog is for life, not just for Christmas, slogan by Dogs Trust charity coined in 1978\n", - "text-embedding-3-large\n", - "Britons have always been a bit silly about animals, keeping pets is an essential way of life\n", - "text-embedding-3-large\n", - "Pets, especially dogs, are an outlet for emotions and impulses like affection and desire to chat with strangers\n", - "text-embedding-3-large\n", - "In the UK, unlike Australia or New Zealand, dogs are openly encouraged on public transport\n", - "text-embedding-3-large\n", - "Many pubs and shops in the UK display signs like Dogs welcome, people tolerated and have treat jars for dogs\n", - "text-embedding-3-large\n", - "Between 2019 and 2022, the number of pet dogs in the UK rose from about 9 million to 13 million\n", - "text-embedding-3-large\n", - "Dogs Trust charity coined the slogan A dog is for life, not just for Christmas in 1978\n", - "text-embedding-3-large\n", - "Britons have always been a bit silly about animals, considering keeping pets as an entire way of life.\n", - "text-embedding-3-large\n", - "Dogs serve as an acceptable outlet for Britons' emotions and impulses they otherwise keep strictly controlled.\n", - "text-embedding-3-large\n", - "In the UK, unlike Australia or New Zealand, dogs are not just permitted on public transport but often openly encouraged.\n", - "text-embedding-3-large\n", - "Between 2019 and 2022, the number of pet dogs in the UK rose from about nine million to 13 million due to the pandemic.\n", - "text-embedding-3-large\n", - "The Dogs Trust charity coined its famous slogan \"A dog is for life, not just for Christmas\" back in 1978.\n", - "text-embedding-3-large\n", - "Britons have always been a bit silly about animals\n", - "text-embedding-3-large\n", - "Kate Fox: Keeping pets is not a leisure activity but an entire way of life for the English\n", - "text-embedding-3-large\n", - "Dogs serve as an outlet for emotions and impulses\n", - "text-embedding-3-large\n", - "British society accommodates dogs in public transport and establishments\n", - "text-embedding-3-large\n", - "Britons' passion for animals has been consistent amid dwindling common ground\n", - "text-embedding-3-large\n", - "The pandemic unleashed a trend of acquiring dogs, increasing UK dog population from about 9 million to 13 million between 2019 and 2022\n", - "text-embedding-3-large\n", - "Dogs Trust slogan coined in 1978: A dog is for life, not just for Christmas\n", - "text-embedding-3-large\n", - "Keeping pets, for the English, is not so much a leisure activity as it is an entire way of life\n", - "text-embedding-3-large\n", - "Kate Fox, anthropologist who wrote Watching the English\n", - "text-embedding-3-large\n", - "Dogs as an acceptable outlet for emotions and impulses kept strictly controlled\n", - "text-embedding-3-large\n", - "Dogs are not just permitted on public transport in the UK but often openly encouraged\n", - "text-embedding-3-large\n", - "Many pubs and shops in the UK display signs reading Dogs welcome, people tolerated and have treat jars on their counters\n", - "text-embedding-3-large\n", - "A dog is for life, not just for Christmas\n", - "text-embedding-3-large\n", - "Between 2019 and 2022, the number of pet dogs in the UK rose from about nine million to 13 million\n", + "Anthropologist who wrote Watching the English, nearly 20 years ago\n", + "Anthropologist who wrote Watching the English, nearly 20 years ago\n", "text-embedding-3-large\n" ] + }, + { + "ename": "CancelledError", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mCancelledError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[126], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m bb: \n\u001b[1;32m 2\u001b[0m \u001b[38;5;28mprint\u001b[39m(item[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdescription\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[0;32m----> 3\u001b[0m vv \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m get_embeddings([item[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdescription\u001b[39m\u001b[38;5;124m'\u001b[39m]])\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m upload_embedding(\u001b[38;5;28mid\u001b[39m \u001b[38;5;241m=\u001b[39m item[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnode_id\u001b[39m\u001b[38;5;124m'\u001b[39m], metadata \u001b[38;5;241m=\u001b[39m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmeta\u001b[39m\u001b[38;5;124m\"\u001b[39m:item[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdescription\u001b[39m\u001b[38;5;124m'\u001b[39m]}, some_embeddings \u001b[38;5;241m=\u001b[39m vv[\u001b[38;5;241m0\u001b[39m], collection_name\u001b[38;5;241m=\u001b[39m item[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlayer_decomposition_uuid\u001b[39m\u001b[38;5;124m'\u001b[39m])\n", + "Cell \u001b[0;32mIn[45], line 13\u001b[0m, in \u001b[0;36mget_embeddings\u001b[0;34m(texts)\u001b[0m\n\u001b[1;32m 11\u001b[0m client \u001b[38;5;241m=\u001b[39m get_llm_client()\n\u001b[1;32m 12\u001b[0m tasks \u001b[38;5;241m=\u001b[39m [ client\u001b[38;5;241m.\u001b[39masync_get_embedding_with_backoff(text, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext-embedding-3-large\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m text \u001b[38;5;129;01min\u001b[39;00m texts]\n\u001b[0;32m---> 13\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39mtasks)\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m results\n", + "File \u001b[0;32m/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/tasks.py:349\u001b[0m, in \u001b[0;36mTask.__wakeup\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__wakeup\u001b[39m(\u001b[38;5;28mself\u001b[39m, future):\n\u001b[1;32m 348\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 349\u001b[0m \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 350\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 351\u001b[0m \u001b[38;5;66;03m# This may also be a cancellation.\u001b[39;00m\n\u001b[1;32m 352\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__step(exc)\n", + "File \u001b[0;32m/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/tasks.py:279\u001b[0m, in \u001b[0;36mTask.__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 277\u001b[0m result \u001b[38;5;241m=\u001b[39m coro\u001b[38;5;241m.\u001b[39msend(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 279\u001b[0m result \u001b[38;5;241m=\u001b[39m coro\u001b[38;5;241m.\u001b[39mthrow(exc)\n\u001b[1;32m 280\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 281\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_must_cancel:\n\u001b[1;32m 282\u001b[0m \u001b[38;5;66;03m# Task is cancelled right before coro stops.\u001b[39;00m\n", + "File \u001b[0;32m~/Projects/cognee/cognitive_architecture/infrastructure/llm/openai/adapter.py:153\u001b[0m, in \u001b[0;36mOpenAIAdapter.async_get_embedding_with_backoff\u001b[0;34m(self, text, model)\u001b[0m\n\u001b[1;32m 151\u001b[0m \u001b[38;5;28mprint\u001b[39m(text)\n\u001b[1;32m 152\u001b[0m \u001b[38;5;28mprint\u001b[39m(model)\n\u001b[0;32m--> 153\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maclient\u001b[38;5;241m.\u001b[39membeddings\u001b[38;5;241m.\u001b[39mcreate(\u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39mtext, model\u001b[38;5;241m=\u001b[39m model)\n\u001b[1;32m 154\u001b[0m \u001b[38;5;66;03m# response = await self.acreate_embedding_with_backoff(input=text, model=model)\u001b[39;00m\n\u001b[1;32m 155\u001b[0m embedding \u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mdata[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39membedding\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/openai/resources/embeddings.py:214\u001b[0m, in \u001b[0;36mAsyncEmbeddings.create\u001b[0;34m(self, input, model, dimensions, encoding_format, user, extra_headers, extra_query, extra_body, timeout)\u001b[0m\n\u001b[1;32m 208\u001b[0m embedding\u001b[38;5;241m.\u001b[39membedding \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mfrombuffer( \u001b[38;5;66;03m# type: ignore[no-untyped-call]\u001b[39;00m\n\u001b[1;32m 209\u001b[0m base64\u001b[38;5;241m.\u001b[39mb64decode(data), dtype\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfloat32\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 210\u001b[0m )\u001b[38;5;241m.\u001b[39mtolist()\n\u001b[1;32m 212\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj\n\u001b[0;32m--> 214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_post(\n\u001b[1;32m 215\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/embeddings\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 216\u001b[0m body\u001b[38;5;241m=\u001b[39mmaybe_transform(params, embedding_create_params\u001b[38;5;241m.\u001b[39mEmbeddingCreateParams),\n\u001b[1;32m 217\u001b[0m options\u001b[38;5;241m=\u001b[39mmake_request_options(\n\u001b[1;32m 218\u001b[0m extra_headers\u001b[38;5;241m=\u001b[39mextra_headers,\n\u001b[1;32m 219\u001b[0m extra_query\u001b[38;5;241m=\u001b[39mextra_query,\n\u001b[1;32m 220\u001b[0m extra_body\u001b[38;5;241m=\u001b[39mextra_body,\n\u001b[1;32m 221\u001b[0m timeout\u001b[38;5;241m=\u001b[39mtimeout,\n\u001b[1;32m 222\u001b[0m post_parser\u001b[38;5;241m=\u001b[39mparser,\n\u001b[1;32m 223\u001b[0m ),\n\u001b[1;32m 224\u001b[0m cast_to\u001b[38;5;241m=\u001b[39mCreateEmbeddingResponse,\n\u001b[1;32m 225\u001b[0m )\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/openai/_base_client.py:1725\u001b[0m, in \u001b[0;36mAsyncAPIClient.post\u001b[0;34m(self, path, cast_to, body, files, options, stream, stream_cls)\u001b[0m\n\u001b[1;32m 1711\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(\n\u001b[1;32m 1712\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 1713\u001b[0m path: \u001b[38;5;28mstr\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1720\u001b[0m stream_cls: \u001b[38;5;28mtype\u001b[39m[_AsyncStreamT] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1721\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ResponseT \u001b[38;5;241m|\u001b[39m _AsyncStreamT:\n\u001b[1;32m 1722\u001b[0m opts \u001b[38;5;241m=\u001b[39m FinalRequestOptions\u001b[38;5;241m.\u001b[39mconstruct(\n\u001b[1;32m 1723\u001b[0m method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpost\u001b[39m\u001b[38;5;124m\"\u001b[39m, url\u001b[38;5;241m=\u001b[39mpath, json_data\u001b[38;5;241m=\u001b[39mbody, files\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mawait\u001b[39;00m async_to_httpx_files(files), \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39moptions\n\u001b[1;32m 1724\u001b[0m )\n\u001b[0;32m-> 1725\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequest(cast_to, opts, stream\u001b[38;5;241m=\u001b[39mstream, stream_cls\u001b[38;5;241m=\u001b[39mstream_cls)\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/openai/_base_client.py:1428\u001b[0m, in \u001b[0;36mAsyncAPIClient.request\u001b[0;34m(self, cast_to, options, stream, stream_cls, remaining_retries)\u001b[0m\n\u001b[1;32m 1419\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\n\u001b[1;32m 1420\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 1421\u001b[0m cast_to: Type[ResponseT],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1426\u001b[0m remaining_retries: Optional[\u001b[38;5;28mint\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1427\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ResponseT \u001b[38;5;241m|\u001b[39m _AsyncStreamT:\n\u001b[0;32m-> 1428\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_request(\n\u001b[1;32m 1429\u001b[0m cast_to\u001b[38;5;241m=\u001b[39mcast_to,\n\u001b[1;32m 1430\u001b[0m options\u001b[38;5;241m=\u001b[39moptions,\n\u001b[1;32m 1431\u001b[0m stream\u001b[38;5;241m=\u001b[39mstream,\n\u001b[1;32m 1432\u001b[0m stream_cls\u001b[38;5;241m=\u001b[39mstream_cls,\n\u001b[1;32m 1433\u001b[0m remaining_retries\u001b[38;5;241m=\u001b[39mremaining_retries,\n\u001b[1;32m 1434\u001b[0m )\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/openai/_base_client.py:1457\u001b[0m, in \u001b[0;36mAsyncAPIClient._request\u001b[0;34m(self, cast_to, options, stream, stream_cls, remaining_retries)\u001b[0m\n\u001b[1;32m 1454\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauth\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcustom_auth\n\u001b[1;32m 1456\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1457\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_client\u001b[38;5;241m.\u001b[39msend(\n\u001b[1;32m 1458\u001b[0m request,\n\u001b[1;32m 1459\u001b[0m stream\u001b[38;5;241m=\u001b[39mstream \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_should_stream_response_body(request\u001b[38;5;241m=\u001b[39mrequest),\n\u001b[1;32m 1460\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 1461\u001b[0m )\n\u001b[1;32m 1462\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m httpx\u001b[38;5;241m.\u001b[39mTimeoutException \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m 1463\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEncountered httpx.TimeoutException\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpx/_client.py:1661\u001b[0m, in \u001b[0;36mAsyncClient.send\u001b[0;34m(self, request, stream, auth, follow_redirects)\u001b[0m\n\u001b[1;32m 1653\u001b[0m follow_redirects \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 1654\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfollow_redirects\n\u001b[1;32m 1655\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(follow_redirects, UseClientDefault)\n\u001b[1;32m 1656\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m follow_redirects\n\u001b[1;32m 1657\u001b[0m )\n\u001b[1;32m 1659\u001b[0m auth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_request_auth(request, auth)\n\u001b[0;32m-> 1661\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_send_handling_auth(\n\u001b[1;32m 1662\u001b[0m request,\n\u001b[1;32m 1663\u001b[0m auth\u001b[38;5;241m=\u001b[39mauth,\n\u001b[1;32m 1664\u001b[0m follow_redirects\u001b[38;5;241m=\u001b[39mfollow_redirects,\n\u001b[1;32m 1665\u001b[0m history\u001b[38;5;241m=\u001b[39m[],\n\u001b[1;32m 1666\u001b[0m )\n\u001b[1;32m 1667\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1668\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m stream:\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpx/_client.py:1689\u001b[0m, in \u001b[0;36mAsyncClient._send_handling_auth\u001b[0;34m(self, request, auth, follow_redirects, history)\u001b[0m\n\u001b[1;32m 1686\u001b[0m request \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m auth_flow\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__anext__\u001b[39m()\n\u001b[1;32m 1688\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m-> 1689\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_send_handling_redirects(\n\u001b[1;32m 1690\u001b[0m request,\n\u001b[1;32m 1691\u001b[0m follow_redirects\u001b[38;5;241m=\u001b[39mfollow_redirects,\n\u001b[1;32m 1692\u001b[0m history\u001b[38;5;241m=\u001b[39mhistory,\n\u001b[1;32m 1693\u001b[0m )\n\u001b[1;32m 1694\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1695\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpx/_client.py:1726\u001b[0m, in \u001b[0;36mAsyncClient._send_handling_redirects\u001b[0;34m(self, request, follow_redirects, history)\u001b[0m\n\u001b[1;32m 1723\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m hook \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_event_hooks[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequest\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 1724\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m hook(request)\n\u001b[0;32m-> 1726\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_send_single_request(request)\n\u001b[1;32m 1727\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 1728\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m hook \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_event_hooks[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresponse\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpx/_client.py:1763\u001b[0m, in \u001b[0;36mAsyncClient._send_single_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 1758\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 1759\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAttempted to send an sync request with an AsyncClient instance.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1760\u001b[0m )\n\u001b[1;32m 1762\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m request_context(request\u001b[38;5;241m=\u001b[39mrequest):\n\u001b[0;32m-> 1763\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m transport\u001b[38;5;241m.\u001b[39mhandle_async_request(request)\n\u001b[1;32m 1765\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response\u001b[38;5;241m.\u001b[39mstream, AsyncByteStream)\n\u001b[1;32m 1766\u001b[0m response\u001b[38;5;241m.\u001b[39mrequest \u001b[38;5;241m=\u001b[39m request\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpx/_transports/default.py:373\u001b[0m, in \u001b[0;36mAsyncHTTPTransport.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 360\u001b[0m req \u001b[38;5;241m=\u001b[39m httpcore\u001b[38;5;241m.\u001b[39mRequest(\n\u001b[1;32m 361\u001b[0m method\u001b[38;5;241m=\u001b[39mrequest\u001b[38;5;241m.\u001b[39mmethod,\n\u001b[1;32m 362\u001b[0m url\u001b[38;5;241m=\u001b[39mhttpcore\u001b[38;5;241m.\u001b[39mURL(\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 370\u001b[0m extensions\u001b[38;5;241m=\u001b[39mrequest\u001b[38;5;241m.\u001b[39mextensions,\n\u001b[1;32m 371\u001b[0m )\n\u001b[1;32m 372\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[0;32m--> 373\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pool\u001b[38;5;241m.\u001b[39mhandle_async_request(req)\n\u001b[1;32m 375\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(resp\u001b[38;5;241m.\u001b[39mstream, typing\u001b[38;5;241m.\u001b[39mAsyncIterable)\n\u001b[1;32m 377\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m Response(\n\u001b[1;32m 378\u001b[0m status_code\u001b[38;5;241m=\u001b[39mresp\u001b[38;5;241m.\u001b[39mstatus,\n\u001b[1;32m 379\u001b[0m headers\u001b[38;5;241m=\u001b[39mresp\u001b[38;5;241m.\u001b[39mheaders,\n\u001b[1;32m 380\u001b[0m stream\u001b[38;5;241m=\u001b[39mAsyncResponseStream(resp\u001b[38;5;241m.\u001b[39mstream),\n\u001b[1;32m 381\u001b[0m extensions\u001b[38;5;241m=\u001b[39mresp\u001b[38;5;241m.\u001b[39mextensions,\n\u001b[1;32m 382\u001b[0m )\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py:216\u001b[0m, in \u001b[0;36mAsyncConnectionPool.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 213\u001b[0m closing \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_assign_requests_to_connections()\n\u001b[1;32m 215\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_close_connections(closing)\n\u001b[0;32m--> 216\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;66;03m# Return the response. Note that in this case we still have to manage\u001b[39;00m\n\u001b[1;32m 219\u001b[0m \u001b[38;5;66;03m# the point at which the response is closed.\u001b[39;00m\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response\u001b[38;5;241m.\u001b[39mstream, AsyncIterable)\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpcore/_async/connection_pool.py:196\u001b[0m, in \u001b[0;36mAsyncConnectionPool.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 192\u001b[0m connection \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m pool_request\u001b[38;5;241m.\u001b[39mwait_for_connection(timeout\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m 194\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 195\u001b[0m \u001b[38;5;66;03m# Send the request on the assigned connection.\u001b[39;00m\n\u001b[0;32m--> 196\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m connection\u001b[38;5;241m.\u001b[39mhandle_async_request(\n\u001b[1;32m 197\u001b[0m pool_request\u001b[38;5;241m.\u001b[39mrequest\n\u001b[1;32m 198\u001b[0m )\n\u001b[1;32m 199\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ConnectionNotAvailable:\n\u001b[1;32m 200\u001b[0m \u001b[38;5;66;03m# In some cases a connection may initially be available to\u001b[39;00m\n\u001b[1;32m 201\u001b[0m \u001b[38;5;66;03m# handle a request, but then become unavailable.\u001b[39;00m\n\u001b[1;32m 202\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[1;32m 203\u001b[0m \u001b[38;5;66;03m# In this case we clear the connection and try again.\u001b[39;00m\n\u001b[1;32m 204\u001b[0m pool_request\u001b[38;5;241m.\u001b[39mclear_connection()\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpcore/_async/connection.py:99\u001b[0m, in \u001b[0;36mAsyncHTTPConnection.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 97\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 98\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connect_failed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m---> 99\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connection\u001b[38;5;241m.\u001b[39mhandle_async_request(request)\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpcore/_async/connection.py:74\u001b[0m, in \u001b[0;36mAsyncHTTPConnection.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 70\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAttempted to send request to \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrequest\u001b[38;5;241m.\u001b[39murl\u001b[38;5;241m.\u001b[39morigin\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m on connection to \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_origin\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 71\u001b[0m )\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 74\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_request_lock:\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connection \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 76\u001b[0m stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connect(request)\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/httpcore/_synchronization.py:76\u001b[0m, in \u001b[0;36mAsyncLock.__aenter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trio_lock\u001b[38;5;241m.\u001b[39macquire()\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backend \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masyncio\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m---> 76\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_anyio_lock\u001b[38;5;241m.\u001b[39macquire()\n\u001b[1;32m 78\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/anyio/_core/_synchronization.py:143\u001b[0m, in \u001b[0;36mLock.acquire\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 143\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m cancel_shielded_checkpoint()\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m:\n\u001b[1;32m 145\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrelease()\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/anyio/lowlevel.py:61\u001b[0m, in \u001b[0;36mcancel_shielded_checkpoint\u001b[0;34m()\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcancel_shielded_checkpoint\u001b[39m() \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 49\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;124;03m Allow the scheduler to switch to another task but without checking for cancellation.\u001b[39;00m\n\u001b[1;32m 51\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 59\u001b[0m \n\u001b[1;32m 60\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 61\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m get_asynclib()\u001b[38;5;241m.\u001b[39mcancel_shielded_checkpoint()\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py:471\u001b[0m, in \u001b[0;36mcancel_shielded_checkpoint\u001b[0;34m()\u001b[0m\n\u001b[1;32m 469\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcancel_shielded_checkpoint\u001b[39m() \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 470\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m CancelScope(shield\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[0;32m--> 471\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m sleep(\u001b[38;5;241m0\u001b[39m)\n", + "File \u001b[0;32m/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/tasks.py:640\u001b[0m, in \u001b[0;36msleep\u001b[0;34m(delay, result)\u001b[0m\n\u001b[1;32m 638\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Coroutine that completes after a given time (in seconds).\"\"\"\u001b[39;00m\n\u001b[1;32m 639\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m delay \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 640\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m __sleep0()\n\u001b[1;32m 641\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[1;32m 643\u001b[0m loop \u001b[38;5;241m=\u001b[39m events\u001b[38;5;241m.\u001b[39mget_running_loop()\n", + "File \u001b[0;32m/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/tasks.py:634\u001b[0m, in \u001b[0;36m__sleep0\u001b[0;34m()\u001b[0m\n\u001b[1;32m 625\u001b[0m \u001b[38;5;129m@types\u001b[39m\u001b[38;5;241m.\u001b[39mcoroutine\n\u001b[1;32m 626\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__sleep0\u001b[39m():\n\u001b[1;32m 627\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Skip one event loop run cycle.\u001b[39;00m\n\u001b[1;32m 628\u001b[0m \n\u001b[1;32m 629\u001b[0m \u001b[38;5;124;03m This is a private helper for 'asyncio.sleep()', used\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 632\u001b[0m \u001b[38;5;124;03m instead of creating a Future object.\u001b[39;00m\n\u001b[1;32m 633\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 634\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m\n", + "\u001b[0;31mCancelledError\u001b[0m: " + ] } ], "source": [ - "for item in node_descriptions: \n", + "for item in bb: \n", + " print(item['description'])\n", " vv = await get_embeddings([item['description']])\n", " await upload_embedding(id = item['node_id'], metadata = {\"meta\":item['description']}, some_embeddings = vv[0], collection_name= item['layer_decomposition_uuid'])" ] }, - { - "cell_type": "code", - "execution_count": 47, - "id": "9c33fe6a-30fc-4ae9-85c3-c3c0563ab7c6", - "metadata": {}, - "outputs": [], - "source": [ - "# import nest_asyncio\n", - "\n", - "# # Apply nest_asyncio to the current event loop\n", - "# nest_asyncio.apply()\n", - "\n", - "# await upload_embeddings(node_descriptions)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "302dc698-2c14-4408-81d7-76282524dba3", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 96, @@ -1786,14 +1724,6 @@ " grouped_data[uuid].append(item)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "a507c77e-9190-4a71-8321-ccc081121534", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 97, @@ -1891,9 +1821,7 @@ "id": "d6253483-4bd3-426d-88fc-7ac29ed1d5dc", "metadata": {}, "outputs": [], - "source": [ - "# b = get_embedding(\"dog\")" - ] + "source": [] }, { "cell_type": "code", @@ -1901,15 +1829,7 @@ "id": "4a7173ab-3e1b-496d-a56e-c7d05b84f0fc", "metadata": {}, "outputs": [], - "source": [ - "# c = []\n", - "# for collection in unique_layer_uuids:\n", - "# try:\n", - "# # c.apend(qdrant_search(collection, b))\n", - "# c.apend(qdrant_batch_search(collection, [b], [3]))\n", - "# except:\n", - "# pass" - ] + "source": [] }, { "cell_type": "code", @@ -1944,11 +1864,199 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 128, "id": "8e517772-d4eb-4e7a-9393-1ea695020e65", "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "{'e800462b-fbe4-4ea9-a71b-fc8eda28728f': [{'node_id': '1377f8b9-9af1-49ad-a29b-ca456a5006b6',\n", + " 'description': 'Britons',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '98329542-0508-4077-87e4-c0fe19f89b49',\n", + " 'description': 'animals',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '0c2f31b3-290b-4bdd-9da2-73eb2bfd1807',\n", + " 'description': 'Kate Fox',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '3c4bf5e9-d95e-4d3c-9204-1d8919ff36c3',\n", + " 'description': 'Watching the English',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '993368e9-4af4-4225-b737-89cbc72acef2',\n", + " 'description': 'dogs',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '50e4358e-1555-42a5-9fca-507f13fa55fd',\n", + " 'description': 'United Kingdom',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '41830c68-b96d-4ff3-84d2-24e9b236df31',\n", + " 'description': 'Australia',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '3216299a-9539-49b3-a563-a15ef8f6d603',\n", + " 'description': 'New Zealand',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': 'b077e06a-b9a5-44e3-90f0-edb6dce26f64',\n", + " 'description': 'Dogs Trust',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'},\n", + " {'node_id': '9714aa6a-d98e-41ef-b4f7-ab5d498502d8',\n", + " 'description': 'the pandemic',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e800462b-fbe4-4ea9-a71b-fc8eda28728f'}],\n", + " 'cac55ec8-d110-4405-8add-4d29be627951': [{'node_id': 'bcdf98d9-99f5-4167-a002-6a297256843b',\n", + " 'description': 'Britons have always had a special relationship with animals, viewing pet-keeping not just as a leisure activity but as an entire way of life. This is particularly true for dogs, which serve as an acceptable outlet for emotions and impulses that are otherwise kept controlled.',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'cac55ec8-d110-4405-8add-4d29be627951'},\n", + " {'node_id': 'c6617ac0-5f84-4d24-b05c-2e3dff3af3ba',\n", + " 'description': 'British society is accommodating to dogs, evident from dogs being encouraged on public transport and welcome signs in many establishments.',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'cac55ec8-d110-4405-8add-4d29be627951'},\n", + " {'node_id': '886d5956-c81a-4c4c-a11d-671954d4c39c',\n", + " 'description': 'The number of pet dogs in the UK increased from about 9 million to 13 million between 2019 and 2022, indicating a pet ownership boom during the pandemic.',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'cac55ec8-d110-4405-8add-4d29be627951'}],\n", + " '3a4b6713-b9bd-44f5-8017-49afc3aecf49': [{'node_id': 'f8768950-c52f-4f37-a4d6-a12d8fc34f91',\n", + " 'description': 'In the nicest possible way, Britons have always been a bit silly about animals',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '3a4b6713-b9bd-44f5-8017-49afc3aecf49'},\n", + " {'node_id': 'cadfd524-29e1-4959-aeb7-03fc61628bde',\n", + " 'description': 'Dogs serve as an acceptable outlet for Britons to be affectionate, to be silly, and to chat with strangers',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '3a4b6713-b9bd-44f5-8017-49afc3aecf49'},\n", + " {'node_id': '49b0246e-6f3f-4e72-88e9-340ed4fe38f4',\n", + " 'description': 'Kate Fox, anthropologist',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '3a4b6713-b9bd-44f5-8017-49afc3aecf49'},\n", + " {'node_id': '6139ec75-06c4-4ae4-9179-4bddc1bb6630',\n", + " 'description': 'Watching the English, book by Kate Fox written nearly 20 years ago',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '3a4b6713-b9bd-44f5-8017-49afc3aecf49'},\n", + " {'node_id': '2c5dee85-6d5a-4eec-a9a1-b66ecda55430',\n", + " 'description': 'Between 2019 and 2022, the number of pet dogs in the UK rose from about nine million to 13 million',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '3a4b6713-b9bd-44f5-8017-49afc3aecf49'},\n", + " {'node_id': 'fda119e0-88b0-42d7-866e-46964b1b72c7',\n", + " 'description': 'A dog is for life, not just for Christmas, slogan by Dogs Trust charity coined in 1978',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '3a4b6713-b9bd-44f5-8017-49afc3aecf49'}],\n", + " '35d5b544-263f-4481-bd5f-63c194977bf7': [{'node_id': 'ac50b623-3467-4140-8178-fcc07fd8d767',\n", + " 'description': 'Britons have always been a bit silly about animals, keeping pets is an essential way of life',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '35d5b544-263f-4481-bd5f-63c194977bf7'},\n", + " {'node_id': '03df32b6-fb1c-4307-aebc-b5f13bb28d00',\n", + " 'description': 'Pets, especially dogs, are an outlet for emotions and impulses like affection and desire to chat with strangers',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '35d5b544-263f-4481-bd5f-63c194977bf7'},\n", + " {'node_id': 'c35a2c40-282a-4fa7-9ad8-33539ba32a7a',\n", + " 'description': 'In the UK, unlike Australia or New Zealand, dogs are openly encouraged on public transport',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '35d5b544-263f-4481-bd5f-63c194977bf7'},\n", + " {'node_id': '96595ca2-dcb5-46a0-beb8-0f5cf81899b8',\n", + " 'description': 'Many pubs and shops in the UK display signs like Dogs welcome, people tolerated and have treat jars for dogs',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '35d5b544-263f-4481-bd5f-63c194977bf7'},\n", + " {'node_id': '0fc96132-962d-4ea2-b21d-a56a43962a43',\n", + " 'description': 'Between 2019 and 2022, the number of pet dogs in the UK rose from about 9 million to 13 million',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '35d5b544-263f-4481-bd5f-63c194977bf7'},\n", + " {'node_id': 'e77fcc9b-4743-4569-a8c3-ed3e550afaa2',\n", + " 'description': 'Dogs Trust charity coined the slogan A dog is for life, not just for Christmas in 1978',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': '35d5b544-263f-4481-bd5f-63c194977bf7'}],\n", + " 'ee770796-286a-469d-96b2-d095bc9ecf54': [{'node_id': '4f8b499c-4b74-4657-9b59-ee12c932c35a',\n", + " 'description': 'Britons have always been a bit silly about animals, considering keeping pets as an entire way of life.',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee770796-286a-469d-96b2-d095bc9ecf54'},\n", + " {'node_id': '8c8039a7-b74a-417c-8869-38abaf169060',\n", + " 'description': \"Dogs serve as an acceptable outlet for Britons' emotions and impulses they otherwise keep strictly controlled.\",\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee770796-286a-469d-96b2-d095bc9ecf54'},\n", + " {'node_id': '8bf7b6e0-247d-4284-a24e-c5d345bdefd7',\n", + " 'description': 'In the UK, unlike Australia or New Zealand, dogs are not just permitted on public transport but often openly encouraged.',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee770796-286a-469d-96b2-d095bc9ecf54'},\n", + " {'node_id': '5fd553e7-108b-4a19-a003-8e8fc6561c79',\n", + " 'description': 'Between 2019 and 2022, the number of pet dogs in the UK rose from about nine million to 13 million due to the pandemic.',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee770796-286a-469d-96b2-d095bc9ecf54'},\n", + " {'node_id': '49850164-7b1e-48e5-b316-fc1e532b5a06',\n", + " 'description': 'The Dogs Trust charity coined its famous slogan \"A dog is for life, not just for Christmas\" back in 1978.',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee770796-286a-469d-96b2-d095bc9ecf54'}],\n", + " 'e1728322-74d9-4b31-b909-82d864252d88': [{'node_id': '3a43b63e-1d9c-4fa6-96d8-86febfe44228',\n", + " 'description': 'Britons have always been a bit silly about animals',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e1728322-74d9-4b31-b909-82d864252d88'},\n", + " {'node_id': '607aa0bb-f815-48ff-99ff-8f5052a8b581',\n", + " 'description': 'Kate Fox: Keeping pets is not a leisure activity but an entire way of life for the English',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e1728322-74d9-4b31-b909-82d864252d88'},\n", + " {'node_id': 'f937cad2-ac7c-4d2e-9e47-acfe65410529',\n", + " 'description': 'Dogs serve as an outlet for emotions and impulses',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e1728322-74d9-4b31-b909-82d864252d88'},\n", + " {'node_id': 'ddb10a2b-8201-49e8-8151-caa45acda64b',\n", + " 'description': 'British society accommodates dogs in public transport and establishments',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e1728322-74d9-4b31-b909-82d864252d88'},\n", + " {'node_id': '7048f574-39c8-482a-98fc-dd3cb333ed0c',\n", + " 'description': \"Britons' passion for animals has been consistent amid dwindling common ground\",\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e1728322-74d9-4b31-b909-82d864252d88'},\n", + " {'node_id': '406c019d-7c19-44a8-a4d1-f4c98764acc8',\n", + " 'description': 'The pandemic unleashed a trend of acquiring dogs, increasing UK dog population from about 9 million to 13 million between 2019 and 2022',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e1728322-74d9-4b31-b909-82d864252d88'},\n", + " {'node_id': 'a9de1054-4cf7-479c-9c5e-40e6c60316ed',\n", + " 'description': 'Dogs Trust slogan coined in 1978: A dog is for life, not just for Christmas',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'e1728322-74d9-4b31-b909-82d864252d88'}],\n", + " 'ee5effad-a527-4fd0-85e3-3928209d18cd': [{'node_id': '45e1e5cd-19b0-4a69-91a9-30d5b5599ac2',\n", + " 'description': 'Keeping pets, for the English, is not so much a leisure activity as it is an entire way of life',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee5effad-a527-4fd0-85e3-3928209d18cd'},\n", + " {'node_id': '0004ad32-86ac-4483-a074-9af1c6f2a02f',\n", + " 'description': 'Kate Fox, anthropologist who wrote Watching the English',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee5effad-a527-4fd0-85e3-3928209d18cd'},\n", + " {'node_id': 'e5ff17ad-ea00-4903-84d2-638810967ad5',\n", + " 'description': 'Dogs as an acceptable outlet for emotions and impulses kept strictly controlled',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee5effad-a527-4fd0-85e3-3928209d18cd'},\n", + " {'node_id': 'e6b72da1-71bb-4d82-972a-df07e0d96608',\n", + " 'description': 'Dogs are not just permitted on public transport in the UK but often openly encouraged',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee5effad-a527-4fd0-85e3-3928209d18cd'},\n", + " {'node_id': 'bd456423-2c74-45ef-9a29-1046cff794ba',\n", + " 'description': 'Many pubs and shops in the UK display signs reading Dogs welcome, people tolerated and have treat jars on their counters',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee5effad-a527-4fd0-85e3-3928209d18cd'},\n", + " {'node_id': 'e08a5c91-6784-45fe-8757-96ce3a4ac4e5',\n", + " 'description': 'A dog is for life, not just for Christmas',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee5effad-a527-4fd0-85e3-3928209d18cd'},\n", + " {'node_id': '0aeb7399-84e5-401d-b1d7-3785b8bc0b33',\n", + " 'description': 'Between 2019 and 2022, the number of pet dogs in the UK rose from about nine million to 13 million',\n", + " 'layer_uuid': 'abab18eb-8eb8-4299-9a6a-96101c7dc87f',\n", + " 'layer_decomposition_uuid': 'ee5effad-a527-4fd0-85e3-3928209d18cd'}]}" + ] + }, + "execution_count": 128, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grouped_data" + ] }, { "cell_type": "code", @@ -1989,34 +2097,6 @@ " return results_to_check\n", "\n", " \n", - "# async def process_items(grouped_data, unique_layer_uuids):\n", - "# relationship_dict = {}\n", - "# task_to_id = {} # Dictionary to map tasks to ids\n", - "# tasks = [] # List to hold all tasks\n", - "\n", - "# for id in unique_layer_uuids:\n", - "# if id not in relationship_dict:\n", - "# relationship_dict[id] = []\n", - " \n", - "# for uuid, items in grouped_data.items():\n", - "# if uuid != id:\n", - "# for item in items:\n", - "# # Create a task for each embedding retrieval\n", - "# task = asyncio.create_task(async_get_embedding_with_backoff(item['description'],\"text-embedding-3-large\" ))\n", - "# tasks.append(task)\n", - "# # Map the task to the corresponding id\n", - "# task_to_id[task] = id\n", - "\n", - "# # Await all tasks to complete and gather results\n", - "# results = await asyncio.gather(*tasks)\n", - "\n", - "# # Process the results and update relationship_dict\n", - "# for task, result in zip(tasks, results):\n", - "# id = task_to_id[task] # Get the id associated with this task\n", - "# if id in relationship_dict:\n", - "# relationship_dict[id].append(result)\n", - "# else:\n", - "# relationship_dict[id] = [result]\n", "\n", "# return relationship_dict" ] @@ -2567,55 +2647,6 @@ "relationship_dict = await process_items(grouped_data, unique_layer_uuids,client)" ] }, - { - "cell_type": "code", - "execution_count": 55, - "id": "7ee6a445-e783-4d43-9d02-e97bc83502f7", - "metadata": {}, - "outputs": [], - "source": [ - "# sample_embeddings = [\n", - "# [0.1, 0.2, 0.3, 0.4], # First sample embedding\n", - "# [-0.5, 0.8, -0.3, 0.2] # Second sample embedding\n", - "# ]\n", - "\n", - "# # Corresponding limits for each embedding search\n", - "# search_limits = [3, 2] # Find top 3 for the first embedding and top 2 for the second\n", - "\n", - "# # Call the qdrant_batch_search function with the test collection name, sample embeddings, and their limits\n", - "# search_results = qdrant_batch_search('97597864-a11a-4058-b854-8f21864c7e06', [ff], search_limits)\n", - "# search_r = qdrant_search('97597864-a11a-4058-b854-8f21864c7e06',ff)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "id": "a9cb5f13-d284-4cc5-ac74-78d7f4316148", - "metadata": {}, - "outputs": [], - "source": [ - "# for i in results_to_check[:4]:\n", - "# print(i[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "83fa7c56-5610-4a4d-a643-2a57a6ecff07", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 105, - "id": "836a7b57-9b65-43e2-b57a-8f2914061b40", - "metadata": {}, - "outputs": [], - "source": [ - "# relationship_dict[1]" - ] - }, { "cell_type": "code", "execution_count": 110, @@ -2689,6 +2720,482 @@ "results = await adapted_qdrant_batch_search(relationship_dict,db)" ] }, + { + "cell_type": "code", + "execution_count": 132, + "id": "6993475a-c55a-431d-9468-6fc131a7326c", + "metadata": {}, + "outputs": [], + "source": [ + "rr = ['6a6d69d6-16b3-4c1a-935b-739d51051b5a', [0.001964783761650324, 0.020349986851215363, -0.023047715425491333, 0.01755371131002903, 0.0040958658792078495, 0.02628745324909687, -0.046637438237667084, -0.05173725262284279, 0.009885511361062527, -0.008505851030349731, -0.010113401338458061, 0.024883154779672623, -0.005355421919375658, -0.018268177285790443, -0.003550776978954673, 0.0035908117424696684, 0.00679359445348382, 0.017196478322148323, 0.03057425282895565, 0.02540052868425846, 0.03936958685517311, -0.02245643176138401, -0.009466685354709625, -0.03569870442152023, 0.025499075651168823, 0.025622259825468063, 0.013771964237093925, 0.003960363566875458, -0.0012503169709816575, 0.036068256944417953, -0.006651932839304209, -0.012614035047590733, 0.008629035204648972, 0.02039925940334797, -0.018613092601299286, -0.03737400844693184, 0.005715734791010618, 0.0028132132720202208, -0.02705119363963604, 0.019869569689035416, -0.0012356889201328158, -0.009448207914829254, -0.011172783561050892, -0.015607405453920364, 0.02463678829371929, 0.040182601660490036, -0.04543023556470871, 0.010575341060757637, 0.006861345376819372, 0.0119488425552845, 0.014424839057028294, -0.003461468731984496, -0.023478858172893524, 0.049815583974123, -0.030229339376091957, 0.03828556835651398, -0.0056418245658278465, -0.04193181172013283, 0.0003674345789477229, 0.00814861711114645, -0.07957682758569717, -0.011690155602991581, -0.006885982118546963, -0.0055987099185585976, -0.024427374824881554, 0.019918842241168022, -0.011868772096931934, -0.003944965545088053, -0.03249592334032059, -0.03796529024839401, -0.017196478322148323, 0.035550884902477264, 0.04417376220226288, -0.010870982892811298, 0.012386145070195198, -0.0053338645957410336, 0.0056110285222530365, 0.02830766886472702, 0.03375240042805672, 0.0344422310590744, 0.005709575489163399, -0.03054961748421192, -0.028603309765458107, 0.047795370221138, 0.036043621599674225, 0.0180094912648201, 0.029761239886283875, -0.03572334349155426, 0.012121299281716347, -0.009429730474948883, -0.009854715317487717, -0.004031194373965263, 0.006189993117004633, -0.02968733012676239, 0.005081337410956621, -0.017196478322148323, -0.004040433093905449, 0.0038341002073138952, 0.012971268966794014, -0.0031119342893362045, -0.0265338197350502, 0.025449801236391068, 0.012798811309039593, -0.02260425314307213, -0.0263120885938406, -0.037546463310718536, 0.005142929498106241, 0.01157929003238678, -0.013624143786728382, 0.007649722509086132, -0.014449476264417171, -0.0046686711721122265, -0.04932285100221634, 0.004721024539321661, -0.005632585845887661, -0.010439839214086533, -0.01090793777257204, -0.014708162285387516, -0.027100466191768646, 0.030500343069434166, -0.007988478057086468, -0.008653671480715275, 0.05006195232272148, -0.02074417471885681, 0.008795333094894886, -0.005176804959774017, -0.029096046462655067, 0.004360711667686701, 0.037176914513111115, -0.02159414440393448, -0.03119017370045185, -0.01697474718093872, -0.011542335152626038, 0.014572660438716412, 0.05415165796875954, 0.026632368564605713, 0.019536972045898438, -0.025351254269480705, 0.010661570355296135, -0.020103618502616882, 0.004453099332749844, -0.016124777495861053, -0.010957211256027222, 0.017775442451238632, 0.008351870812475681, -0.0007160066743381321, -0.002609959803521633, 0.01242309994995594, 0.01446179486811161, -0.002991829998791218, 0.026755550876259804, -0.018945690244436264, 0.014892938546836376, -0.002220390597358346, -0.019771022722125053, 0.03207709640264511, -0.005765008274465799, -0.0301800649613142, -0.012601717375218868, 0.03449150174856186, 0.006399405654519796, -0.0005139079876244068, -0.00670120632275939, -0.02892358973622322, 0.028751131147146225, 0.0518358014523983, -0.03128872066736221, -0.01496684830635786, 0.017233433201909065, 0.015324082225561142, -0.03256983309984207, -0.026903372257947922, -0.02414405159652233, 0.002038694219663739, 0.008893880061805248, 0.01841599866747856, 0.0007371788960881531, 0.018268177285790443, 0.000972383248154074, 0.0034861054737120867, -0.01860077492892742, 0.013882829807698727, -0.022899894043803215, 0.02616426907479763, -0.007477265316992998, 0.011924205347895622, 0.024464329704642296, 0.01093873381614685, 0.014301654882729053, 0.035403065383434296, 0.02616426907479763, -0.017110249027609825, 0.03296402096748352, -0.004798014648258686, -0.011560812592506409, 0.014732799492776394, -0.02564689703285694, 0.050998151302337646, 0.004631716292351484, -0.02049780823290348, 0.040453605353832245, -0.028874315321445465, 0.006516430526971817, -0.0036185281351208687, -0.017529074102640152, -0.020571717992424965, -0.031781457364559174, 0.09667475521564484, 0.02781493403017521, 0.02567153237760067, 0.05356037616729736, 0.03207709640264511, -0.017627621069550514, 0.0005104434676468372, -0.009620665572583675, -0.01340241264551878, 0.030278611928224564, 0.017873989418148994, 0.049002569168806076, 0.0339987687766552, -0.017701532691717148, 0.018366726115345955, -0.030032243579626083, -0.01409224234521389, 0.03510742262005806, 0.035920437425374985, -0.016629831865429878, 0.03222491964697838, 0.025474438443779945, 0.01874859631061554, 0.016309553757309914, 0.020165210589766502, -0.0419071763753891, -0.02147096022963524, -0.015558131970465183, 0.021618781611323357, -0.011265171691775322, -0.022111516445875168, 0.01490525621920824, 0.01409224234521389, -0.030475705862045288, -0.02705119363963604, -0.039960868656635284, 0.005321545992046595, 0.0419071763753891, 0.0071138725616037846, -0.02719901315867901, 0.01773848757147789, -0.016469692811369896, -0.013624143786728382, 0.04124198108911514, 0.03146117925643921, -0.0172211155295372, 0.007809861563146114, 0.026016447693109512, -0.029539508745074272, 0.01736893504858017, 0.016432737931609154, -0.027371471747756004, 0.044838953763246536, 0.057108074426651, -0.01516394317150116, 0.05962102487683296, 0.06499184668064117, 0.007495742756873369, -0.04257236793637276, -0.03141190484166145, -0.06031085550785065, 0.017418209463357925, 0.023700589314103127, 0.02705119363963604, -0.012897358275949955, 0.016703741624951363, -0.018908735364675522, -0.019894206896424294, 0.05070250853896141, 0.05134306475520134, -0.024920111522078514, 0.012065866962075233, -0.0032766927033662796, 0.01435092929750681, -0.030869895592331886, -0.014695843681693077, 0.0032428172416985035, -0.051540158689022064, -0.007563493680208921, -0.04099561646580696, -0.014868301339447498, -0.0026854099705815315, -0.0056418245658278465, -0.020411578938364983, 0.03789138048887253, 0.016617514193058014, 0.009029382839798927, 0.017295025289058685, -0.023441903293132782, -0.036289989948272705, -0.0074218325316905975, -0.035181332379579544, 0.008924676105380058, 0.008696786127984524, -0.027864206582307816, 0.03249592334032059, -0.0050690192729234695, 0.06336581707000732, 0.01948769949376583, -4.87442703160923e-05, -0.01633419096469879, -0.008512009866535664, 0.01440020278096199, -0.023885365575551987, 0.0031812251545488834, -0.021236911416053772, -0.0049643125385046005, -0.012250642292201519, 0.03582188859581947, -0.026459909975528717, -0.007138509303331375, 0.018268177285790443, 0.005512481089681387, -0.004311437718570232, -0.005629505962133408, -0.03385094553232193, -0.02870185859501362, 0.024957066401839256, -0.0033105683978646994, 0.0398869588971138, -0.005555595736950636, 0.0024867758620530367, -0.05193435028195381, 0.004958153702318668, -0.038187019526958466, 0.0009523658081889153, 0.030500343069434166, 0.04419839754700661, -0.010273540392518044, -0.006032933481037617, -0.028110574930906296, -0.011271330527961254, 0.018034128472208977, -0.0038679756689816713, 0.021951377391815186, 0.005118292756378651, 0.006812071893364191, 0.02894822508096695, 0.029662692919373512, -0.023183217272162437, 0.02870185859501362, -0.003418354317545891, 0.0007548865978606045, 0.0005404695402830839, 0.022283975034952164, -0.051392339169979095, 0.02781493403017521, 0.003612369066104293, 0.0035199811682105064, 0.013710372149944305, -0.03220028057694435, -0.019290605559945107, 0.0019540053326636553, 0.00020248359942343086, -0.046883806586265564, -0.05033295601606369, -0.006094525568187237, 0.0037663490511476994, -0.00903554167598486, -0.025129523128271103, -0.017319662496447563, -0.0005412394530139863, -0.0006216939655132592, -0.025597622618079185, 0.008635194040834904, 0.018342088907957077, -0.03813774883747101, -0.060458678752183914, 0.026238178834319115, 0.008684467524290085, -0.015558131970465183, 0.030919168144464493, -0.03434368222951889, -0.00021114497212693095, 0.011973478831350803, -0.01715952344238758, -0.02039925940334797, -0.0386551208794117, -0.0018200427293777466, 0.002716205781325698, 0.007187782786786556, -0.019155101850628853, 0.06001521646976471, 0.005373899359256029, 0.013661098666489124, -0.001363492221571505, -0.021865149959921837, 0.020731857046484947, 0.012367667630314827, -0.030278611928224564, -0.003926488105207682, -0.018514545634388924, -0.0010039490880444646, 0.031140899285674095, -0.04341002181172371, 0.011967319995164871, -0.0020694902632385492, -0.011326762847602367, -0.013242273591458797, 0.006596500054001808, -0.013057497330009937, 0.007662041112780571, -0.002962573664262891, -0.027223650366067886, -0.008629035204648972, -0.016654469072818756, 0.010741639882326126, -0.009651461616158485, 0.028012027963995934, -0.027026556432247162, 0.019968116655945778, 0.0022496466990560293, -0.023934639990329742, -0.01577986218035221, -0.03256983309984207, -0.04355784133076668, -0.02719901315867901, -0.033037930727005005, 0.03222491964697838, -0.05311691388487816, 0.0352306067943573, -0.007957682013511658, -0.0017877068603411317, -0.013969059102237225, -0.004816492088139057, 0.034269772469997406, -0.02200065180659294, 0.0002958339173346758, -0.02744538150727749, 0.0028855837881565094, 0.006602659355849028, -0.014166153036057949, 0.019906524568796158, -0.035304516553878784, 0.011955001391470432, -0.03801456466317177, 0.010304336436092854, 0.04225208982825279, 0.02803666517138481, 0.03912321850657463, 0.02272743731737137, 0.04178399220108986, 0.008961631916463375, 0.03513205796480179, -0.021212274208664894, -0.023873047903180122, -0.018711639568209648, -0.04678526148200035, -0.031042352318763733, 0.033382847905159, 0.009823919273912907, 0.01901960000395775, -0.026484547182917595, 0.008123980835080147, 0.041192710399627686, 0.014375565573573112, 0.009417411871254444, -0.053905293345451355, -0.03784210607409477, 0.0009331183391623199, 0.03712764009833336, 0.000550093303900212, 0.02426723577082157, 0.013784282840788364, -0.03330893814563751, -0.04124198108911514, 0.0131560442969203, -0.020177530124783516, -0.01794789917767048, -0.0031334913801401854, 0.052427083253860474, 0.0067258430644869804, -0.011973478831350803, -0.005651063285768032, -0.015496539883315563, -0.0024821562692523003, 0.023700589314103127, 0.006959892343729734, -0.011283649131655693, -0.03143654018640518, -0.006497952621430159, 0.017319662496447563, -0.026262816041707993, -0.008844606578350067, 0.029293140396475792, 0.004708705935627222, 0.023811455816030502, -0.030524980276823044, 0.016888517886400223, 0.028406215831637383, -0.007711314596235752, 0.066420778632164, 0.03850729763507843, -0.0285540372133255, -0.04570124298334122, 0.02453824132680893, 0.03959131985902786, 0.013956740498542786, 0.004043512977659702, -0.021976014599204063, -0.029982971027493477, -0.006196151953190565, -0.003948045428842306, -0.00493967579677701, -0.008801492862403393, -0.013537914492189884, 0.010840186849236488, 0.0029009818099439144, -0.011406832374632359, 0.02395927533507347, 0.008844606578350067, 0.0025252706836909056, -0.006325495429337025, 0.018428318202495575, 0.0034368319902569056, 0.006676569581031799, -0.013735009357333183, 0.004899641033262014, 0.03096844255924225, -0.02542516589164734, 0.019216693937778473, -0.05222998932003975, 0.0018446794711053371, 0.016962427645921707, -0.004945835098624229, -0.030229339376091957, -0.0016860801260918379, -0.02944096177816391, 0.0005504782311618328, -0.013796601444482803, -0.003661642549559474, 0.006362450309097767, -0.015976956114172935, -0.03431904688477516, -0.010027172975242138, 0.025720806792378426, -0.0169993843883276, 0.0020987463649362326, 0.0029317778535187244, 0.030229339376091957, 0.013599506579339504, -0.01586609147489071, 0.027470018714666367, -0.006941414903849363, 0.004490054678171873, 0.01072316151112318, 0.02904677204787731, 0.017011702060699463, -0.011111191473901272, 0.0032120211981236935, 0.004551646765321493, -0.03431904688477516, 0.009017064236104488, 0.028135212138295174, 0.020793449133634567, -0.013488641008734703, -0.031165536493062973, -0.025893263518810272, -0.0059220679104328156, 0.011425310745835304, -0.012724900618195534, 0.010045650415122509, -0.005721894092857838, 0.004816492088139057, -0.028455490246415138, 0.021741965785622597, -0.007132350001484156, -0.021705010905861855, -0.023503495380282402, 0.002468298189342022, -0.030229339376091957, -0.016642149537801743, -0.013254592195153236, -0.0033690808340907097, 0.031904641538858414, -0.025375891476869583, -0.007200101390480995, -0.016063185408711433, -0.024895474314689636, -0.003886453341692686, 0.011363718658685684, -0.028135212138295174, -0.04880547523498535, 0.017812397330999374, 0.012860403396189213, 0.006374768912792206, -0.01761530339717865, -0.0007845276850275695, 0.0134640047326684, 0.011523857712745667, 0.008665990084409714, -0.004853447433561087, 0.00900474563241005, 0.029243867844343185, -0.0005805043037980795, 0.018243541941046715, 0.05198362097144127, 0.014363246969878674, 0.025191115215420723, -0.02410709671676159, 0.05193435028195381, 0.0028640266973525286, 0.023084670305252075, -0.019536972045898438, -0.0235897246748209, 0.006350132171064615, -0.006134560331702232, 0.01276185642927885, -0.004560885485261679, 0.02754392847418785, 0.022591933608055115, -0.0004265243769623339, 0.021828193217515945, -0.032151009887456894, -0.004499293398112059, 0.05784717947244644, -0.0360189825296402, 0.04412448778748512, -0.036807361990213394, 0.002463678829371929, -0.013858193531632423, 0.021126044914126396, -0.009565233252942562, -0.007569652982056141, -0.016802288591861725, -0.009774645790457726, -0.04730263352394104, 0.01018731202930212, 0.014252381399273872, -0.02426723577082157, -0.0013534835306927562, 0.010027172975242138, -0.014301654882729053, 0.036560993641614914, -0.006149958353489637, -0.014203107915818691, -0.012552442960441113, -0.0132669098675251, -0.005774246994405985, -0.005238397046923637, -0.016531284898519516, 0.03008151799440384, -0.011720951646566391, -0.02756856568157673, 0.0007479574996978045, -0.008875402621924877, -0.014141515828669071, 0.022370202466845512, -0.0033629215322434902, -0.024612151086330414, -0.019463062286376953, 0.021914422512054443, 0.0265338197350502, 0.008450417779386044, -0.005681859329342842, 0.008844606578350067, 0.007735951337963343, -0.012256802059710026, -0.030106155201792717, -0.028627946972846985, -0.02037462405860424, -0.0024960145819932222, -0.0018477591220289469, 0.013747327961027622, -0.02579471655189991, -0.01788630709052086, -0.011259011924266815, 0.017085611820220947, 0.009325023740530014, -0.04784464091062546, 0.03266838192939758, 0.020842721685767174, -0.04727799445390701, -0.005450889468193054, -0.04602152109146118, -0.022678162902593613, -0.013808919116854668, 0.04299119487404823, 0.003871055319905281, 0.005925147328525782, 0.007310966961085796, -0.003464548382908106, 0.0008276420994661748, -0.02181587554514408, -0.010932574979960918, -0.0058296797797083855, -0.03335821256041527, 0.02616426907479763, 0.015619723126292229, -0.011647041887044907, 0.009528277441859245, 0.008087025955319405, -0.019229013472795486, -0.019216693937778473, -0.003135031322017312, -0.013599506579339504, 0.0015220915665850043, -0.017455164343118668, -0.018526865169405937, 0.014646570198237896, -0.004222129471600056, -0.018403680995106697, -0.012330712750554085, -0.020239120349287987, 0.011283649131655693, 0.0018354406347498298, 0.028972862288355827, 0.03998550772666931, 0.015410310588777065, -0.01440020278096199, 0.008567443117499352, 0.008025433868169785, 0.013414731249213219, -0.001830821274779737, -0.003981920890510082, 0.011277489364147186, 0.03811310976743698, 0.011160464957356453, 0.009934784844517708, 0.03286547586321831, 0.01736893504858017, -0.005675700027495623, 0.0011332922149449587, -0.012133617885410786, 0.015952320769429207, -0.0002848628500942141, -0.008204050362110138, 0.016580557450652122, -0.008616716600954533, 0.006750479806214571, -0.007255534175783396, -0.0318060927093029, 0.02457519620656967, 0.021581826731562614, -0.0027839569374918938, 0.011905727908015251, 0.025770079344511032, 0.0012634053127840161, 0.003935726825147867, -0.0032736132852733135, 0.0009092514519579709, -0.014116879552602768, -0.02184051275253296, 0.023454222828149796, -0.005167566239833832, -0.04215354472398758, -0.0169131550937891, 0.008431940339505672, -0.009509800001978874, 0.013833556324243546, -0.008900039829313755, 0.007298648357391357, -0.013303865678608418, -0.009380456991493702, 0.005324625875800848, -0.02779029682278633, 0.004320676438510418, 0.017726168036460876, 0.012958950363099575, -0.008093184791505337, -0.007132350001484156, 0.011813339777290821, 0.0018893337110057473, 0.0014835965121164918, 0.011973478831350803, -0.01255860272794962, 0.009559073485434055, 0.010944892652332783, 0.019105829298496246, -0.028898952528834343, 0.013636462390422821, -0.027149740606546402, -0.025166478008031845, 0.01988188736140728, -0.0131560442969203, -0.012934313155710697, -0.014006013981997967, 0.013439367525279522, 0.023417266085743904, 0.013414731249213219, -0.012041229754686356, 0.005925147328525782, 0.01635882630944252, -0.011782543733716011, -0.012306075543165207, -0.00585123710334301, 0.024858519434928894, -0.003313648048788309, 0.016013912856578827, 0.017479801550507545, 0.002269664080813527, -0.014437157660722733, -0.03569870442152023, 0.01685156300663948, -0.01006412785500288, 0.024365782737731934, -0.00040843174792826176, 0.014819027855992317, 0.007298648357391357, -0.05494003742933273, -0.022998441010713577, -0.005512481089681387, 0.0182312224060297, 0.009860874153673649, 0.010994166135787964, 1.3304827916726936e-05, 0.0032797723542898893, -0.015262490138411522, 0.010341291315853596, -0.025375891476869583, 0.013562551699578762, 0.01104344055056572, 0.06016303598880768, -0.002993369707837701, -0.0457751527428627, -0.010255062952637672, -0.0012233705492690206, -0.03207709640264511, -0.009522118605673313, -0.0015220915665850043, 0.009737689979374409, -0.04501141235232353, 0.008031592704355717, -0.024464329704642296, 0.008561283349990845, -0.02591790072619915, 0.01148690190166235, 0.011905727908015251, 0.007988478057086468, 0.027987390756607056, 0.02138473093509674, 0.01239230390638113, 0.019931161776185036, 0.006254664622247219, 0.038310203701257706, -0.014782072976231575, -0.008715263567864895, 0.0417347177863121, 0.007138509303331375, -0.015976956114172935, 0.02803666517138481, 0.009479003958404064, 0.04784464091062546, -0.018095720559358597, -0.024550558999180794, -0.006239266600459814, 0.0196355190128088, -0.021668054163455963, 0.005651063285768032, -0.012872721999883652, 0.017910944297909737, -0.029342414811253548, -0.0033875582739710808, 0.006485634483397007, 0.0030149267986416817, -0.026016447693109512, -0.004807253368198872, -0.017331980168819427, -0.04693308100104332, 0.020078981295228004, -0.030106155201792717, -0.02052244357764721, -0.011339081451296806, 0.020978225395083427, -0.03170754760503769, -0.007089235819876194, 0.012145936489105225, -0.001341165159828961, -0.017048656940460205, 0.013205318711698055, -0.0018585376674309373, 0.020054345950484276, -0.007569652982056141, -0.019438425078988075, -0.032890111207962036, 0.005534038413316011, 0.016802288591861725, -0.00033663862268440425, -0.01522553525865078, 0.03032788634300232, 0.033136479556560516, 0.013895148411393166, -0.030229339376091957, -0.011677836999297142, 0.022468751296401024, -0.009337342344224453, 0.015582768246531487, -0.017898626625537872, -0.013993695378303528, -0.018329769372940063, -0.023146262392401695, 0.04281873628497124, -0.033284299075603485, 0.019869569689035416, 0.015324082225561142, -0.0035169015172868967, -0.023441903293132782, -0.009368138387799263, -0.008296438492834568, 0.00212184339761734, -0.04242454841732979, -0.034171223640441895, -0.025819353759288788, -0.009429730474948883, -0.019906524568796158, 0.01064309198409319, 0.018699321895837784, 0.02068258263170719, 0.032890111207962036, -0.0285540372133255, -0.018588457256555557, -0.005441650282591581, 0.011560812592506409, 0.018711639568209648, 0.001977102132514119, -0.03695518150925636, -0.012244483456015587, -0.009047860279679298, 0.004222129471600056, 0.0005624116747640073, 0.009897829964756966, -0.009078656323254108, 0.029145320877432823, -0.02264120802283287, 0.03611753135919571, 0.013427048921585083, -0.0324220135807991, -0.03298866003751755, -0.0015890728682279587, 0.04244918376207352, -0.03146117925643921, -0.008573601953685284, 0.0035569362808018923, -0.000979312346316874, 0.0009207999682985246, 0.02147096022963524, -0.0030842178966850042, 0.004794934764504433, -0.06016303598880768, 0.03486105427145958, 0.0019986594561487436, -0.0019031919073313475, -0.006380928214639425, 0.018921053037047386, -0.007267852313816547, -0.012441577389836311, 0.0041389805264770985, 0.014474112540483475, 0.02616426907479763, -0.010587659664452076, -0.011468424461781979, 0.016161732375621796, -0.024957066401839256, 0.004299119580537081, 0.002517571672797203, -0.007828339003026485, -0.02251802384853363, -0.046095430850982666, 0.015299445018172264, -0.00842578150331974, 0.015570449642837048, -0.019931161776185036, -0.012749537825584412, 0.015607405453920364, 0.010840186849236488, -0.037694286555051804, 0.0001284769969061017, -0.01810803823173046, 0.009121770970523357, -0.008659831248223782, -0.01490525621920824, 0.030475705862045288, -0.023084670305252075, 0.017750805243849754, -0.0019493858562782407, 0.01236150786280632, -0.011505380272865295, 0.02202528901398182, 0.0028855837881565094, 0.009990217164158821, 0.03621607646346092, 0.03333357349038124, 0.00039014662615954876, -0.01263867225497961, -0.005383138079196215, -0.0386551208794117, 0.0017445925623178482, 0.003402956295758486, -0.018785551190376282, -0.025942537933588028, 0.0185391828417778, 0.008136299438774586, -0.036560993641614914, -0.040207237005233765, 0.01104344055056572, -0.01024890411645174, -0.005213760305196047, -0.0014166153268888593, 0.020731857046484947, -0.004794934764504433, 0.017295025289058685, -0.019315240904688835, -0.005651063285768032, 0.01251548808068037, 0.010150356218218803, -0.006559544708579779, -0.006063729524612427, 0.03998550772666931, -0.00931270606815815, 0.0032920907251536846, -0.010581500828266144, -0.006522589363157749, -0.023910002782940865, 0.002448280807584524, -0.024649105966091156, -0.00857976172119379, -0.0004908110131509602, -0.002725444734096527, 0.00448697479441762, -0.009368138387799263, 0.025696169584989548, 0.007477265316992998, -0.009318864904344082, -0.019869569689035416, 0.005832759663462639, -0.01446179486811161, -0.008370348252356052, -0.01605086773633957, -0.0018185029039159417, -0.016555922105908394, -0.025006338953971863, 0.011683996766805649, -0.010538386180996895, -0.021643418818712234, -0.0126078762114048, -0.004567044321447611, 0.0021002860739827156, 0.004172855988144875, -0.005774246994405985, 0.001699938322417438, -0.007292489055544138, 0.0020664106123149395, -0.0053184665739536285, -0.01709793135523796, 0.016284916549921036, -0.0015713651664555073, -0.02591790072619915, -0.009411253035068512, -0.006750479806214571, -0.004028114955872297, 0.0150900324806571, 0.019475379958748817, -0.0007591210305690765, -0.008752218447625637, 0.002289681462571025, 0.01428933721035719, -0.000648640445433557, 0.018083402886986732, 0.00937429815530777, -0.007193942088633776, 0.0439273938536644, 0.009854715317487717, -0.013661098666489124, 0.0313626304268837, 0.012398463673889637, 0.03195391222834587, -0.012737219221889973, 0.007729792036116123, -0.001975562423467636, 0.00906633771955967, 0.007267852313816547, 0.012096663005650043, 0.005986739415675402, -0.005589471198618412, 0.014486431144177914, 0.0017030179733410478, -0.0030919169075787067, -0.014437157660722733, 0.04230136424303055, 0.026336725801229477, 0.01015651598572731, -0.00500126788392663, -0.00911561120301485, -0.019709430634975433, 0.017258070409297943, -0.003997318912297487, -0.01521321665495634, 0.03700445592403412, 0.006522589363157749, -0.007230897434055805, 0.02707582898437977, 0.0010154976043850183, 0.001217211247421801, -0.021852830424904823, -0.015767544507980347, -0.0009385076118633151, -0.021852830424904823, 0.009565233252942562, -0.0390985831618309, -0.0015105430502444506, -0.025572985410690308, 0.004163617268204689, -0.02402086742222309, 0.00833339337259531, -0.009485163725912571, -0.007243215572088957, -0.013143726624548435, 0.01598927564918995, -0.009552914649248123, -0.030130792409181595, -0.013599506579339504, -0.013537914492189884, -0.0043761092238128185, 0.017344297841191292, 0.009232636541128159, 0.0017907865112647414, -0.021544871851801872, 0.004037353675812483, 0.028012027963995934, -0.002252726349979639, -0.024427374824881554, -0.01185645442456007, 0.029293140396475792, 0.0050844172947108746, 0.021310821175575256, -0.018982645124197006, -0.0059374659322202206, 0.014055287465453148, -0.016371145844459534, 0.011074235662817955, 0.012170572765171528, -0.007508060894906521, -0.020633310079574585, 0.023946957662701607, 0.006898300722241402, 0.013624143786728382, -0.010384405963122845, -0.007785224821418524, -0.014375565573573112, -0.021680373698472977, 0.004348393063992262, 0.0016383463516831398, -0.032027825713157654, 0.0019001122564077377, -0.008900039829313755, 0.020325349643826485, 0.026188906282186508, 0.021606462076306343, 0.028775768354535103, -0.009195680730044842, -0.009306546300649643, 0.0018108038930222392, 0.007920727133750916, -0.020017389208078384, 0.020448533818125725, -0.0016753015806898475, 0.0012618654873222113, -0.008056229911744595, 0.007988478057086468, -0.017578348517417908, 0.008031592704355717, -0.016531284898519516, 0.027716387063264847, -0.0022250099573284388, 0.031781457364559174, 0.011271330527961254, 0.03739864379167557, 0.015188579447567463, 0.002814752981066704, -0.01577986218035221, 0.022074561566114426, 0.025843990966677666, -0.0360189825296402, 0.007914568297564983, 0.0067874351516366005, -0.006966051645576954, 0.00621770927682519, 0.010353609919548035, -0.03141190484166145, -0.009072497487068176, -0.00997173972427845, 0.04915039241313934, 0.02122459188103676, -0.0036647222004830837, -0.011979637667536736, 0.018305134028196335, -0.022099198773503304, 0.015311763621866703, 0.028233759105205536, 0.01611245982348919, -0.016716061159968376, 0.0024421215057373047, 0.008906198665499687, -0.01072316151112318, 0.0049951085820794106, -0.009435889311134815, -0.006996847689151764, 0.011967319995164871, 0.010538386180996895, -0.008339553140103817, 0.00931270606815815, -0.02993369661271572, 0.0056264265440404415, 0.01724575087428093, 0.0037016773130744696, -0.030524980276823044, 0.010489112697541714, 0.013439367525279522, -0.011105031706392765, 0.008844606578350067, -0.011573131196200848, -0.004539328161627054, -0.005478605628013611, 0.02408245950937271, -0.010470635257661343, 0.006614977493882179, -0.005019745323807001, -0.016346508637070656, 0.0013965979451313615, -0.019894206896424294, 0.001615249435417354, -0.009657620452344418, 0.0028655664063990116, 0.014166153036057949, -0.017134886234998703, 0.0026407556142657995, 0.028283031657338142, 0.001699938322417438, 0.011757906526327133, -0.012860403396189213, 0.00854896567761898, -0.013427048921585083, 0.01075395755469799, 0.0026053402107208967, -0.006128401029855013, -0.004958153702318668, -0.009700735099613667, 0.0016706822207197547, -0.014042968861758709, 0.014375565573573112, -0.003168906783685088, 0.005604869220405817, -0.017639940604567528, -0.005383138079196215, -0.04986485838890076, -0.012848084792494774, 0.01403065025806427, -0.03648708388209343, 0.01584145426750183, 0.014609615318477154, 0.008641352877020836, -0.008099343627691269, -0.019783340394496918, -0.007138509303331375, -0.004958153702318668, 0.0052106804214417934, 0.017208795994520187, 0.05902974307537079, -0.00530614797025919, 0.019500017166137695, 0.025228070095181465, -0.03419586271047592, 0.02163109928369522, -0.0033382847905158997, -0.010655410587787628, 0.02480924502015114, 0.02919459342956543, -0.006651932839304209, -0.0010740099241957068, -0.017590666189789772, 0.012349190190434456, 0.01504075899720192, -0.013722690753638744, 0.015755224972963333, -0.012041229754686356, 0.012675627134740353, 0.012786492705345154, 0.023318719118833542, 0.0018292814493179321, -0.006910618860274553, 0.003640085458755493, -0.010261221788823605, 0.013500959612429142, 0.023675953969359398, 0.002385149011388421, -0.0073171257972717285, -0.017960218712687492, -0.018514545634388924, 0.012626353651285172, -0.04478967934846878, -0.026854097843170166, 0.004394587129354477, -0.01967247575521469, 0.03722618520259857, 0.0036062097642570734, 0.03560015931725502, 0.002697728341445327, -0.007323285099118948, 0.005222999025136232, -0.0018831745255738497, 0.013131408020853996, -0.013858193531632423, -0.008351870812475681, 0.030500343069434166, 0.01270026434212923, 0.006916778162121773, -0.0010940273059532046, 0.004748740699142218, 0.005681859329342842, -0.005136770196259022, 0.001715336344204843, -0.013771964237093925, -0.012564761564135551, 0.010316655039787292, 0.006467156577855349, -0.02221006341278553, -0.016247961670160294, 0.023873047903180122, -0.01924133114516735, -0.0031011556275188923, 0.003596971044316888, -0.011308285407721996, -0.012687945738434792, -0.0024267237167805433, -0.02052244357764721, -0.01265099085867405, 0.0172211155295372, 0.00545396888628602, 0.010328973643481731, 0.014732799492776394, -0.005004347302019596, -0.027223650366067886, 0.03495960310101509, -0.00011750592966564, -0.008746059611439705, 1.2474779396143276e-05, 0.00154518848285079, -0.004369950387626886, 0.009743849746882915, 0.009349660947918892, -0.012533965520560741, 0.013008223846554756, 0.009903988800942898, -0.00843810010701418, 0.018034128472208977, 0.0033690808340907097, 0.008444258943200111, 0.0012934313854202628, -0.006565704010426998, 0.012675627134740353, -0.019746385514736176, 0.011419150978326797, -0.014301654882729053, 0.001239538425579667, -0.029490234330296516, 0.01093873381614685, -0.04089706763625145, 0.023602042347192764, 0.02414405159652233, -0.007908408530056477, 0.009873192757368088, 0.01115430612117052, -0.0021865149028599262, 0.015200898051261902, 0.023207854479551315, -0.013303865678608418, -0.016642149537801743, -0.008142458274960518, 0.017849352210760117, 0.005407774820923805, 0.03247128799557686, 0.017960218712687492, 0.008709104731678963, 0.00755117554217577, -0.014042968861758709, -0.004785696044564247, -0.00048426687135361135, 0.01905655488371849, -0.024279555305838585, -0.014313973486423492, -0.020078981295228004, 0.016777653247117996, 0.004490054678171873, 0.017652258276939392, 0.017997173592448235, 0.03069743700325489, 0.03032788634300232, 0.017701532691717148, -0.007249374873936176, -0.021150682121515274, 0.009072497487068176, 0.003957284148782492, 0.0098177595064044, 0.03483641892671585, 0.005734212230890989, -0.011357558891177177, 0.01924133114516735, -0.02012825571000576, -0.019857250154018402, 0.0017846273258328438, 0.010809390805661678, 0.033407483249902725, 0.028751131147146225, 0.01792326383292675, -0.014498749747872353, -0.005854316521435976, 0.012170572765171528, 0.017344297841191292, 0.007711314596235752, 0.02781493403017521, -0.03577261418104172, -0.028603309765458107, 0.03922176733613014, -0.0131560442969203, -0.01015651598572731, 0.002840929664671421, -0.01590304635465145, -0.028135212138295174, 0.027593202888965607, -0.01527480874210596, 0.01271258294582367, -0.010242744348943233, -0.014141515828669071, 0.04175935685634613, -0.0016244881553575397, -0.0027223650831729174, 0.0015813738573342562, 0.004342233762145042, 0.039073944091796875, -0.016703741624951363, -0.0380392000079155, 0.007760588079690933, 0.0010940273059532046, 0.008832287974655628, -0.007304807659238577, 0.009700735099613667, 0.02076881192624569, -0.0221854280680418, -0.008093184791505337, -0.00043460834422148764, -0.011265171691775322, -0.014079924672842026, -0.017110249027609825, 0.0025745443999767303, -0.029367050155997276, -0.00933118350803852, -0.003892612410709262, 0.003449150361120701, -0.004810332786291838, 0.03764501214027405, 0.007206260226666927, -0.006082206964492798, 0.010803231038153172, -0.010840186849236488, 0.003566175000742078, -0.006571863312274218, -0.001827741740271449, 0.01794789917767048, -0.0023250968661159277, 0.012343030422925949, -0.017344297841191292, -0.007403354626148939, -0.002642295556142926, 0.005897431168705225, -0.015114669688045979, 0.008721423335373402, -0.0182312224060297, -0.01761530339717865, 0.014560341835021973, -0.026139631867408752, 0.01446179486811161, -0.014658888801932335, 0.0044038258492946625, 0.008536647073924541, 0.004114343784749508, -0.0012095123529434204, -0.008592079393565655, -0.0005801193765364587, 0.00166606274433434, 0.00914640724658966, 0.0011125049786642194, 0.0025191116146743298, -0.0023651316296309233, -0.005099814850836992, -0.0016753015806898475, -0.010452156886458397, 0.013747327961027622, -0.007230897434055805, 0.01282344851642847, -0.0008576681721024215, -0.026016447693109512, 0.007926886901259422, -0.019413789734244347, -0.007982319220900536, 0.0324220135807991, 0.0006686578271910548, 0.01265099085867405, 0.0070645990781486034, 0.029145320877432823, 0.007840657606720924, 0.006984529551118612, 0.011191261000931263, -0.014843665063381195, 0.011462265625596046, -0.002743922173976898, 0.005817361641675234, 0.011868772096931934, -0.009442049078643322, -0.012724900618195534, -0.000588203314691782, 0.015410310588777065, 0.017849352210760117, -0.023047715425491333, -0.007224738132208586, 0.013710372149944305, 0.0027239047922194004, -0.02867722138762474, -0.024796927347779274, 0.02095358818769455, -0.00500126788392663, 0.022222382947802544, -0.003892612410709262, 0.016494330018758774, 0.0035169015172868967, -0.00042690933332778513, 0.015126987360417843, 0.015422629192471504, -0.0012441577855497599, 0.018711639568209648, 0.001040134346112609, -0.006947574205696583, -0.0052414764650166035, -0.007477265316992998, 0.0186254121363163, -0.0025191116146743298, 0.017812397330999374, -0.015940001234412193, 0.009571392089128494, 0.027248287573456764, -0.00333520513959229, -0.0004211351042613387, -0.033801671117544174, -0.003535378957167268, 0.010556863620877266, 0.028480127453804016, 0.011302126571536064, 0.00845657754689455, 0.009903988800942898, -0.004490054678171873, 0.004813412204384804, -0.06050794944167137, -0.019031917676329613, 0.01899496279656887, -0.01574290730059147, -0.0028363103047013283, -0.0101257199421525, -0.00918952189385891, 0.020263757556676865, 0.023331038653850555, 0.01257708016782999, -0.03449150174856186, -0.0263120885938406, 0.011332922615110874, -0.011523857712745667, 0.027839569374918938, 0.009010905399918556, 0.03964059054851532, 0.008598239161074162, -0.012490851804614067, -0.0005189123330637813, 0.013808919116854668, -0.009060178883373737, -0.022690482437610626, 0.02478460781276226, 0.017750805243849754, 0.02177892066538334, -0.0014966848539188504, 0.01095105241984129, -0.012281438335776329, 0.024686060845851898, -0.002352813258767128, -0.009442049078643322, 0.010821709409356117, -0.011394514702260494, -0.0017738487804308534, 0.007150827441364527, -0.009768486022949219, -0.006411724258214235, 0.044099848717451096, -0.006239266600459814, -0.004203652031719685, -0.0010724702151492238, -0.023121625185012817, 0.02616426907479763, 0.014868301339447498, -0.0006586491363123059, 0.0061530377715826035, -0.03808847442269325, 0.012182891368865967, 0.014264700002968311, -0.005247635766863823, 0.0338263101875782, 0.011979637667536736, -0.0017753884894773364, -0.013648780062794685, 0.020657947286963463, -0.0021649578120559454, -0.01924133114516735, -0.005592550616711378, -0.027248287573456764, 0.008296438492834568, -0.00930038746446371, -0.017270388081669807, 0.009232636541128159, -0.011653200723230839, 0.01678997091948986, -0.019955798983573914, -0.011086554266512394, 0.011690155602991581, 0.028455490246415138, 0.0020864279940724373, -0.006343972869217396, 0.003978841006755829, 0.018305134028196335, 0.0006817461107857525, -0.009515958838164806, -0.0008345711976289749, 0.03596971184015274, -0.0109818484634161, -0.013944421894848347, 0.0024390420876443386, -0.0024621388874948025, -0.004764138720929623, 0.0196355190128088, 0.004252925515174866, 0.01654360257089138, 0.023688271641731262, 0.0015921524027362466, -0.0004627096641343087, -0.011801021173596382, 0.011141987517476082, 0.020251439884305, 0.014991485513746738, -0.004533168859779835, 0.00379098579287529, -0.020596355199813843, 0.02432882785797119, -0.007797543425112963, 0.000923109648283571, 0.021360095590353012, -0.021360095590353012, -0.01433861069381237, 0.018391361460089684, 0.02956414595246315, -0.02070721983909607, -0.0004311437951400876, -0.0019232092890888453, 0.013143726624548435, -0.008850766345858574, 0.014190790243446827, 0.006399405654519796, -0.013427048921585083, 0.002977971686050296, 2.3963126295711845e-05, -0.024981703609228134, 0.012885039672255516, 0.00925111398100853, -0.005213760305196047, -0.01605086773633957, -0.001494375173933804, -0.009793123230338097, -0.031017715111374855, -0.012447737157344818, 0.033161114901304245, 0.027346834540367126, 0.015755224972963333, 0.0071138725616037846, -0.00020999011758249253, -0.00330440909601748, -0.010618455708026886, -0.006399405654519796, -0.009731531143188477, 0.012330712750554085, 0.0009269591537304223, -0.01678997091948986, -0.023367993533611298, 0.002794735599309206, -0.0031519690528512, -0.007760588079690933, 0.012921995483338833, 0.006892141420394182, 0.010384405963122845, 0.002289681462571025, 0.018945690244436264, 0.02542516589164734, 0.007822180166840553, -0.020608672872185707, 0.01780007965862751, 0.008752218447625637, -0.013353139162063599, -0.007662041112780571, 0.0006767417653463781, -0.01161008607596159, -0.007896090857684612, 0.03343212231993675, -0.010310495272278786, -0.0015020741848275065, -0.001994040096178651, 0.033161114901304245, -0.015915364027023315, -0.008080866187810898, -0.0010924875969067216, 0.011289807967841625, -0.011160464957356453, -0.0011702474439516664, 0.01151769794523716, 0.009546754881739616, 0.003640085458755493, -0.023762181401252747, -0.0014273938722908497, -0.0050844172947108746, -0.012472373433411121, 0.025967175140976906, 0.008733741007745266, 0.003926488105207682, 0.012108981609344482, -0.019167421385645866, -0.005355421919375658, 0.022616570815443993, 0.004499293398112059, 0.012287598103284836, 0.026090357452630997, 0.010433679446578026, 0.0016691423952579498, 0.010741639882326126, -0.011259011924266815, 0.009706894867122173, 0.0022342486772686243, -0.005574073176831007, 0.013488641008734703, 0.005878953263163567, -0.032766927033662796, 0.00011433009058237076, 0.010144197382032871, 0.012312234379351139, 0.017331980168819427, -0.01258939877152443, 0.013242273591458797, -0.035058148205280304, 0.0036031301133334637, -0.00333212548866868, -0.015570449642837048, 0.023318719118833542, 0.0014866761630401015, 0.004490054678171873, 0.005087496712803841, -0.012860403396189213, -0.004145139362663031, -8.117629477055743e-05, -0.005099814850836992, 0.009503641165792942, -0.041956450790166855, -0.023552769795060158, -0.00897394958883524, -0.005447809584438801, -0.040478240698575974, -0.00552172027528286, -0.006128401029855013, 0.0053307851776480675, 0.015324082225561142, -0.007046121172606945, 0.02171732857823372, 0.038556572049856186, 0.017541393637657166, 0.0036154487170279026, -0.02307235263288021, -0.0010917176259681582, -0.003871055319905281, -0.001964783761650324, -0.015705952420830727, -0.003399876644834876, 0.012503169476985931, -0.0001480132050346583, -0.00758197158575058, -0.00094004743732512, -0.026459909975528717, -0.0109818484634161, -0.0075942897237837315, 0.014819027855992317, 0.018637729808688164, 0.0061715152114629745, 0.004382268525660038, 0.00047502806410193443, 0.004052751697599888, -0.005700336769223213, -0.030500343069434166, -0.004582442343235016, 0.016038548201322556, -0.03860584646463394, 0.010211948305368423, 0.010347451083362103, 0.017873989418148994, 0.0006748170126229525, 0.011425310745835304, -0.007212419528514147, -0.023047715425491333, 0.008105503395199776, 0.02867722138762474, -0.01895800791680813, 0.017529074102640152, 0.011431469582021236, 0.015853773802518845, -0.005259954370558262, -9.7969728813041e-05, 0.025770079344511032, 0.001618328969925642, -0.0003506892826408148, 0.02410709671676159, 0.013377775438129902, 0.016555922105908394, -0.007846817374229431, 0.009694576263427734, 0.009423571638762951, -0.023121625185012817, -3.115687650279142e-05, -0.007101553957909346, -0.005601789802312851, 0.011985797435045242, -0.0037971448618918657, 0.010532226413488388, 0.01327922847121954, 0.009596029296517372, 0.01510235108435154, 0.02971196547150612, -0.004018876235932112, 0.003479946171864867, 0.00039919294067658484, 0.0017923263367265463, 0.018403680995106697, -0.004628636408597231, -0.01433861069381237, 0.0023635919205844402, 0.007754428777843714, 0.0009023223537951708, 0.03355530649423599, 0.008191731758415699, -0.022554978728294373, 0.01479439064860344, -0.00045732036232948303, -0.030524980276823044, -0.01666678674519062, 0.010963371023535728, -0.02288757637143135, 0.020103618502616882, 0.011049599386751652, 0.015003804117441177, 0.02177892066538334, -0.02062099054455757, -0.004757979419082403, 0.012145936489105225, 0.009078656323254108, -0.004631716292351484, -0.010242744348943233, 0.030229339376091957, 0.03697981685400009, -0.0017107169842347503, -0.0016568240243941545, 0.007772906683385372, 0.020054345950484276, -0.0006455608527176082, 0.003452229779213667, 0.01807108335196972, 0.0033690808340907097, -0.024341145530343056, 0.00987935159355402, -0.00679359445348382, 0.013747327961027622, 0.029416324570775032, 0.013673417270183563, -0.00758197158575058, 0.014523386023938656, 0.02288757637143135, 0.013969059102237225, 0.005604869220405817, -0.006534907966852188, 0.039936233311891556, 0.007249374873936176, 0.0018924132455140352, -0.0059220679104328156, 0.00920799933373928, 0.013611825183033943, 0.016888517886400223, 0.03446686640381813, 0.0028008949011564255, 0.0002250031684525311, 0.002830151002854109, -0.001488985843025148, -0.015496539883315563, 0.013759645633399487, -0.03459005057811737, -0.012934313155710697, -0.009750008583068848, -0.034048039466142654, 0.010304336436092854, 0.013648780062794685, 0.010285858996212482, 0.0025699250400066376, 0.007206260226666927, -0.011406832374632359, -0.0033783195540308952, -0.0009046320337802172, 0.0011040359968319535, -0.028233759105205536, 0.01188725046813488, -0.006916778162121773, 0.009854715317487717, -0.014227745123207569, -0.0020879677031189203, 0.0021449404302984476, 0.054545849561691284, 0.01639578305184841, 0.008277961052954197, -0.009589869529008865, -0.006319336127489805, 0.0012387684546411037, 0.014548023231327534, 0.011973478831350803, 0.00424368679523468, -0.010470635257661343, 0.036043621599674225, -0.005534038413316011, 0.013870511204004288, -0.0025899424217641354, -0.00834571197628975, 0.017541393637657166, -0.003741712076589465, -0.013796601444482803, 0.0020155971869826317, 0.01872395910322666, 0.0014589597703889012, 0.028972862288355827, 0.005915908608585596, 0.00549092423170805, 0.015767544507980347, -0.008604397997260094, 0.0016537443734705448, -0.02062099054455757, 0.009448207914829254, 0.011086554266512394, -0.016654469072818756, -0.009010905399918556, 0.015730589628219604, -0.011259011924266815, 0.00578964501619339, 0.0004253695369698107, 0.004462338052690029, -0.0043915072456002235, -0.01170247420668602, 0.008204050362110138, 0.001003179233521223, 0.023786818608641624, 0.016777653247117996, 0.013094453141093254, 0.0109818484634161, 0.021421687677502632, 0.002520651323720813, 0.00020556319213937968, -0.003476866753771901, 0.006297778803855181, -0.010181152261793613, 0.004927357658743858, 0.00530614797025919, 0.015397991985082626, 0.01095105241984129, 0.0052568744868040085, -0.001990960445255041, 0.0031781457364559174, -0.02983514964580536, -0.013636462390422821, -0.020855041220784187, 0.0061684357933700085, -0.0038002245128154755, -0.011363718658685684, 0.012256802059710026, -0.04114343598484993, -0.000792226695921272, -0.015890728682279587, -0.016740696504712105, -0.019709430634975433, -0.0052999891340732574, -0.0319785512983799, -0.008481213822960854, -0.004985869862139225, 0.024550558999180794, 0.0007645103032700717, 0.0012010433711111546, 0.006541067268699408, -0.006362450309097767, 0.023565087467432022, -0.005863555707037449, 0.009639143012464046, -0.003136571031063795, 0.033407483249902725, -0.00745878741145134, -0.014892938546836376, -0.028406215831637383, -0.003135031322017312, 0.012016593478620052, -0.002220390597358346, 0.00833339337259531, 0.014868301339447498, -0.005047461949288845, 0.017997173592448235, -0.04244918376207352, -0.014387884177267551, -0.001355023356154561, -0.007384877186268568, 0.0016753015806898475, -0.0020202165469527245, 0.008887721225619316, 0.0036493241786956787, -0.009811600670218468, -0.00933118350803852, 0.0232817642390728, 0.008290278725326061, 0.014141515828669071, -0.01693779230117798, 0.011006484739482403, -0.00833339337259531, -0.001210282207466662, -0.0035415382590144873, -0.014745117165148258, 0.0044654179364442825, 0.01154849398881197, 0.010119560174643993, 0.005293829832226038, 0.01527480874210596, 0.019315240904688835, 0.0002259655448142439, 0.00836418941617012, 0.0019216694636270404, -0.034910328686237335, -0.008234846405684948, -0.003972682170569897, 0.018637729808688164, -0.01856382004916668, 0.006805912591516972, 0.011751747690141201, 0.0012895818799734116, 0.007261693011969328, 0.01339009404182434, -0.02175428345799446, 0.015693634748458862, -0.0036831998731940985, -0.03685663640499115, 0.004988949745893478, 0.002291221171617508, -0.0010670808842405677, 0.0037540304474532604, 0.005176804959774017, 0.0013534835306927562, 0.011357558891177177, 0.0030857576057314873, 0.007058439776301384, -0.03890148922801018, 0.007015325129032135, -0.024858519434928894, -0.010556863620877266, 0.014819027855992317, 0.01252780668437481, 0.02668164111673832, 0.0029656533151865005, 0.003102695569396019, 0.0019894205033779144, -0.009842396713793278, -0.007939204573631287, 0.00906633771955967, -0.0027685589157044888, 4.638645259547047e-05, -0.003892612410709262, -0.010076446458697319, -0.010039490647614002, -0.0009300387464463711, 0.004114343784749508, 0.01914278417825699, 0.017479801550507545, -0.007945364341139793, -0.006356291472911835, -0.0046378751285374165, -0.01491757482290268, 0.011776384897530079, 0.025696169584989548, 0.0025129523128271103, -0.01565667800605297, 0.007834498770534992, 0.013821237720549107, -0.01334082055836916, 0.004551646765321493, 0.01788630709052086, 0.005897431168705225, -0.0016537443734705448, -0.016839245334267616, 0.006374768912792206, -0.0016968587879091501, -0.004292960278689861, 0.022012969478964806, -0.012460055761039257, 0.019500017166137695, 0.009023224003612995, -0.00853048823773861, -0.0016044707736000419, 0.01798485592007637, 0.022961486130952835, -0.003550776978954673, -0.009140248410403728, 0.01780007965862751, 0.004619397688657045, 0.006830549333244562, -0.017171841114759445, 0.023417266085743904, -0.011530016548931599, 0.005900510586798191, 0.004061990417540073, 0.020386941730976105, 0.004761059302836657, 0.009423571638762951, 0.001452030730433762, 0.004234448075294495, -0.0032120211981236935, -0.01924133114516735, 0.01075395755469799, -0.004175935406237841, -0.014634251594543457, 0.0014343230286613107, 0.014055287465453148, 0.006504111923277378, 0.011209738440811634, -0.011708633042871952, 0.035403065383434296, -0.0007491123396903276, -0.0017137965187430382, -0.036807361990213394, 0.034540776163339615, 0.012312234379351139, -0.016284916549921036, -0.0011209738440811634, -0.008431940339505672, 0.0032058621291071177, -0.01580449938774109, 0.012004274874925613, 0.005771167576313019, -0.0008592079975642264, -0.0011048059677705169, 0.005161406937986612, -0.024131733924150467, -0.0015413390938192606, 0.02742074429988861, -0.014634251594543457, 0.012189051136374474, -0.016679106280207634, -0.015484221279621124, 0.009232636541128159, 0.01945074461400509, -0.018329769372940063, -0.0003762884298339486, 0.008031592704355717, -0.023294083774089813, -0.0031273323111236095, -0.015348718501627445, -0.0249077919870615, -0.009645302779972553, 0.0014920654939487576, -0.002211151644587517, -0.008154776878654957, 0.0017630701186135411, -0.0005485534784384072, 0.0016368066426366568, -0.0032489763107150793, -0.014880619943141937, 0.010298177599906921, 0.019475379958748817, 0.0025760841090232134, 0.0011602387530729175, -0.015065396204590797, 0.01000253576785326, -0.03128872066736221, 0.007187782786786556, 0.017393572255969048, 0.0047302632592618465, 0.0021141443867236376, -0.016321871429681778, 0.0069906883873045444, 0.010082605294883251, 0.009965580888092518, -0.0028686460573226213, -5.100584894535132e-05, 0.0116408821195364, -0.020103618502616882, -0.00894315354526043, -0.003840259276330471, -0.027987390756607056, 0.01804644800722599, -0.015607405453920364, 0.007822180166840553, -0.009786964394152164, 0.020534763112664223, 0.009164884686470032, 0.013439367525279522, 0.02441505715250969, -0.033161114901304245, 0.01510235108435154, -0.0034953441936522722, -0.005441650282591581, 0.019660156220197678, -0.014991485513746738, -0.013944421894848347, 0.013045178726315498, -0.007015325129032135, 0.0006875203689560294, 0.010211948305368423, -0.014843665063381195, -0.02095358818769455, 0.007107713259756565, 0.010575341060757637, 0.01516394317150116, -0.022542661055922508, -0.019918842241168022, -0.0055525158531963825, 0.007255534175783396, 0.015016121789813042, 0.010027172975242138, -0.004933516960591078, 0.017381254583597183, 0.012275279499590397, -0.023503495380282402, -0.006658092141151428, -0.022173108533024788, 0.0041389805264770985, 0.023047715425491333, 0.01611245982348919, -0.023885365575551987, -0.022998441010713577, 0.012373826466500759, 0.002626897534355521, 0.012096663005650043, -0.012207528576254845, 0.0059066698886454105, -0.023380311205983162, 0.013291547074913979, 0.00503206392750144, -0.023491177707910538, 0.0061684357933700085, -0.004545487463474274, 0.01073548011481762, 0.0013912086142227054, -0.016950109973549843, -0.013032861053943634, 0.009571392089128494, -0.02089199610054493, -1.4435617856634053e-07, 0.003926488105207682, -0.0002738917828537524, 0.007434150669723749, 0.003316727466881275, 0.0050536212511360645, 0.003218180499970913, -0.00908481515944004, -0.005229158326983452, -0.0216557364910841, 0.005971341393887997, -0.0036154487170279026, 0.013993695378303528, 0.01263867225497961, -0.008819970302283764, 0.010082605294883251, -0.013094453141093254, 0.011603927239775658, -0.005105974152684212, -0.03261910751461983, -0.001528250752016902, -0.017442846670746803, -0.007760588079690933, -0.023996232077479362, -0.00821020919829607, -0.006824390031397343, -0.00273006409406662, -0.006528748665004969, -0.011024962179362774, 0.03835947811603546, 0.016383463516831398, -0.020325349643826485, 0.01794789917767048, -0.005004347302019596, -0.009719212539494038, 0.003298250027000904, 0.020818086341023445, 0.00936197955161333, 0.013131408020853996, -0.0010077985934913158, 0.014055287465453148, 0.0027762579265981913, 0.0058604758232831955, 0.0031812251545488834, 0.02128618396818638, -0.0070645990781486034, -0.001037054811604321, -0.020879678428173065, -0.02628745324909687, -0.0008314916049130261, -0.012971268966794014, -0.015311763621866703, 0.001166397938504815, 0.004296039696782827, -0.02604108490049839, -0.021926742047071457, -0.018305134028196335, 0.019105829298496246, -0.018699321895837784, 0.02184051275253296, -0.03747255355119705, 0.010445998050272465, -0.030894532799720764, -0.01334082055836916, 0.010747798718512058, 0.013574870303273201, 0.004764138720929623, -0.0032089415472000837, 0.00033721604268066585, -0.012355349026620388, -0.0033690808340907097, 0.010310495272278786, 0.014498749747872353, 0.015114669688045979, 0.004246766213327646, 0.009836237877607346, 0.01000253576785326, -0.012429259717464447, -0.009103293530642986, -0.01115430612117052, 0.004425383172929287, 0.005952863954007626, 0.01788630709052086, -0.00457936292514205, 0.0019232092890888453, -0.015607405453920364, 0.01685156300663948, 0.0037201549857854843, -0.003892612410709262, -0.0076250857673585415, -0.014609615318477154, 0.012958950363099575, 0.0031904641073197126, 0.009663780219852924, -0.000433453475125134, 0.016716061159968376, 0.01409224234521389, 0.019044237211346626, -0.001783087500371039, -0.011332922615110874, -0.000532000616658479, -0.0007256303797475994, 0.0036831998731940985, 0.00854896567761898, 0.004440781194716692, -0.017824716866016388, -0.004413064569234848, 0.007193942088633776, 0.014708162285387516, -0.011172783561050892, 0.0029487155843526125, 0.006036012899130583, 0.004591681528836489, -0.016580557450652122, -0.01685156300663948, -0.019647838547825813, 0.0040496718138456345, -0.008154776878654957, -0.008259482681751251, 0.016592876985669136, 0.0017907865112647414, 0.015200898051261902, -0.011905727908015251, 0.01095105241984129, 0.02184051275253296, 0.01678997091948986, 0.0012226005783304572, 0.009195680730044842, 0.00552172027528286, 0.01321763638406992, 0.03387558460235596, -0.010994166135787964, -0.019623201340436935, -0.020633310079574585, 0.05144161358475685, -0.03072207421064377, 0.004570124205201864, -0.016950109973549843, -0.002725444734096527, 0.008031592704355717, -0.0012572461273521185, -0.009398934431374073, 0.0018523784819990396, 0.00212184339761734, -0.018674684688448906, 0.00536774005740881, 0.00012982432963326573, -0.004810332786291838, -0.011881090700626373, -0.0003658947825897485, 0.003544617909938097, -0.013193000108003616, 0.010150356218218803, 0.022567298263311386, -0.009232636541128159, 0.017689213156700134, 0.008265642449259758, -0.02815984934568405, 0.021852830424904823, 0.0037509510293602943, 0.004135900642722845, 0.005527879111468792, -0.02779029682278633, 0.023367993533611298, -0.011665519326925278, 0.001699938322417438, 0.008930835872888565, 0.006424042396247387, 0.017639940604567528, -0.012533965520560741, 0.007304807659238577, 0.0091587258502841, 0.016605194658041, -0.024772290140390396, 0.009460526518523693, 0.014252381399273872, -0.007698995992541313, 0.004739501979202032, -0.007846817374229431, 0.004862686153501272, -0.009688416495919228, -0.007816021330654621, 0.03991159796714783, 1.1771043318731245e-05, 0.01027970016002655, 0.008555124513804913, 0.002908680820837617, -0.007033803034573793, -0.022197745740413666, -0.007538856938481331, 0.010107242502272129, 0.010107242502272129, 0.0299583338201046, -0.01167167816311121, 0.017652258276939392, 0.012065866962075233, -0.008259482681751251, -0.014572660438716412, -0.003267453983426094, -0.01621100679039955, -0.014584978111088276, -0.026484547182917595, 0.009497481398284435, -0.004825730808079243, 0.0005154478130862117, 0.00026927239377982914, -0.027001919224858284, 0.025449801236391068, 0.011252853088080883, 0.010236585512757301, 0.011234375648200512, 0.020337669178843498, 0.0029656533151865005, -0.01819426752626896, 0.01151769794523716, -0.0031627477146685123, 0.004037353675812483, 0.0032766927033662796, -0.02126154862344265, 0.006781275849789381, -0.003994239028543234, 0.011683996766805649, 0.04843592643737793, 0.013254592195153236, 0.02171732857823372, 0.014190790243446827, 0.0183174517005682, -0.0024729175493121147, -0.008087025955319405, 0.00048272707499563694, 0.021520234644412994, -0.005724973510950804, -0.002089507644996047, 0.02591790072619915, -0.014695843681693077, 0.009442049078643322, 0.020276077091693878, -0.005965182092040777, -0.016445055603981018, -0.00751422019675374, -0.016075504943728447, 0.0059374659322202206, 0.005872794426977634, -0.003630846505984664, -0.014597296714782715, 0.010698525235056877, 0.007206260226666927, -0.0006617287290282547, -0.0012718741782009602, -0.01064309198409319, -0.012373826466500759, -0.004770298022776842, -0.02818448469042778, -0.012601717375218868, 0.022899894043803215, -0.007446469273418188, 0.016740696504712105, 0.0019370674854144454, -0.024759972468018532, -0.015373355709016323, -0.021273866295814514, 0.017639940604567528, -0.002565305447205901, 0.002811673330143094, -0.017196478322148323, -0.003104235278442502, 0.00836418941617012, 0.020473171025514603, -0.0015598166501149535, -0.015299445018172264, 0.02843085303902626, -0.0015644360100850463, -0.008653671480715275, 0.029243867844343185, -0.011222057044506073, -0.02153255231678486, -0.02039925940334797, 0.012490851804614067, -0.0016229484463110566, -0.00041805548244155943, 0.007230897434055805, -0.021976014599204063, -0.006202311255037785, 0.01706097647547722, -0.012614035047590733, -0.0071138725616037846, -0.002169577172026038, 0.011431469582021236, 0.010396724566817284, -0.026854097843170166, -0.018760913982987404, 0.010673888027668, 0.006534907966852188, 0.005195282865315676, 0.010162674821913242, -0.0384826622903347, -0.0015005343593657017, -0.007323285099118948, 0.0150900324806571, -0.01096952985972166, -0.007828339003026485, -0.011881090700626373, 0.002905601169914007, -0.006596500054001808, 0.01967247575521469, 0.03394949436187744, 0.005250715184956789, -0.022012969478964806, -0.034688595682382584, 0.003951124846935272, 0.012201368808746338, -0.006516430526971817, -0.009368138387799263, 0.01687620021402836, -0.012687945738434792, -0.014942212030291557, 0.00918952189385891, 0.02126154862344265, 0.01924133114516735, 0.016839245334267616, 0.011893409304320812, -0.0034891851246356964, 0.03560015931725502, 0.000689445121679455, -0.022283975034952164, 0.0033228867687284946, 0.012848084792494774, 0.043212924152612686, 0.004899641033262014, 0.016987064853310585, -0.007735951337963343, 0.01926596835255623, 0.04380420967936516, 0.007218578830361366, -0.00333212548866868, 0.0034399114083498716, -0.007859135046601295, -0.002973352326080203, -0.022530343383550644, -0.0034953441936522722, -0.025696169584989548, -0.0070214844308793545, -0.0014358627377077937, 0.009306546300649643, 0.014535704627633095, 0.0010501430369913578, -0.0034399114083498716, 0.001127133029513061, 0.00761276762932539, -0.003282852005213499, -0.0007783684995956719, -0.022752072662115097, -0.02423028089106083, 0.013082134537398815, -0.009392775595188141, 0.00676279840990901, -0.0009508260409347713, -0.01000253576785326, 0.006565704010426998, 0.023121625185012817, -0.009589869529008865, 0.008598239161074162, -0.0044192238710820675, 0.020904313772916794, -0.02705119363963604, 0.021705010905861855, -0.019906524568796158, 0.008475054986774921, 0.01245389599353075, -0.0062485053204, -0.0021495597902685404, 0.00752037949860096, 0.00287942448630929, -0.0058881924487650394, 0.008900039829313755, -0.008419622667133808, 0.0058758738450706005, -0.008770696818828583, 0.0007622006232850254, 0.0011617785785347223, -0.003019546391442418, 0.00015561595500912517, 0.00015263259410858154, 0.0018847142346203327, -0.011979637667536736, 0.0032489763107150793, 0.0006116852746345103, 0.00037609596620313823, -0.013722690753638744, 0.012398463673889637, 0.02453824132680893, -0.012774174101650715, -0.002616118872538209, -0.008105503395199776, -0.009540596045553684, -0.0030934566166251898, 0.0061561171896755695, -0.013082134537398815, -0.005761928856372833, 0.026336725801229477, 0.014252381399273872, -0.013168362900614738, 0.030229339376091957, 0.006892141420394182, 0.007742110639810562, 0.005407774820923805, -0.008819970302283764, 0.0041235825046896935, -0.0027700988575816154, 0.004539328161627054, -0.0030795985367149115, 0.0074218325316905975, -0.008789174258708954, -0.03141190484166145, -0.0028132132720202208, 0.02346654050052166, 0.007181623484939337, 0.022074561566114426, 0.005780406296253204, -0.009466685354709625, -0.005115212872624397, 0.0007826029905118048, -0.00022269345936365426, -0.002154179150238633, -0.0036185281351208687, 0.017381254583597183, 0.0004203651915304363, -0.006892141420394182, 0.011573131196200848, 0.017208795994520187, -0.01510235108435154, -0.0008830748847685754, 0.012724900618195534, -0.008093184791505337, -0.015422629192471504, -0.01967247575521469, 0.015767544507980347, 0.008592079393565655, -0.021668054163455963, 0.0007799083250574768, 0.008154776878654957, 0.030229339376091957, 0.024562876671552658, -0.004296039696782827, 0.0012888119090348482, 0.005016665905714035, 0.003319807117804885, -0.012065866962075233, 0.00836418941617012, 0.029514871537685394, 0.002111064735800028, -0.02451360411942005, 0.02092895098030567, -0.012016593478620052, 0.005734212230890989, 0.004511612001806498, 0.03345675766468048, 0.003202782478183508, 0.01018731202930212, 0.015114669688045979, -0.013476323336362839, 0.012312234379351139, 0.016826925799250603, 0.01072316151112318, -0.0031273323111236095, -0.004733342677354813, 0.018674684688448906, 0.00082610227400437, -0.002977971686050296, -0.0010686207097023726, -0.01282344851642847, 0.014227745123207569, 0.001004719058983028, -0.0064917937852442265, -0.0058881924487650394, 0.008961631916463375, 0.01015651598572731, -0.013106770813465118, -0.00575884897261858, 0.002808593912050128, -0.0005624116747640073, 0.0010416741715744138, 0.0026407556142657995, -0.004416143987327814, 0.0056726206094026566, -0.00997173972427845, -0.003415274666622281, -0.02414405159652233, -0.0058204410597682, 0.0016706822207197547, 0.015755224972963333, -0.016826925799250603, -0.0034214339684695005, 0.0020679503213614225, 0.0313626304268837, -0.011659359559416771, -0.019783340394496918, -0.0028332306537777185, -0.024759972468018532, -0.007292489055544138, -0.006935255602002144, 0.026854097843170166, -0.008006956428289413, -0.01239230390638113, -0.0034891851246356964, 0.00015090031956788152, 0.01574290730059147, 0.0041697765700519085, 0.01428933721035719, -0.008358030579984188, 0.027765659615397453, -0.012275279499590397, -0.004098945762962103, 0.004588601645082235, -0.006066808942705393, -0.0027901162393391132, 0.00083303137216717, -0.028283031657338142, 0.01270026434212923, 0.008598239161074162, 0.011807180941104889, -0.005811202339828014, 0.0022819824516773224, 0.02126154862344265, -0.008758378215134144, 0.016740696504712105, -0.007033803034573793, -0.013574870303273201, -0.013550233095884323, 0.01409224234521389, -0.008512009866535664, 0.023244809359312057, 0.0014843664830550551, 0.00817941315472126, -0.021150682121515274, -0.023946957662701607, -0.03128872066736221, 0.0162972342222929, 0.027741022408008575, -0.005417013540863991, -0.01648201048374176, -0.0024421215057373047, 0.004055831115692854, -0.011252853088080883, -0.03229882940649986, -0.034023404121398926, -0.005278431810438633, -0.021544871851801872, -0.014806709252297878, -0.007508060894906521, 0.021236911416053772, -0.0015906125772744417, 0.002609959803521633, -0.005481685511767864, 0.013661098666489124, 0.0005504782311618328, -0.03284083679318428, 0.003452229779213667, -0.003221259918063879, 0.021347776055336, 0.0044069052673876286, 0.0012295297347009182, -0.0005797343910671771, 0.013106770813465118, -0.0032397375907748938, 0.008247165009379387, 0.022431794553995132, 0.003624687436968088, -0.0014358627377077937, -0.01887177862226963, -0.008746059611439705, -0.02006666362285614, 0.011431469582021236, -0.011295966804027557, -0.014548023231327534, 0.007816021330654621, 0.005438570864498615, 0.016605194658041, -0.008050070144236088, 0.0183174517005682, 8.603628521086648e-05, 0.0132669098675251, 0.010963371023535728, 0.0052999891340732574, -0.007920727133750916, -0.002385149011388421, 0.012102821841835976, 0.0011864153202623129, 0.02641063742339611, 0.005857396405190229, 0.005263033788651228, -0.013242273591458797, -0.0036708812694996595, 0.012835766188800335, 0.03254519775509834, 0.0027716385666280985, -0.011690155602991581, 0.019229013472795486, -0.005072098691016436, -0.01339009404182434, 0.010088765062391758, -0.020842721685767174, 0.02705119363963604, 0.0008699865429662168, -0.017282705754041672, -0.010421360842883587, 0.002306619193404913, -0.01088946033269167, -0.014658888801932335, 0.0061869132332503796, 0.006202311255037785, 0.028652584180235863, -0.00914640724658966, 0.00851816963404417, 0.004132821224629879, 0.007089235819876194, 0.012072025798261166, -0.003578493371605873, -0.040059417486190796, -0.0019848011434078217, 0.010476794093847275, 0.00166606274433434, 0.017837034538388252, 0.0017877068603411317, -0.02944096177816391, -0.006430201698094606, -0.0040958658792078495, -0.01780007965862751, -0.005164486821740866, 0.004508532118052244, 0.006072968244552612, 0.0007333293906413019, 0.017935581505298615, 0.0021988332737237215, 0.008672148920595646, -0.008321074768900871, 0.0036062097642570734, 0.017344297841191292, -0.015940001234412193, 0.0042498460970819, 0.008234846405684948, 0.008869243785738945, -0.03577261418104172, -0.014449476264417171, -0.002155718859285116, 0.0026607729960232973, -0.01835440658032894, -0.003082678187638521, -0.008185572922229767, -0.0067443205043673515, -0.029909061267971992, 0.011111191473901272, -0.0324220135807991, -0.007243215572088957, 0.01340241264551878, -0.0012757236836478114, -0.03902467340230942, 0.003087297547608614, 0.021335458382964134, -0.009417411871254444, -0.012330712750554085, -0.00020344597578514367, 0.015607405453920364, 0.006676569581031799, 0.015484221279621124, -0.01681460812687874, 0.01568131521344185, 0.012416941113770008, -0.008321074768900871, -0.014449476264417171, -0.016494330018758774, 0.00470254709944129, -0.019623201340436935, 0.03005688078701496, 0.010821709409356117, -0.0014019871596246958, -0.001157929073087871, 0.014560341835021973, 0.007126190699636936, 0.006313176825642586, -0.00233587552793324, 0.011037280783057213, -0.01358718890696764, 0.033161114901304245, 0.008265642449259758, -0.022505706176161766, -0.0035322995390743017, 0.011271330527961254, 0.0265338197350502, -0.011985797435045242, -0.005133690778166056, 0.007865294814109802, -0.011302126571536064, 0.020904313772916794, 0.01635882630944252, 0.023257127031683922, 0.002531429985538125, -0.012687945738434792, 0.01773848757147789, -0.011573131196200848, -0.0150900324806571, 0.0017907865112647414, -0.001169477473013103, 0.005934386048465967, 0.003135031322017312, 0.005016665905714035, 0.015311763621866703, -0.023331038653850555, -0.007267852313816547, -0.012687945738434792, 0.005149088799953461, 0.0027839569374918938, -0.013045178726315498, 0.0017230353550985456, -0.0038156225346028805, -0.0013842794578522444, -0.006313176825642586, -0.004890402313321829, -0.01693779230117798, 0.018945690244436264, -0.012835766188800335, -0.01592768356204033, -0.00627930136397481, 0.020349986851215363, 0.012084344401955605, 0.005472446326166391, -0.014868301339447498, 0.009823919273912907, -0.01258939877152443, 0.0011841056402772665, -0.009454367682337761, 0.017602985724806786, 0.016247961670160294, 0.027617838233709335, 0.010353609919548035, 0.016617514193058014, 0.0037817468401044607, 0.01282344851642847, -0.008752218447625637, -0.0029810513369739056, -0.01938915252685547, -0.004527009557932615, 0.01614941470324993, -0.006380928214639425, -0.007588130421936512, 0.012472373433411121, 0.0036739609204232693, -0.008105503395199776, 0.008893880061805248, 0.005013586487621069, 0.019807977601885796, 0.016321871429681778, 0.001599851413629949, 0.005666461307555437, -0.0014035269850865006, -0.004776457324624062, -0.008555124513804913, -0.001394288265146315, 0.01491757482290268, -0.005035143345594406, 0.014326292090117931, -0.0032458968926221132, 0.00663961423560977, 0.016186369583010674, 0.007249374873936176, -0.0005720354383811355, 0.009540596045553684, -0.01924133114516735, 0.0019155102781951427, -0.017837034538388252, 0.009171044453978539, -0.03594507277011871, -0.013550233095884323, -0.0015798340318724513, -0.00761276762932539, -0.028652584180235863, -0.004804173484444618, 0.005644903983920813, -0.004265244118869305, -0.0014443317195400596, 0.006442519836127758, 0.001830821274779737, -0.012540125288069248, 0.022899894043803215, -0.01611245982348919, -0.002272743731737137, 0.03520597144961357, 0.004363791085779667, 0.01332850195467472, 0.0031488894019275904, -0.0053338645957410336, 0.005413934122771025, -0.011924205347895622, 0.011505380272865295, 0.007933045737445354, 0.010944892652332783, 0.015595086850225925, -0.016962427645921707, -0.00624234601855278, 0.01635882630944252, -0.012010433711111546, -0.012053548358380795, 0.014597296714782715, -0.0009354280191473663, -0.00621770927682519, 0.005413934122771025], 'Britons have always had a special relationship with animals, especially considering pets as an integral part of their lifestyle.', '1bd998ea-9a10-4f0b-8328-75bddc885c1a', '94a73201-001a-4296-ab4f-cd4c4d98c44a']" + ] + }, + { + "cell_type": "code", + "execution_count": 149, + "id": "83767dbc-7ace-48ea-8c88-ee032a9304c7", + "metadata": {}, + "outputs": [], + "source": [ + "hits = qdrant.search(\n", + " collection_name='e3702209-2c4b-47c2-834f-25b3f92c41a7',\n", + " query_vector=(\n", + " \"content\", rr[1]\n", + " ),\n", + " limit=3,\n", + ")\n", + "for hit in hits:\n", + " print(hit)\n", + " print(hit.payload, \"score:\", hit.score)" + ] + }, + { + "cell_type": "code", + "execution_count": 154, + "id": "f4027ca9-ee51-4d22-846e-354f8f6e77d8", + "metadata": {}, + "outputs": [ + { + "ename": "UnexpectedResponse", + "evalue": "Unexpected Response: 400 (Bad Request)\nRaw response content:\nb'{\"status\":{\"error\":\"Format error in JSON body: EOF while parsing a value at line 1 column 0\"},\"time\":0.0}'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mUnexpectedResponse\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[154], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28mfilter\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;66;03m# Scroll through points in the collection\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m scroll_result \u001b[38;5;241m=\u001b[39m \u001b[43mqdrant\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhttp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpoints_api\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscroll_points\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[43m \u001b[49m\u001b[43mcollection_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcollection_name\u001b[49m\n\u001b[1;32m 13\u001b[0m \n\u001b[1;32m 14\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;66;03m# Print point ids and their vectors\u001b[39;00m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m point \u001b[38;5;129;01min\u001b[39;00m scroll_result\u001b[38;5;241m.\u001b[39mresult\u001b[38;5;241m.\u001b[39mpoints:\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/qdrant_client/http/api/points_api.py:1338\u001b[0m, in \u001b[0;36mSyncPointsApi.scroll_points\u001b[0;34m(self, collection_name, consistency, scroll_request)\u001b[0m\n\u001b[1;32m 1329\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mscroll_points\u001b[39m(\n\u001b[1;32m 1330\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 1331\u001b[0m collection_name: \u001b[38;5;28mstr\u001b[39m,\n\u001b[1;32m 1332\u001b[0m consistency: m\u001b[38;5;241m.\u001b[39mReadConsistency \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1333\u001b[0m scroll_request: m\u001b[38;5;241m.\u001b[39mScrollRequest \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1334\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m m\u001b[38;5;241m.\u001b[39mInlineResponse20014:\n\u001b[1;32m 1335\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1336\u001b[0m \u001b[38;5;124;03m Scroll request - paginate over all points which matches given filtering condition\u001b[39;00m\n\u001b[1;32m 1337\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1338\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_for_scroll_points\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1339\u001b[0m \u001b[43m \u001b[49m\u001b[43mcollection_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcollection_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1340\u001b[0m \u001b[43m \u001b[49m\u001b[43mconsistency\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconsistency\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1341\u001b[0m \u001b[43m \u001b[49m\u001b[43mscroll_request\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mscroll_request\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1342\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/qdrant_client/http/api/points_api.py:534\u001b[0m, in \u001b[0;36m_PointsApi._build_for_scroll_points\u001b[0;34m(self, collection_name, consistency, scroll_request)\u001b[0m\n\u001b[1;32m 532\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mContent-Type\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m headers:\n\u001b[1;32m 533\u001b[0m headers[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mContent-Type\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplication/json\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 534\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapi_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 535\u001b[0m \u001b[43m \u001b[49m\u001b[43mtype_\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mInlineResponse20014\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 536\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mPOST\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 537\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m/collections/\u001b[39;49m\u001b[38;5;132;43;01m{collection_name}\u001b[39;49;00m\u001b[38;5;124;43m/points/scroll\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 538\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 539\u001b[0m \u001b[43m \u001b[49m\u001b[43mpath_params\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpath_params\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 540\u001b[0m \u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquery_params\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 541\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 542\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/qdrant_client/http/api_client.py:74\u001b[0m, in \u001b[0;36mApiClient.request\u001b[0;34m(self, type_, method, url, path_params, **kwargs)\u001b[0m\n\u001b[1;32m 72\u001b[0m url \u001b[38;5;241m=\u001b[39m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhost \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;241m+\u001b[39m url\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mpath_params)\n\u001b[1;32m 73\u001b[0m request \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_client\u001b[38;5;241m.\u001b[39mbuild_request(method, url, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 74\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtype_\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Projects/cognee/.venv/lib/python3.11/site-packages/qdrant_client/http/api_client.py:97\u001b[0m, in \u001b[0;36mApiClient.send\u001b[0;34m(self, request, type_)\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ValidationError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 96\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ResponseHandlingException(e)\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m UnexpectedResponse\u001b[38;5;241m.\u001b[39mfor_response(response)\n", + "\u001b[0;31mUnexpectedResponse\u001b[0m: Unexpected Response: 400 (Bad Request)\nRaw response content:\nb'{\"status\":{\"error\":\"Format error in JSON body: EOF while parsing a value at line 1 column 0\"},\"time\":0.0}'" + ] + } + ], + "source": [ + "collection_name = 'e3702209-2c4b-47c2-834f-25b3f92c41a7'\n", + "\n", + "# Define a filter if needed, otherwise set to None\n", + "# Example filter: only retrieve points where the `category` field equals `example_category`\n", + "# filter = Filter(must=[FieldCondition(key=\"category\", match={\"value\": \"example_category\"})])\n", + "\n", + "# In this example, we're not using any filters\n", + "filter = None\n", + "\n", + "# Scroll through points in the collection\n", + "scroll_result = qdrant.http.points_api.scroll_points(\n", + " collection_name=collection_name\n", + "\n", + ")\n", + "\n", + "# Print point ids and their vectors\n", + "for point in scroll_result.result.points:\n", + " print(f\"Point ID: {point.id}, Vector: {point.vector}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 148, + "id": "cee9ecfe-ece9-40fc-8e8e-dfadeccb9eff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleting collection: e1728322-74d9-4b31-b909-82d864252d88\n", + "Collection 'e1728322-74d9-4b31-b909-82d864252d88' deleted successfully.\n", + "Deleting collection: 42c4e357-f1c3-4e8c-90e5-c56fb65cbdb5\n", + "Collection '42c4e357-f1c3-4e8c-90e5-c56fb65cbdb5' deleted successfully.\n", + "Deleting collection: 07256d64-2aec-48dd-9bf1-c4fb06729a74\n", + "Collection '07256d64-2aec-48dd-9bf1-c4fb06729a74' deleted successfully.\n", + "Deleting collection: 52de6d02-5b6f-4dd4-82a9-0fead1691400\n", + "Collection '52de6d02-5b6f-4dd4-82a9-0fead1691400' deleted successfully.\n", + "Deleting collection: d1e9000b-6db6-4b0c-8dd1-3d46d29253f4\n", + "Collection 'd1e9000b-6db6-4b0c-8dd1-3d46d29253f4' deleted successfully.\n", + "Deleting collection: 5073778b-f8e1-4eb5-b84f-aa28be7a7531\n", + "Collection '5073778b-f8e1-4eb5-b84f-aa28be7a7531' deleted successfully.\n", + "Deleting collection: 84f3a369-eea2-45c9-a82d-ff9748aa2446\n", + "Collection '84f3a369-eea2-45c9-a82d-ff9748aa2446' deleted successfully.\n", + "Deleting collection: 1d8431bb-ac44-4b95-82a6-876081179e18\n", + "Collection '1d8431bb-ac44-4b95-82a6-876081179e18' deleted successfully.\n", + "Deleting collection: 7aa14ede-a665-4e91-9869-c50a87f0b164\n", + "Collection '7aa14ede-a665-4e91-9869-c50a87f0b164' deleted successfully.\n", + "Deleting collection: 7739896d-a8c0-44fe-b84b-a46e8a53ed53\n", + "Collection '7739896d-a8c0-44fe-b84b-a46e8a53ed53' deleted successfully.\n", + "Deleting collection: 9f15948d-2df3-48b8-92ed-b94f91b1ddb7\n", + "Collection '9f15948d-2df3-48b8-92ed-b94f91b1ddb7' deleted successfully.\n", + "Deleting collection: ef851ab4-dbae-4c1f-97eb-81bf11c899af\n", + "Collection 'ef851ab4-dbae-4c1f-97eb-81bf11c899af' deleted successfully.\n", + "Deleting collection: f8879f0a-22f2-4525-be9c-ad1d2b5984a0\n", + "Collection 'f8879f0a-22f2-4525-be9c-ad1d2b5984a0' deleted successfully.\n", + "Deleting collection: 4113359d-c260-44a8-8623-a9519b1efe4b\n", + "Collection '4113359d-c260-44a8-8623-a9519b1efe4b' deleted successfully.\n", + "Deleting collection: 62340f0e-938d-4f31-a9c6-993ac7ba7bbf\n", + "Collection '62340f0e-938d-4f31-a9c6-993ac7ba7bbf' deleted successfully.\n", + "Deleting collection: ee5c02e2-ff64-4c5d-b6e9-fbc966128303\n", + "Collection 'ee5c02e2-ff64-4c5d-b6e9-fbc966128303' deleted successfully.\n", + "Deleting collection: 154694d5-be00-49f0-869b-e78e98ec6dc9\n", + "Collection '154694d5-be00-49f0-869b-e78e98ec6dc9' deleted successfully.\n", + "Deleting collection: 777e7e1a-47ae-41ad-9aef-6d9a5f788395\n", + "Collection '777e7e1a-47ae-41ad-9aef-6d9a5f788395' deleted successfully.\n", + "Deleting collection: 0e27f2f0-ec51-4ec6-a0b4-2440d0cd2423\n", + "Collection '0e27f2f0-ec51-4ec6-a0b4-2440d0cd2423' deleted successfully.\n", + "Deleting collection: 49c14e73-080a-4d6d-8906-1ac1c1a128dd\n", + "Collection '49c14e73-080a-4d6d-8906-1ac1c1a128dd' deleted successfully.\n", + "Deleting collection: 04b4b816-45c3-43c1-a88c-6bbded33cb0b\n", + "Collection '04b4b816-45c3-43c1-a88c-6bbded33cb0b' deleted successfully.\n", + "Deleting collection: 699a4650-244b-4ef8-93e8-985cb6e85ead\n", + "Collection '699a4650-244b-4ef8-93e8-985cb6e85ead' deleted successfully.\n", + "Deleting collection: aaf6aaf4-3ddf-4ad2-ae8a-f9fc0cb90bea\n", + "Collection 'aaf6aaf4-3ddf-4ad2-ae8a-f9fc0cb90bea' deleted successfully.\n", + "Deleting collection: 4066d2f3-c1d1-4854-a182-f5dc1965258c\n", + "Collection '4066d2f3-c1d1-4854-a182-f5dc1965258c' deleted successfully.\n", + "Deleting collection: 807f07e6-8fd5-4382-ad02-e6392404e737\n", + "Collection '807f07e6-8fd5-4382-ad02-e6392404e737' deleted successfully.\n", + "Deleting collection: 4515c126-0718-455d-88e8-4341545ada48\n", + "Collection '4515c126-0718-455d-88e8-4341545ada48' deleted successfully.\n", + "Deleting collection: blabla\n", + "Collection 'blabla' deleted successfully.\n", + "Deleting collection: 3949f338-757f-47de-92c5-ccaf61cea32f\n", + "Collection '3949f338-757f-47de-92c5-ccaf61cea32f' deleted successfully.\n", + "Deleting collection: test_memory_1__dlt_version\n", + "Collection 'test_memory_1__dlt_version' deleted successfully.\n", + "Deleting collection: 82661dd8-41ef-4b6c-a114-b5984fe0b31b\n", + "Collection '82661dd8-41ef-4b6c-a114-b5984fe0b31b' deleted successfully.\n", + "Deleting collection: 60733ef5-7a7a-4753-8263-59380dfe136b\n", + "Collection '60733ef5-7a7a-4753-8263-59380dfe136b' deleted successfully.\n", + "Deleting collection: af2d600f-dc8b-45af-ab2b-94f72a2738ed\n", + "Collection 'af2d600f-dc8b-45af-ab2b-94f72a2738ed' deleted successfully.\n", + "Deleting collection: fff33e71-09c4-4b28-a64c-5a74f6779c26\n", + "Collection 'fff33e71-09c4-4b28-a64c-5a74f6779c26' deleted successfully.\n", + "Deleting collection: 2e92ad70-56d7-492a-95a2-079f090cf67f\n", + "Collection '2e92ad70-56d7-492a-95a2-079f090cf67f' deleted successfully.\n", + "Deleting collection: 78afcf7c-ff61-4b0d-bafa-2eba80b3160c\n", + "Collection '78afcf7c-ff61-4b0d-bafa-2eba80b3160c' deleted successfully.\n", + "Deleting collection: b26a8485-a94f-4633-959d-38b906678871\n", + "Collection 'b26a8485-a94f-4633-959d-38b906678871' deleted successfully.\n", + "Deleting collection: 9d9467c0-3747-4aac-ae92-a39e2d55e92c\n", + "Collection '9d9467c0-3747-4aac-ae92-a39e2d55e92c' deleted successfully.\n", + "Deleting collection: 87bba54c-f9f9-4b04-8e66-e2fff176bfc0\n", + "Collection '87bba54c-f9f9-4b04-8e66-e2fff176bfc0' deleted successfully.\n", + "Deleting collection: 237d3dd8-b78c-495a-b53b-1b4e5fe50dfa\n", + "Collection '237d3dd8-b78c-495a-b53b-1b4e5fe50dfa' deleted successfully.\n", + "Deleting collection: 89ab525d-fcfe-4c0a-bcbe-1779d85d406d\n", + "Collection '89ab525d-fcfe-4c0a-bcbe-1779d85d406d' deleted successfully.\n", + "Deleting collection: f7f4121b-391b-46dd-97e7-568e9111c5fb\n", + "Collection 'f7f4121b-391b-46dd-97e7-568e9111c5fb' deleted successfully.\n", + "Deleting collection: 36ed86e1-2d1b-409c-9617-f2643fe5a75c\n", + "Collection '36ed86e1-2d1b-409c-9617-f2643fe5a75c' deleted successfully.\n", + "Deleting collection: ae71c599-b3ba-4819-8e5a-572cdf069a61\n", + "Collection 'ae71c599-b3ba-4819-8e5a-572cdf069a61' deleted successfully.\n", + "Deleting collection: b47fe3d3-c84e-4fdb-b90d-746c4d42dc04\n", + "Collection 'b47fe3d3-c84e-4fdb-b90d-746c4d42dc04' deleted successfully.\n", + "Deleting collection: e589b3d4-8750-44e2-8fab-1cc7e1b237de\n", + "Collection 'e589b3d4-8750-44e2-8fab-1cc7e1b237de' deleted successfully.\n", + "Deleting collection: 4cb184cd-924a-4315-81bd-74dda55b1ca4\n", + "Collection '4cb184cd-924a-4315-81bd-74dda55b1ca4' deleted successfully.\n", + "Deleting collection: f1fa0fcc-2271-4c51-81d8-4e4196f0d7d4\n", + "Collection 'f1fa0fcc-2271-4c51-81d8-4e4196f0d7d4' deleted successfully.\n", + "Deleting collection: 969f3423-2b21-4a6a-8adc-4c7b1c86d47a\n", + "Collection '969f3423-2b21-4a6a-8adc-4c7b1c86d47a' deleted successfully.\n", + "Deleting collection: db2d7dd0-6ecf-4c65-8cc4-4989c331bff6\n", + "Collection 'db2d7dd0-6ecf-4c65-8cc4-4989c331bff6' deleted successfully.\n", + "Deleting collection: 0b40a1cf-0aac-42ca-929e-f8f5e1de5e6f\n", + "Collection '0b40a1cf-0aac-42ca-929e-f8f5e1de5e6f' deleted successfully.\n", + "Deleting collection: dc155228-ab9d-47c3-97f4-d23ff285a9f6\n", + "Collection 'dc155228-ab9d-47c3-97f4-d23ff285a9f6' deleted successfully.\n", + "Deleting collection: f7ebec37-1ab9-4325-b4a2-c7423fff9fc7\n", + "Collection 'f7ebec37-1ab9-4325-b4a2-c7423fff9fc7' deleted successfully.\n", + "Deleting collection: afaa168f-3002-4128-a036-424f18f7afe3\n", + "Collection 'afaa168f-3002-4128-a036-424f18f7afe3' deleted successfully.\n", + "Deleting collection: 2fd8476d-d5f9-4d7c-8d9f-ae8b91b17e70\n", + "Collection '2fd8476d-d5f9-4d7c-8d9f-ae8b91b17e70' deleted successfully.\n", + "Deleting collection: test_memory_1\n", + "Collection 'test_memory_1' deleted successfully.\n", + "Deleting collection: 0a8be4df-38ac-46a8-b2ba-20a6189d795c\n", + "Collection '0a8be4df-38ac-46a8-b2ba-20a6189d795c' deleted successfully.\n", + "Deleting collection: b8150163-785b-49c0-a915-dcc2b22bbde0\n", + "Collection 'b8150163-785b-49c0-a915-dcc2b22bbde0' deleted successfully.\n", + "Deleting collection: 9ab34416-c1bf-4e85-aea9-cb20e77aa4eb\n", + "Collection '9ab34416-c1bf-4e85-aea9-cb20e77aa4eb' deleted successfully.\n", + "Deleting collection: 180b18bb-a2cc-4bf2-9fef-0ab976c03e08\n", + "Collection '180b18bb-a2cc-4bf2-9fef-0ab976c03e08' deleted successfully.\n", + "Deleting collection: e30981c4-a857-4694-be11-9f327d664120\n", + "Collection 'e30981c4-a857-4694-be11-9f327d664120' deleted successfully.\n", + "Deleting collection: f53f3f38-a3d3-4393-9554-45031541bba3\n", + "Collection 'f53f3f38-a3d3-4393-9554-45031541bba3' deleted successfully.\n", + "Deleting collection: 7fe97a64-b0a9-40f6-bc73-c41d79da9bd4\n", + "Collection '7fe97a64-b0a9-40f6-bc73-c41d79da9bd4' deleted successfully.\n", + "Deleting collection: 18dde4c4-8bad-4eff-9e67-155df9776746\n", + "Collection '18dde4c4-8bad-4eff-9e67-155df9776746' deleted successfully.\n", + "Deleting collection: 07117fe1-3816-43ad-a6bc-5b3f7d6abe4d\n", + "Collection '07117fe1-3816-43ad-a6bc-5b3f7d6abe4d' deleted successfully.\n", + "Deleting collection: d3ee4fdc-04df-451a-ac68-8b1b22a16429\n", + "Collection 'd3ee4fdc-04df-451a-ac68-8b1b22a16429' deleted successfully.\n", + "Deleting collection: e3702209-2c4b-47c2-834f-25b3f92c41a7\n", + "Collection 'e3702209-2c4b-47c2-834f-25b3f92c41a7' deleted successfully.\n", + "Deleting collection: 87ab3c73-de78-4779-b1be-c7c2cdfdf84c\n", + "Collection '87ab3c73-de78-4779-b1be-c7c2cdfdf84c' deleted successfully.\n", + "Deleting collection: 4183e91d-7534-446a-896b-9e2654e71eda\n", + "Collection '4183e91d-7534-446a-896b-9e2654e71eda' deleted successfully.\n", + "Deleting collection: 80d2b9f4-813a-43b4-94dc-174b4f2c3e22\n", + "Collection '80d2b9f4-813a-43b4-94dc-174b4f2c3e22' deleted successfully.\n", + "Deleting collection: ddeb724e-fef7-4048-9322-605a66fa2acf\n", + "Collection 'ddeb724e-fef7-4048-9322-605a66fa2acf' deleted successfully.\n", + "Deleting collection: d1c77a26-e35a-4a41-a221-7b655b6de0b5\n", + "Collection 'd1c77a26-e35a-4a41-a221-7b655b6de0b5' deleted successfully.\n", + "Deleting collection: 5a7d07d3-2588-4417-b3ac-f35c6502d85c\n", + "Collection '5a7d07d3-2588-4417-b3ac-f35c6502d85c' deleted successfully.\n", + "Deleting collection: f5a47430-a61c-42cd-b4a1-65ba75c04b01\n", + "Collection 'f5a47430-a61c-42cd-b4a1-65ba75c04b01' deleted successfully.\n", + "Deleting collection: 1faa1ddf-d27b-425d-a19f-8c7c64b3a6c3\n", + "Collection '1faa1ddf-d27b-425d-a19f-8c7c64b3a6c3' deleted successfully.\n", + "Deleting collection: 0991c482-08cd-4bec-b420-1ba8c78084fc\n", + "Collection '0991c482-08cd-4bec-b420-1ba8c78084fc' deleted successfully.\n", + "Deleting collection: 53f96999-912a-4873-bba9-aa545903f5fd\n", + "Collection '53f96999-912a-4873-bba9-aa545903f5fd' deleted successfully.\n", + "Deleting collection: 60106d01-36f8-45c1-bbf2-946ebba42bc5\n", + "Collection '60106d01-36f8-45c1-bbf2-946ebba42bc5' deleted successfully.\n", + "Deleting collection: 3adbb5a3-ee6d-4f66-8fe8-7d8deb923d6e\n", + "Collection '3adbb5a3-ee6d-4f66-8fe8-7d8deb923d6e' deleted successfully.\n", + "Deleting collection: 56568078-1fbc-415a-9f97-06c192717e77\n", + "Collection '56568078-1fbc-415a-9f97-06c192717e77' deleted successfully.\n", + "Deleting collection: 3a4b6713-b9bd-44f5-8017-49afc3aecf49\n", + "Collection '3a4b6713-b9bd-44f5-8017-49afc3aecf49' deleted successfully.\n", + "Deleting collection: e40ff65f-761f-42f9-9f25-cd11d69657fc\n", + "Collection 'e40ff65f-761f-42f9-9f25-cd11d69657fc' deleted successfully.\n", + "Deleting collection: 4ff664fc-87ad-47b6-9801-2b761c387e57\n", + "Collection '4ff664fc-87ad-47b6-9801-2b761c387e57' deleted successfully.\n", + "Deleting collection: ad7699c7-7b7c-4ba4-a75e-80e0d1e2fb77\n", + "Collection 'ad7699c7-7b7c-4ba4-a75e-80e0d1e2fb77' deleted successfully.\n", + "Deleting collection: d661c80f-e925-4d5b-bbd6-f6bc2a621454\n", + "Collection 'd661c80f-e925-4d5b-bbd6-f6bc2a621454' deleted successfully.\n", + "Deleting collection: 3e513b95-32f8-4538-a7ef-2e497891b0df\n", + "Collection '3e513b95-32f8-4538-a7ef-2e497891b0df' deleted successfully.\n", + "Deleting collection: 83e3fdca-d413-4764-b338-d8856e9c7f43\n", + "Collection '83e3fdca-d413-4764-b338-d8856e9c7f43' deleted successfully.\n", + "Deleting collection: 12199d7e-768e-47fb-b38d-0e96d88c4678\n", + "Collection '12199d7e-768e-47fb-b38d-0e96d88c4678' deleted successfully.\n", + "Deleting collection: 9a13b6b7-2e9b-420d-a610-28e30a045060\n", + "Collection '9a13b6b7-2e9b-420d-a610-28e30a045060' deleted successfully.\n", + "Deleting collection: 35d5b544-263f-4481-bd5f-63c194977bf7\n", + "Collection '35d5b544-263f-4481-bd5f-63c194977bf7' deleted successfully.\n", + "Deleting collection: 3da673d6-c098-4cdf-8486-c85cac4cc884\n", + "Collection '3da673d6-c098-4cdf-8486-c85cac4cc884' deleted successfully.\n", + "Deleting collection: baebf18a-a3a6-4378-beeb-72d7790d17d6\n", + "Collection 'baebf18a-a3a6-4378-beeb-72d7790d17d6' deleted successfully.\n", + "Deleting collection: 81103e61-a095-4171-8ee4-a49f21d98b37\n", + "Collection '81103e61-a095-4171-8ee4-a49f21d98b37' deleted successfully.\n", + "Deleting collection: ec947375-7086-416a-9884-dd7565b5f4de\n", + "Collection 'ec947375-7086-416a-9884-dd7565b5f4de' deleted successfully.\n", + "Deleting collection: 2d7e775b-183c-40ba-ae93-1cdf8b43c90e\n", + "Collection '2d7e775b-183c-40ba-ae93-1cdf8b43c90e' deleted successfully.\n", + "Deleting collection: e3b39f26-af24-4eb3-b363-812ea127284c\n", + "Collection 'e3b39f26-af24-4eb3-b363-812ea127284c' deleted successfully.\n", + "Deleting collection: 0c0996a6-eaf1-4453-b80e-b125eb0ecb2d\n", + "Collection '0c0996a6-eaf1-4453-b80e-b125eb0ecb2d' deleted successfully.\n", + "Deleting collection: b98c440a-9bcd-46e4-90a8-d48f028c0168\n", + "Collection 'b98c440a-9bcd-46e4-90a8-d48f028c0168' deleted successfully.\n", + "Deleting collection: 6a6d69d6-16b3-4c1a-935b-739d51051b5a\n", + "Collection '6a6d69d6-16b3-4c1a-935b-739d51051b5a' deleted successfully.\n", + "Deleting collection: 6656889f-73c4-4b61-9fe2-11939dcde325\n", + "Collection '6656889f-73c4-4b61-9fe2-11939dcde325' deleted successfully.\n", + "Deleting collection: a2ec45e4-4cb4-48c3-a586-caa380e7ecfa\n", + "Collection 'a2ec45e4-4cb4-48c3-a586-caa380e7ecfa' deleted successfully.\n", + "Deleting collection: bdb5caae-ffc0-48ae-9941-6441598ec091\n", + "Collection 'bdb5caae-ffc0-48ae-9941-6441598ec091' deleted successfully.\n", + "Deleting collection: d514037b-02b9-42a6-8afc-dd475c682e7b\n", + "Collection 'd514037b-02b9-42a6-8afc-dd475c682e7b' deleted successfully.\n", + "Deleting collection: test_memory_1__dlt_loads\n", + "Collection 'test_memory_1__dlt_loads' deleted successfully.\n", + "Deleting collection: f78c2fbd-a6bc-40d1-a771-2e17a47c6ce3\n", + "Collection 'f78c2fbd-a6bc-40d1-a771-2e17a47c6ce3' deleted successfully.\n", + "Deleting collection: 34290f17-0d86-4a3a-965b-d68bda14bbe8\n", + "Collection '34290f17-0d86-4a3a-965b-d68bda14bbe8' deleted successfully.\n", + "Deleting collection: fe887a77-0c35-44c0-9684-6acc0a7780bb\n", + "Collection 'fe887a77-0c35-44c0-9684-6acc0a7780bb' deleted successfully.\n", + "Deleting collection: 52cd77ea-7c10-41cc-aed8-a18ba166faf7\n", + "Collection '52cd77ea-7c10-41cc-aed8-a18ba166faf7' deleted successfully.\n", + "Deleting collection: f0e301a6-7e87-4d7b-8bbd-b684e67049f1\n", + "Collection 'f0e301a6-7e87-4d7b-8bbd-b684e67049f1' deleted successfully.\n", + "Deleting collection: f3eaa77c-d3f8-4e1d-8621-a07ce48b4949\n", + "Collection 'f3eaa77c-d3f8-4e1d-8621-a07ce48b4949' deleted successfully.\n", + "Deleting collection: bc4224dc-935e-474f-8f28-62cdc45693f7\n", + "Collection 'bc4224dc-935e-474f-8f28-62cdc45693f7' deleted successfully.\n", + "Deleting collection: 573e39d7-18d5-4a43-88c0-5122e2bff778\n", + "Collection '573e39d7-18d5-4a43-88c0-5122e2bff778' deleted successfully.\n", + "Deleting collection: 665a98f3-b28a-44a7-a28f-6d1416e99a84\n", + "Collection '665a98f3-b28a-44a7-a28f-6d1416e99a84' deleted successfully.\n", + "Deleting collection: cac55ec8-d110-4405-8add-4d29be627951\n", + "Collection 'cac55ec8-d110-4405-8add-4d29be627951' deleted successfully.\n", + "Deleting collection: cfacc589-f86a-4664-beb2-d5436e29ef94\n", + "Collection 'cfacc589-f86a-4664-beb2-d5436e29ef94' deleted successfully.\n", + "Deleting collection: 82da1cbb-e200-4516-868e-5d29b9dcf307\n", + "Collection '82da1cbb-e200-4516-868e-5d29b9dcf307' deleted successfully.\n", + "Deleting collection: b2864798-403d-46ae-82fd-db0ca5a44f04\n", + "Collection 'b2864798-403d-46ae-82fd-db0ca5a44f04' deleted successfully.\n", + "Deleting collection: 3528bf94-2474-45d1-a309-155fd429b3de\n", + "Collection '3528bf94-2474-45d1-a309-155fd429b3de' deleted successfully.\n", + "Deleting collection: 0a78eb20-14e7-4431-a8e9-35188b84bd28\n", + "Collection '0a78eb20-14e7-4431-a8e9-35188b84bd28' deleted successfully.\n", + "Deleting collection: e6adbb7d-cb23-40a2-b2dd-4383e2ea1ad1\n", + "Collection 'e6adbb7d-cb23-40a2-b2dd-4383e2ea1ad1' deleted successfully.\n", + "Deleting collection: d196a1ef-8da2-4e24-bae4-eb5fe0a06b1a\n", + "Collection 'd196a1ef-8da2-4e24-bae4-eb5fe0a06b1a' deleted successfully.\n", + "Deleting collection: 19491281-44ab-494d-ad6d-6ac1864312a6\n", + "Collection '19491281-44ab-494d-ad6d-6ac1864312a6' deleted successfully.\n", + "Deleting collection: test_memory_1__dlt_pipeline_state\n", + "Collection 'test_memory_1__dlt_pipeline_state' deleted successfully.\n", + "Deleting collection: cd154db9-1a63-4c75-a632-fee11b0cbab2\n", + "Collection 'cd154db9-1a63-4c75-a632-fee11b0cbab2' deleted successfully.\n", + "Deleting collection: 8e57e5d0-3451-4ec6-b970-df1259f56880\n", + "Collection '8e57e5d0-3451-4ec6-b970-df1259f56880' deleted successfully.\n", + "Deleting collection: 8c02bd4b-85f5-45f8-9257-27ab5d586e37\n", + "Collection '8c02bd4b-85f5-45f8-9257-27ab5d586e37' deleted successfully.\n", + "Deleting collection: 9d59c164-5674-41be-bb6b-420a5303cf3c\n", + "Collection '9d59c164-5674-41be-bb6b-420a5303cf3c' deleted successfully.\n", + "Deleting collection: 6d543624-7c07-4eec-9782-d32c0f582661\n", + "Collection '6d543624-7c07-4eec-9782-d32c0f582661' deleted successfully.\n", + "Deleting collection: ab76d07e-257b-451e-8883-35ec919614d5\n", + "Collection 'ab76d07e-257b-451e-8883-35ec919614d5' deleted successfully.\n", + "Deleting collection: ee770796-286a-469d-96b2-d095bc9ecf54\n", + "Collection 'ee770796-286a-469d-96b2-d095bc9ecf54' deleted successfully.\n", + "Deleting collection: 35f86dc5-5394-40b6-a04c-379772ed936c\n", + "Collection '35f86dc5-5394-40b6-a04c-379772ed936c' deleted successfully.\n", + "Deleting collection: 8c0a0959-e093-40da-9224-67fb8c079534\n", + "Collection '8c0a0959-e093-40da-9224-67fb8c079534' deleted successfully.\n", + "Deleting collection: 2e58f6e5-6fd0-42d2-a5b9-2167521df2c2\n", + "Collection '2e58f6e5-6fd0-42d2-a5b9-2167521df2c2' deleted successfully.\n", + "Deleting collection: f84ee8f5-a419-4ac1-834a-e4742fcf0e05\n", + "Collection 'f84ee8f5-a419-4ac1-834a-e4742fcf0e05' deleted successfully.\n", + "Deleting collection: 61ffd6af-b2c6-4d36-8dd2-0835e3107f72\n", + "Collection '61ffd6af-b2c6-4d36-8dd2-0835e3107f72' deleted successfully.\n", + "Deleting collection: beb07e3e-6c86-4b09-9a7a-3857708ca0a8\n", + "Collection 'beb07e3e-6c86-4b09-9a7a-3857708ca0a8' deleted successfully.\n", + "Deleting collection: f119c22d-95f6-47bb-88c6-2b33b74937d5\n", + "Collection 'f119c22d-95f6-47bb-88c6-2b33b74937d5' deleted successfully.\n", + "Deleting collection: ee5effad-a527-4fd0-85e3-3928209d18cd\n", + "Collection 'ee5effad-a527-4fd0-85e3-3928209d18cd' deleted successfully.\n", + "Deleting collection: acb151dd-4dd1-47ad-91d3-cc99b822c8c3\n", + "Collection 'acb151dd-4dd1-47ad-91d3-cc99b822c8c3' deleted successfully.\n", + "Deleting collection: 2649b0a7-23b9-4e7d-9a96-f3b4eee3d3fe\n", + "Collection '2649b0a7-23b9-4e7d-9a96-f3b4eee3d3fe' deleted successfully.\n", + "Deleting collection: 61850606-f718-4076-a0c4-eb14feaa7ab4\n", + "Collection '61850606-f718-4076-a0c4-eb14feaa7ab4' deleted successfully.\n", + "Deleting collection: 782a7518-91f3-473d-9c83-acd626f5a505\n", + "Collection '782a7518-91f3-473d-9c83-acd626f5a505' deleted successfully.\n", + "Deleting collection: abbb9e6a-c00e-4653-80a3-f6a2ceafe25c\n", + "Collection 'abbb9e6a-c00e-4653-80a3-f6a2ceafe25c' deleted successfully.\n", + "Deleting collection: d63ecd4d-4822-4419-a0f7-b8e5abe2ca00\n", + "Collection 'd63ecd4d-4822-4419-a0f7-b8e5abe2ca00' deleted successfully.\n", + "Deleting collection: e800462b-fbe4-4ea9-a71b-fc8eda28728f\n", + "Collection 'e800462b-fbe4-4ea9-a71b-fc8eda28728f' deleted successfully.\n", + "Deleting collection: 94a73201-001a-4296-ab4f-cd4c4d98c44a\n", + "Collection '94a73201-001a-4296-ab4f-cd4c4d98c44a' deleted successfully.\n", + "Deleting collection: 58428a33-0262-47f7-9602-ba9e31818269\n", + "Collection '58428a33-0262-47f7-9602-ba9e31818269' deleted successfully.\n", + "Deleting collection: 3ec201a4-6d31-4f60-86ba-c0b14828a86a\n", + "Collection '3ec201a4-6d31-4f60-86ba-c0b14828a86a' deleted successfully.\n", + "Deleting collection: 895bc85c-633a-4f64-9b5e-0ceb2ed7e89b\n", + "Collection '895bc85c-633a-4f64-9b5e-0ceb2ed7e89b' deleted successfully.\n", + "Deleting collection: ce1f95d0-21d3-4fe2-b938-eb917eceb96b\n", + "Collection 'ce1f95d0-21d3-4fe2-b938-eb917eceb96b' deleted successfully.\n", + "Deleting collection: f8126dcf-1202-4668-ab2b-8d714578961a\n", + "Collection 'f8126dcf-1202-4668-ab2b-8d714578961a' deleted successfully.\n", + "Deleting collection: aedcd320-3855-4912-b5cd-481679fdf3f7\n", + "Collection 'aedcd320-3855-4912-b5cd-481679fdf3f7' deleted successfully.\n", + "Deleting collection: 8c8fb9ca-fada-4a42-aada-34826ea3ab88\n", + "Collection '8c8fb9ca-fada-4a42-aada-34826ea3ab88' deleted successfully.\n", + "Deleting collection: 14a5840d-39c7-4399-ba7a-34792b04a950\n", + "Collection '14a5840d-39c7-4399-ba7a-34792b04a950' deleted successfully.\n", + "Deleting collection: ab113151-10d7-4ae7-aed8-fbd03109f6e0\n", + "Collection 'ab113151-10d7-4ae7-aed8-fbd03109f6e0' deleted successfully.\n", + "Deleting collection: 03c1865f-86ef-40e8-9dfd-809aec4f247a\n", + "Collection '03c1865f-86ef-40e8-9dfd-809aec4f247a' deleted successfully.\n", + "Deleting collection: 5217cb0e-cc79-4b65-a702-b841f33cb0df\n", + "Collection '5217cb0e-cc79-4b65-a702-b841f33cb0df' deleted successfully.\n", + "Deleting collection: 9d930558-996b-4aae-9be1-ea326ecdd178\n", + "Collection '9d930558-996b-4aae-9be1-ea326ecdd178' deleted successfully.\n", + "Deleting collection: 370492a9-f9ec-4a43-add8-2c9a00122bc7\n", + "Collection '370492a9-f9ec-4a43-add8-2c9a00122bc7' deleted successfully.\n", + "Deleting collection: 4e2a2902-3154-443f-bb70-bd3e04602171\n", + "Collection '4e2a2902-3154-443f-bb70-bd3e04602171' deleted successfully.\n", + "Deleting collection: 00cb0190-8a9d-4904-8478-1965585d7511\n", + "Collection '00cb0190-8a9d-4904-8478-1965585d7511' deleted successfully.\n", + "Deleting collection: 3fc2a9a3-a810-443d-a463-fe24c74a3116\n", + "Collection '3fc2a9a3-a810-443d-a463-fe24c74a3116' deleted successfully.\n", + "Deleting collection: 0af274b4-96e2-44cf-876a-c02db53299ab\n", + "Collection '0af274b4-96e2-44cf-876a-c02db53299ab' deleted successfully.\n", + "Deleting collection: a7172c77-a13e-485b-abf8-3eb091b12459\n", + "Collection 'a7172c77-a13e-485b-abf8-3eb091b12459' deleted successfully.\n", + "Deleting collection: cad2276f-bdc2-497a-a75f-067a162f2bab\n", + "Collection 'cad2276f-bdc2-497a-a75f-067a162f2bab' deleted successfully.\n", + "Deleting collection: c9d3622c-b34b-4956-910b-7d381864b4e3\n", + "Collection 'c9d3622c-b34b-4956-910b-7d381864b4e3' deleted successfully.\n", + "Deleting collection: 08ca73d3-17d6-4488-96cd-60fef616e54a\n", + "Collection '08ca73d3-17d6-4488-96cd-60fef616e54a' deleted successfully.\n", + "Deleting collection: 51189101-a16f-440f-adeb-1e34df5d7659\n", + "Collection '51189101-a16f-440f-adeb-1e34df5d7659' deleted successfully.\n", + "Deleting collection: 58e8480e-8d61-41d1-9744-f5cbf264da67\n", + "Collection '58e8480e-8d61-41d1-9744-f5cbf264da67' deleted successfully.\n", + "Deleting collection: ed9a23d8-8a82-4761-b173-3b38a7b5ad96\n", + "Collection 'ed9a23d8-8a82-4761-b173-3b38a7b5ad96' deleted successfully.\n", + "Deleting collection: test_memory_1_content\n", + "Collection 'test_memory_1_content' deleted successfully.\n", + "All collections have been deleted.\n" + ] + } + ], + "source": [ + "collections_response = qdrant.http.collections_api.get_collections()\n", + "collections = collections_response.result.collections\n", + "\n", + "# # Delete each collection\n", + "# for collection in collections:\n", + "# collection_name = collection.name\n", + "# print(f\"Deleting collection: {collection_name}\")\n", + "# delete_response = qdrant.http.collections_api.delete_collection(collection_name=collection_name)\n", + "# if delete_response.status == \"ok\":\n", + "# print(f\"Collection '{collection_name}' deleted successfully.\")\n", + "# else:\n", + "# print(f\"Failed to delete collection '{collection_name}'. Response: {delete_response}\")\n", + "\n", + "# print(\"All collections have been deleted.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 136, + "id": "f0b5f7fd-8cc2-48c5-9a8a-04410e835ea2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error during batch search for ID 6: 1 validation error for NamedVector\n", + "vector\n", + " Input should be a valid list [type=list_type, input_value='a', input_type=str]\n", + " For further information visit https://errors.pydantic.dev/2.6/v/list_type\n" + ] + }, + { + "ename": "TypeError", + "evalue": "object of type 'float' has no len()", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[136], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m resultso \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m adapted_qdrant_batch_search(rr,db)\n", + "Cell \u001b[0;32mIn[110], line 12\u001b[0m, in \u001b[0;36madapted_qdrant_batch_search\u001b[0;34m(results_to_check, vector_client)\u001b[0m\n\u001b[1;32m 9\u001b[0m b\u001b[38;5;241m=\u001b[39m result[\u001b[38;5;241m4\u001b[39m]\n\u001b[1;32m 11\u001b[0m \u001b[38;5;66;03m# Assuming each result in results_to_check contains a single embedding\u001b[39;00m\n\u001b[0;32m---> 12\u001b[0m limits \u001b[38;5;241m=\u001b[39m [\u001b[38;5;241m3\u001b[39m] \u001b[38;5;241m*\u001b[39m \u001b[38;5;28mlen\u001b[39m(embedding) \u001b[38;5;66;03m# Set a limit of 3 results for this embedding\u001b[39;00m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 15\u001b[0m \u001b[38;5;66;03m#Perform the batch search for this id with its embedding\u001b[39;00m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;66;03m#Assuming qdrant_batch_search function accepts a single embedding and a list of limits\u001b[39;00m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;66;03m#qdrant_batch_search\u001b[39;00m\n\u001b[1;32m 18\u001b[0m id_search_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m qdrant_batch_search(collection_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mid\u001b[39m, embeddings\u001b[38;5;241m=\u001b[39m embedding, with_vectors\u001b[38;5;241m=\u001b[39mlimits)\n", + "\u001b[0;31mTypeError\u001b[0m: object of type 'float' has no len()" + ] + } + ], + "source": [ + "resultso = await adapted_qdrant_batch_search(rr,db)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -3850,500 +4357,15 @@ "id": "217fcdd1-e1f7-48f3-a835-cfd003bd6da9", "metadata": {}, "outputs": [], - "source": [ - "# import networkx as nx\n", - "# import uuid\n", - "# from datetime import datetime\n", - "\n", - "# def create_user_content_graph(user_id, custom_user_properties=None, required_layers=None, default_fields=None, existing_graph=None):\n", - "\n", - "# category_name = required_layers['context_name']\n", - "# subgroup_names = [required_layers['layer_name']]\n", - "\n", - " \n", - "# # Construct the additional_categories structure\n", - "# additional_categories = {\n", - "# category_name: subgroup_names\n", - "# }\n", - "\n", - "# # Define default fields for all nodes if not provided\n", - "# if default_fields is None:\n", - "# default_fields = {\n", - "# 'created_at': datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"),\n", - "# 'updated_at': datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n", - "# }\n", - "\n", - "# # Merge custom user properties with default properties; custom properties take precedence\n", - "# user_properties = {**default_fields, **(custom_user_properties or {})}\n", - "\n", - "# # Default content categories\n", - "# content_categories = {\n", - "# \"Temporal\": [\"Historical events\", \"Schedules and timelines\"],\n", - "# \"Positional\": [\"Geographical locations\", \"Spatial data\"],\n", - "# \"Propositions\": [\"Hypotheses and theories\", \"Claims and arguments\"],\n", - "# \"Personalization\": [\"User preferences\", \"User information\"]\n", - "# }\n", - "\n", - "# # Update content categories with any additional categories provided\n", - "# if additional_categories:\n", - "# content_categories.update(additional_categories)\n", - "\n", - "# G = existing_graph if existing_graph else nx.MultiDiGraph()\n", - "\n", - "# # Check if the user node already exists, if not, add the user node with properties\n", - "# if not G.has_node(user_id):\n", - "# G.add_node(user_id, **user_properties)\n", - "\n", - "# # Add or update content category nodes and their edges\n", - "# for category, subclasses in content_categories.items():\n", - "# category_properties = {**default_fields, 'type': 'category'}\n", - "\n", - "# # Add or update the category node\n", - "# if not G.has_node(category):\n", - "# G.add_node(category, **category_properties)\n", - "# G.add_edge(user_id, category, relationship='created')\n", - "\n", - "# # Add or update subclass nodes and their edges\n", - "# for subclass in subclasses:\n", - "# # Using both category and subclass names to ensure uniqueness within categories\n", - "# subclass_node_id = f\"{category}:{subclass}\"\n", - "\n", - "# # Check if subclass node exists before adding, based on node content\n", - "# if not any(subclass == data.get('content') for _, data in G.nodes(data=True)):\n", - "# subclass_properties = {**default_fields, 'type': 'subclass', 'content': subclass}\n", - "# G.add_node(subclass_node_id, **subclass_properties)\n", - "# G.add_edge(category, subclass_node_id, relationship='includes')\n", - "\n", - "# return G\n", - "\n", - "# # # # Add content category nodes and their edges\n", - "# # # for category, subclasses in content_categories.items():\n", - "# # # category_properties = {**default_fields, 'type': 'category'}\n", - "# # # G.add_node(category, **category_properties)\n", - "# # # G.add_edge(user_id, category, relationship='created')\n", - "\n", - "# # # # Add subclass nodes and their edges\n", - "# # # for subclass in subclasses:\n", - "# # # unique_id = str(uuid.uuid4())\n", - "# # # subclass_node_id = f\"{subclass} - {unique_id}\"\n", - "# # # subclass_properties = {**default_fields, 'type': 'subclass', 'content': subclass}\n", - "# # # G.add_node(subclass_node_id, **subclass_properties)\n", - "# # # G.add_edge(category, subclass_node_id, relationship='includes')\n", - "\n", - "# # # return G\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f974c87e-4117-4f93-a96c-e4fcd741aed9", - "metadata": {}, - "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "8c9b14de-85f1-4aec-ab80-54e4b2a8f317", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eebe4f53-c6ba-466a-8ba2-adee16fb6e21", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 400, - "id": "cca2637a-eace-4763-ada4-0ce925afd7ce", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e67c6679-f339-4ad1-b1bc-0c896a973abe", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 214, - "id": "418ef758-64e1-4c44-a844-9e3960d9db50", - "metadata": {}, - "outputs": [], - "source": [ - "from pydantic import BaseModel, Field\n", - "from typing import List, Optional, Dict\n", - "from datetime import datetime\n", - "\n", - "class Relationship(BaseModel):\n", - " type: str\n", - " properties: Optional[Dict[str, str]] = None\n", - "\n", - "class DocumentType(BaseModel):\n", - " type_id: str\n", - " description: str\n", - " default_relationship: Relationship = Relationship(type='is_type')\n", - "\n", - "class Category(BaseModel):\n", - " category_id: str\n", - " name: str\n", - " default_relationship: Relationship = Relationship(type='categorized_as')\n", - "\n", - "class Document(BaseModel):\n", - " doc_id: str\n", - " title: str\n", - " summary: Optional[str] = None\n", - " content_id: Optional[str] = None # Reference to external content storage\n", - " doc_type: Optional[DocumentType] = None\n", - " categories: List[Category] = []\n", - " default_relationship: Relationship = Relationship(type='has_document')\n", - "\n", - "class UserLocation(BaseModel):\n", - " location_id: str\n", - " description: str\n", - " default_relationship: Relationship = Relationship(type='located_in')\n", - "\n", - "class UserProperties(BaseModel):\n", - " custom_properties: Optional[Dict[str, str]] = None\n", - " location: Optional[UserLocation] = None\n", - "\n", - "class GraphModel(BaseModel):\n", - " id: str = Field(..., alias='user_id')\n", - " user_properties: UserProperties = UserProperties()\n", - " documents: List[Document] = []\n", - " default_fields: Optional[Dict[str, str]] = Field(default_factory=lambda: {\n", - " 'created_at': datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"),\n", - " 'updated_at': datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n", - " })\n" - ] - }, - { - "cell_type": "code", - "execution_count": 137, - "id": "1061c73b-271d-4044-8291-340a69b7f2cb", - "metadata": {}, - "outputs": [], - "source": [ - "# import networkx as nx\n", - "# from pydantic import BaseModel\n", - "# from typing import Optional, Any, Dict, Union\n", - "\n", - "# def add_node_and_edge(G, parent_id: Optional[str], node_id: str, node_data: Dict[str, Any], relationship_data: Dict[str, Any]):\n", - "# G.add_node(node_id, **node_data)\n", - "# if parent_id is not None:\n", - "# G.add_edge(parent_id, node_id, **relationship_data)\n", - "\n", - "# def process_attribute(G, parent_id: Optional[str], attribute: str, value: Union[BaseModel, List[BaseModel], Any], model_instance: Optional[BaseModel] = None):\n", - "# if isinstance(value, BaseModel): # Single Pydantic model instance\n", - "# # Generate a unique node identifier based on the model instance type and a relevant unique field\n", - "# instance_id = f\"{value.__class__.__name__}:{getattr(value, 'id', getattr(value, 'doc_id', value.__class__.__name__))}\"\n", - "# node_data = value.dict(exclude={'default_relationship', 'categories', 'doc_type'})\n", - "# relationship_data = value.default_relationship.dict() if hasattr(value, 'default_relationship') else {}\n", - "\n", - "# # Special handling for UserProperties to flatten its structure and incorporate it into the user node\n", - "# if isinstance(value, UserProperties):\n", - "# node_data = {**node_data.get('custom_properties', {}), **node_data.get('location', {})}\n", - "# instance_id = parent_id # Use the user node ID for UserProperties\n", - "\n", - "# add_node_and_edge(G, parent_id, instance_id, node_data, relationship_data)\n", - "\n", - "# # Recursively process nested attributes\n", - "# for sub_attr, sub_val in value.dict(exclude={'id', 'doc_id'}).items():\n", - "# process_attribute(G, instance_id, sub_attr, sub_val, model_instance=value)\n", - "\n", - "# elif isinstance(value, list) and all(isinstance(item, BaseModel) for item in value):\n", - "# for item in value:\n", - "# process_attribute(G, parent_id, attribute, item, model_instance)\n", - "\n", - "# # Handle the creation of category nodes from the Document model instance\n", - "# elif attribute == 'categories' and model_instance and isinstance(model_instance, Document):\n", - "# for category in value:\n", - "# cat_node_id = f\"Category:{category['category_id']}\"\n", - "# G.add_node(cat_node_id, **category)\n", - "# G.add_edge(parent_id, cat_node_id, **category['default_relationship'].dict())\n", - "\n", - "# def create_dynamic(graph_model: BaseModel, existing_graph: Optional[nx.Graph] = None) -> nx.Graph:\n", - "# G = existing_graph if existing_graph else nx.Graph()\n", - "\n", - "# # The root node is the user node\n", - "# root_id = getattr(graph_model, 'id', None)\n", - "# if root_id is None:\n", - "# raise ValueError(\"GraphModel instance must have an 'id' field.\")\n", - "\n", - "# # User node creation with properties from user_properties\n", - "# user_properties = graph_model.user_properties.custom_properties if graph_model.user_properties else {}\n", - "# user_node_data = {**graph_model.default_fields, **user_properties}\n", - "# add_node_and_edge(G, None, root_id, user_node_data, {})\n", - "\n", - "# # Process top-level attributes of the graph model, excluding handled fields\n", - "# for attribute_name, attribute_value in graph_model.dict(exclude={'id', 'user_properties', 'default_fields'}).items():\n", - "# process_attribute(G, root_id, attribute_name, attribute_value)\n", - "\n", - "# return G\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 215, - "id": "31994fee-6ab1-4132-8ade-b29fd12f940b", - "metadata": {}, - "outputs": [], - "source": [ - "# import networkx as nx\n", - "# from pydantic import BaseModel\n", - "# from typing import Optional, Any, Union, List\n", - "\n", - "# def add_node_and_edge(G, parent_id: Optional[str], node_id: str, node_data: dict, relationship_data: dict):\n", - "# G.add_node(node_id, **node_data)\n", - "# if parent_id is not None:\n", - "# G.add_edge(parent_id, node_id, **relationship_data)\n", - "\n", - "# def generate_node_id(instance: BaseModel) -> str:\n", - "# \"\"\"Generates a unique node ID based on the Pydantic model instance.\"\"\"\n", - "# for field in ['id', 'doc_id', 'location_id', 'type_id']:\n", - "# if hasattr(instance, field):\n", - "# return f\"{instance.__class__.__name__}:{getattr(instance, field)}\"\n", - "# return f\"{instance.__class__.__name__}:default\"\n", - "\n", - "\n", - "\n", - "\n", - "# def process_attribute(G, parent_id: Optional[str], attribute: str, value: Any):\n", - "# if isinstance(value, BaseModel): # Single Pydantic model instance\n", - "# node_id = generate_node_id(value)\n", - "# node_data = value.dict(exclude={'default_relationship'})\n", - "# relationship_data = value.default_relationship.dict() if hasattr(value, 'default_relationship') else {}\n", - "# add_node_and_edge(G, parent_id, node_id, node_data, relationship_data)\n", - "\n", - "# # Recursively process nested attributes\n", - "# for sub_attr, sub_val in value:\n", - "# process_attribute(G, node_id, sub_attr, sub_val)\n", - "\n", - "# elif isinstance(value, list) and all(isinstance(item, BaseModel) for item in value):\n", - "# for item in value:\n", - "# process_attribute(G, parent_id, attribute, item)\n", - "\n", - "# def create_dynamic(graph_model: BaseModel, existing_graph: Optional[nx.Graph] = None) -> nx.Graph:\n", - "# G = existing_graph if existing_graph else nx.Graph()\n", - "\n", - "# # Dynamically get the root node ID based on the presence of a unique identifier\n", - "# root_id = generate_node_id(graph_model)\n", - "# G.add_node(root_id, **graph_model.dict(exclude={'default_relationship'}))\n", - "\n", - "# # Dynamically process each attribute of the root Pydantic model\n", - "# for attribute_name, attribute_value in graph_model:\n", - "# process_attribute(G, root_id, attribute_name, attribute_value)\n", - "\n", - "# return G\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "id": "e6bb6a71-1e8d-4695-a3d0-0d25b1e44fa0", - "metadata": {}, - "outputs": [], - "source": [ - "import networkx as nx\n", - "from pydantic import BaseModel\n", - "from typing import Optional, Any, List, Dict\n", - "from datetime import datetime\n", - "\n", - "# Models for representing different entities\n", - "class Relationship(BaseModel):\n", - " type: str\n", - " properties: Optional[Dict[str, Any]] = None\n", - "\n", - "class DocumentType(BaseModel):\n", - " type_id: str\n", - " description: str\n", - " default_relationship: Relationship = Relationship(type='is_type')\n", - "\n", - "class Category(BaseModel):\n", - " category_id: str\n", - " name: str\n", - " default_relationship: Relationship = Relationship(type='categorized_as')\n", - "\n", - "class Document(BaseModel):\n", - " doc_id: str\n", - " title: str\n", - " summary: Optional[str] = None\n", - " content_id: Optional[str] = None\n", - " doc_type: Optional[DocumentType] = None\n", - " categories: List[Category] = []\n", - " default_relationship: Relationship = Relationship(type='has_document')\n", - "\n", - "class UserLocation(BaseModel):\n", - " location_id: str\n", - " description: str\n", - " default_relationship: Relationship = Relationship(type='located_in')\n", - "\n", - "class UserProperties(BaseModel):\n", - " custom_properties: Optional[Dict[str, Any]] = None\n", - " location: Optional[UserLocation] = None\n", - "\n", - "class GraphModel(BaseModel):\n", - " id: str\n", - " user_properties: UserProperties = UserProperties()\n", - " documents: List[Document] = []\n", - " default_fields: Optional[Dict[str, Any]] = {}\n", - "\n", - "def generate_node_id(instance: BaseModel) -> str:\n", - " for field in ['id', 'doc_id', 'location_id', 'type_id']:\n", - " if hasattr(instance, field):\n", - " return f\"{instance.__class__.__name__}:{getattr(instance, field)}\"\n", - " return f\"{instance.__class__.__name__}:default\"\n", - "\n", - "def add_node_and_edge(G, parent_id: Optional[str], node_id: str, node_data: dict, relationship_data: dict):\n", - " G.add_node(node_id, **node_data) # Add the current node with its data\n", - " if parent_id:\n", - " # Add an edge between the parent node and the current node with the correct relationship data\n", - " G.add_edge(parent_id, node_id, **relationship_data)\n", - "\n", - "def process_attribute(G, parent_id: Optional[str], attribute: str, value: Any):\n", - " if isinstance(value, BaseModel):\n", - " node_id = generate_node_id(value)\n", - " node_data = value.dict(exclude={'default_relationship'})\n", - " # Use the specified default relationship for the edge between the parent node and the current node\n", - " relationship_data = value.default_relationship.dict() if hasattr(value, 'default_relationship') else {}\n", - " add_node_and_edge(G, parent_id, node_id, node_data, relationship_data)\n", - "\n", - " # Recursively process nested attributes to ensure all nodes and relationships are added to the graph\n", - " for sub_attr, sub_val in value.__dict__.items(): # Access attributes and their values directly\n", - " process_attribute(G, node_id, sub_attr, sub_val)\n", - "\n", - " elif isinstance(value, list) and all(isinstance(item, BaseModel) for item in value):\n", - " # For lists of BaseModel instances, process each item in the list\n", - " for item in value:\n", - " process_attribute(G, parent_id, attribute, item)\n", - "\n", - "def create_dynamic(graph_model: BaseModel, existing_graph: Optional[nx.Graph] = None) -> nx.Graph:\n", - " G = existing_graph or nx.Graph()\n", - " root_id = generate_node_id(graph_model)\n", - " print(root_id)\n", - " G.add_node(root_id, **graph_model.dict(exclude={'default_relationship'}))\n", - "\n", - " for attribute_name, attribute_value in graph_model:\n", - " process_attribute(G, root_id, attribute_name, attribute_value)\n", - "\n", - " return G\n", - "\n", - "# Example usage with GraphModel instance\n" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "id": "82ed3638-6252-4629-b411-b7e9029dd282", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'user123',\n", - " 'user_properties': {'custom_properties': {'age': '30'},\n", - " 'location': {'location_id': 'ny',\n", - " 'description': 'New York',\n", - " 'default_relationship': {'type': 'located_in', 'properties': None}}},\n", - " 'documents': [{'doc_id': 'doc1',\n", - " 'title': 'Document 1',\n", - " 'summary': 'Summary of Document 1',\n", - " 'content_id': 'content_id_for_doc1',\n", - " 'doc_type': {'type_id': 'PDF',\n", - " 'description': 'Portable Document Format',\n", - " 'default_relationship': {'type': 'is_type', 'properties': None}},\n", - " 'categories': [{'category_id': 'finance',\n", - " 'name': 'Finance',\n", - " 'default_relationship': {'type': 'belongs_to', 'properties': None}},\n", - " {'category_id': 'tech',\n", - " 'name': 'Technology',\n", - " 'default_relationship': {'type': 'belongs_to', 'properties': None}}],\n", - " 'default_relationship': {'type': 'has_document', 'properties': None}},\n", - " {'doc_id': 'doc2',\n", - " 'title': 'Document 2',\n", - " 'summary': 'Summary of Document 2',\n", - " 'content_id': 'content_id_for_doc2',\n", - " 'doc_type': {'type_id': 'TXT',\n", - " 'description': 'Text File',\n", - " 'default_relationship': {'type': 'is_type', 'properties': None}},\n", - " 'categories': [{'category_id': 'health',\n", - " 'name': 'Health',\n", - " 'default_relationship': {'type': 'belongs_to', 'properties': None}},\n", - " {'category_id': 'wellness',\n", - " 'name': 'Wellness',\n", - " 'default_relationship': {'type': 'belongs_to', 'properties': None}}],\n", - " 'default_relationship': {'type': 'has_document', 'properties': None}}],\n", - " 'default_fields': {'created_at': '2024-03-10 22:09:31',\n", - " 'updated_at': '2024-03-10 22:09:31'}}" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "graph_model_instance.dict(exclude={'default_relationship'})" - ] - }, - { - "cell_type": "code", - "execution_count": 67, "id": "b32c4472-fa5b-4358-b35d-2fb675a90563", "metadata": {}, "outputs": [], - "source": [ - "graph_model_instance = GraphModel(\n", - " id=\"user123\",\n", - " documents=[\n", - " Document(\n", - " doc_id=\"doc1\",\n", - " title=\"Document 1\",\n", - " summary=\"Summary of Document 1\",\n", - " content_id=\"content_id_for_doc1\", # Assuming external content storage ID\n", - " doc_type=DocumentType(type_id=\"PDF\", description=\"Portable Document Format\"),\n", - " categories=[\n", - " Category(category_id=\"finance\", name=\"Finance\", default_relationship=Relationship(type=\"belongs_to\")),\n", - " Category(category_id=\"tech\", name=\"Technology\", default_relationship=Relationship(type=\"belongs_to\"))\n", - " ],\n", - " default_relationship=Relationship(type='has_document')\n", - " ),\n", - " Document(\n", - " doc_id=\"doc2\",\n", - " title=\"Document 2\",\n", - " summary=\"Summary of Document 2\",\n", - " content_id=\"content_id_for_doc2\",\n", - " doc_type=DocumentType(type_id=\"TXT\", description=\"Text File\"),\n", - " categories=[\n", - " Category(category_id=\"health\", name=\"Health\", default_relationship=Relationship(type=\"belongs_to\")),\n", - " Category(category_id=\"wellness\", name=\"Wellness\", default_relationship=Relationship(type=\"belongs_to\"))\n", - " ],\n", - " default_relationship=Relationship(type='has_document')\n", - " )\n", - " ],\n", - " user_properties=UserProperties(\n", - " custom_properties={\"age\": \"30\"},\n", - " location=UserLocation(location_id=\"ny\", description=\"New York\", default_relationship=Relationship(type='located_in'))\n", - " ),\n", - " default_fields={\n", - " 'created_at': datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"),\n", - " 'updated_at': datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n", - " }\n", - ")\n" - ] + "source": [] }, { "cell_type": "code", @@ -4646,14 +4668,14 @@ } ], "source": [ - " print(\"Nodes and their data:\")\n", - " for node, data in graph.nodes(data=True):\n", - " print(node, data)\n", + " # print(\"Nodes and their data:\")\n", + " # for node, data in graph.nodes(data=True):\n", + " # print(node, data)\n", "\n", - " # Print edges with their data\n", - " print(\"\\nEdges and their data:\")\n", - " for source, target, data in graph.edges(data=True):\n", - " print(f\"{source} -> {target} {data}\")" + " # # Print edges with their data\n", + " # print(\"\\nEdges and their data:\")\n", + " # for source, target, data in graph.edges(data=True):\n", + " # print(f\"{source} -> {target} {data}\")" ] }, { diff --git a/cognitive_architecture/api/v1/cognify/cognify.py b/cognitive_architecture/api/v1/cognify/cognify.py index 6c7850f50..8bf8f4480 100644 --- a/cognitive_architecture/api/v1/cognify/cognify.py +++ b/cognitive_architecture/api/v1/cognify/cognify.py @@ -17,7 +17,7 @@ from cognitive_architecture.infrastructure.databases.vector.qdrant.adapter impor from cognitive_architecture.infrastructure.llm.get_llm_client import get_llm_client from cognitive_architecture.modules.cognify.graph.add_classification_nodes import add_classification_nodes from cognitive_architecture.modules.cognify.graph.add_node_connections import add_node_connection, graph_ready_output, \ - connect_nodes_in_graph + connect_nodes_in_graph, extract_node_descriptions from cognitive_architecture.modules.cognify.graph.add_propositions import append_to_graph from cognitive_architecture.modules.cognify.llm.add_node_connection_embeddings import process_items from cognitive_architecture.modules.cognify.vector.batch_search import adapted_qdrant_batch_search @@ -100,11 +100,14 @@ async def cognify(input_text:str): # Run the async function for each set of cognitive layers layer_1_graph = await async_graph_per_layer(input_article_one, cognitive_layers_one) - print(layer_1_graph) - - - + # print(layer_1_graph) + # + # + # graph_client = get_graph_client(GraphDBType.NETWORKX) + # + # ADD SUMMARY + # ADD CATEGORIES # Define a GraphModel instance with example data graph_model_instance = DefaultGraphModel( @@ -154,23 +157,15 @@ async def cognify(input_text:str): F, unique_layer_uuids = await append_to_graph(layer_1_graph, required_layers_one, graph_client) - def extract_node_descriptions(data): - descriptions = [] - for node_id, attributes in data: - if 'description' in attributes and 'id' in attributes: - descriptions.append({'node_id': attributes['id'], 'description': attributes['description'], - 'layer_uuid': attributes['layer_uuid'], - 'layer_decomposition_uuid': attributes['layer_decomposition_uuid']}) - return descriptions - - # Extract the node descriptions + # # Extract the node descriptions + await graph_client.load_graph_from_file() graph = graph_client.graph - node_descriptions = extract_node_descriptions(graph.nodes(data=True)) - # unique_layer_uuids = set(node['layer_decomposition_uuid'] for node in node_descriptions) - # + node_descriptions = await extract_node_descriptions(graph.nodes(data=True)) + unique_layer_uuids = set(node['layer_decomposition_uuid'] for node in node_descriptions) + # # db = get_vector_database() - - + # + # collection_config = CollectionConfig( vector_config={ 'content': models.VectorParams( @@ -181,27 +176,49 @@ async def cognify(input_text:str): # Set other configs as needed ) - for layer in unique_layer_uuids: - await db.create_collection(layer,collection_config) + from qdrant_client import QdrantClient + try: + for layer in unique_layer_uuids: + await db.create_collection(layer,collection_config) + except: + pass - # #to check if it works + # qdrant = QdrantClient( + # url=os.getenv('QDRANT_URL'), + # api_key=os.getenv('QDRANT_API_KEY')) # - await add_propositions(node_descriptions, db) + # collections_response = qdrant.http.collections_api.get_collections() + # collections = collections_response.result.collections + # print(collections) + + + # print(node_descriptions) + # + await add_propositions(node_descriptions) from cognitive_architecture.infrastructure.databases.vector.qdrant.adapter import AsyncQdrantClient grouped_data = await add_node_connection(graph_client, db, node_descriptions) + # print("we are here, grouped_data", grouped_data) + llm_client = get_llm_client() + + relationship_dict = await process_items(grouped_data, unique_layer_uuids, llm_client) + + # print("we are here", relationship_dict[0]) + results = await adapted_qdrant_batch_search(relationship_dict, db) + # print(results) relationship_d = graph_ready_output(results) + # print(relationship_d) CONNECTED_GRAPH = connect_nodes_in_graph(F, relationship_d) - out = await render_graph(CONNECTED_GRAPH, graph_type='networkx') + out = await render_graph(CONNECTED_GRAPH.graph, graph_type='networkx') print(out) return CONNECTED_GRAPH @@ -228,4 +245,8 @@ async def cognify(input_text:str): if __name__ == "__main__": - asyncio.run(cognify("The quick brown fox jumps over the lazy dog")) \ No newline at end of file + asyncio.run(cognify("""In the nicest possible way, Britons have always been a bit silly about animals. “Keeping pets, for the English, is not so much a leisure activity as it is an entire way of life,” wrote the anthropologist Kate Fox in Watching the English, nearly 20 years ago. Our dogs, in particular, have been an acceptable outlet for emotions and impulses we otherwise keep strictly controlled – our latent desire to be demonstratively affectionate, to be silly and chat to strangers. If this seems like an exaggeration, consider the different reactions you’d get if you struck up a conversation with someone in a park with a dog, versus someone on the train. +Indeed, British society has been set up to accommodate these four-legged ambassadors. In the UK – unlike Australia, say, or New Zealand – dogs are not just permitted on public transport but often openly encouraged. Many pubs and shops display waggish signs, reading, “Dogs welcome, people tolerated”, and have treat jars on their counters. The other day, as I was waiting outside a cafe with a friend’s dog, the barista urged me to bring her inside. +For years, Britons’ non-partisan passion for animals has been consistent amid dwindling common ground. But lately, rather than bringing out the best in us, our relationship with dogs is increasingly revealing us at our worst – and our supposed “best friends” are paying the price. +As with so many latent traits in the national psyche, it all came unleashed with the pandemic, when many people thought they might as well make the most of all that time at home and in local parks with a dog. Between 2019 and 2022, the number of pet dogs in the UK rose from about nine million to 13 million. But there’s long been a seasonal surge around this time of year, substantial enough for the Dogs Trust charity to coin its famous slogan back in 1978: “A dog is for life, not just for Christmas.” +""")) \ No newline at end of file diff --git a/cognitive_architecture/infrastructure/databases/vector/qdrant/adapter.py b/cognitive_architecture/infrastructure/databases/vector/qdrant/adapter.py index 152a8891f..c0579a4b1 100644 --- a/cognitive_architecture/infrastructure/databases/vector/qdrant/adapter.py +++ b/cognitive_architecture/infrastructure/databases/vector/qdrant/adapter.py @@ -60,6 +60,16 @@ class QDrantAdapter(VectorDBInterface): points = data_points ) + async def search(self, collection_name: str, query_vector: List[float], limit: int, with_vector: bool = False): + client = self.get_qdrant_client() + + return await client.search( + collection_name = collection_name, + query_vector = query_vector, + limit = limit, + with_vectors = with_vector + ) + async def batch_search(self, collection_name: str, embeddings: List[List[float]], with_vectors: List[bool] = None): diff --git a/cognitive_architecture/infrastructure/databases/vector/vector_db_interface.py b/cognitive_architecture/infrastructure/databases/vector/vector_db_interface.py index 5f4ad9031..3f4e04d96 100644 --- a/cognitive_architecture/infrastructure/databases/vector/vector_db_interface.py +++ b/cognitive_architecture/infrastructure/databases/vector/vector_db_interface.py @@ -68,12 +68,15 @@ class VectorDBInterface(Protocol): # data_point_id: str # ): raise NotImplementedError """ Search """ - # @abstractmethod - # async def search( - # self, - # collection_name: str, - # query: object - # ): raise NotImplementedError + @abstractmethod + async def search( + self, + collection_name: str, + query_vector: List[float], + limit: int, + with_vector: bool = False + + ): raise NotImplementedError @abstractmethod async def batch_search( diff --git a/cognitive_architecture/modules/cognify/graph/add_node_connections.py b/cognitive_architecture/modules/cognify/graph/add_node_connections.py index 66032d682..493aeb97b 100644 --- a/cognitive_architecture/modules/cognify/graph/add_node_connections.py +++ b/cognitive_architecture/modules/cognify/graph/add_node_connections.py @@ -2,11 +2,11 @@ from cognitive_architecture.infrastructure.databases.graph.get_graph_client impo from cognitive_architecture.shared.data_models import GraphDBType -def extract_node_descriptions(data): +async def extract_node_descriptions(data): descriptions = [] for node_id, attributes in data: - if 'description' in attributes and 'id' in attributes: - descriptions.append({'node_id': attributes['id'], 'description': attributes['description'], 'layer_uuid': attributes['layer_uuid'], 'layer_decomposition_uuid': attributes['layer_decomposition_uuid'] }) + if 'description' in attributes and 'unique_id' in attributes: + descriptions.append({'node_id': attributes['unique_id'], 'description': attributes['description'], 'layer_uuid': attributes['layer_uuid'], 'layer_decomposition_uuid': attributes['layer_decomposition_uuid'] }) return descriptions @@ -15,7 +15,7 @@ def extract_node_descriptions(data): async def add_node_connection(graph_client, vector_database_client, data): graph = graph_client.graph - node_descriptions = extract_node_descriptions(graph.nodes(data=True)) + node_descriptions = data grouped_data = {} @@ -56,10 +56,10 @@ def connect_nodes_in_graph(graph, relationship_dict): # Find nodes in the graph that match the searched_node_id and score_id from their attributes for node, attrs in graph.nodes(data=True): - if 'id' in attrs: # Ensure there is an 'id' attribute - if attrs['id'] == searched_node_attr_id: + if 'unique_id' in attrs: # Ensure there is an 'id' attribute + if attrs['unique_id'] == searched_node_attr_id: searched_node_key = node - elif attrs['id'] == score_attr_id: + elif attrs['unique_id'] == score_attr_id: score_node_key = node # If both nodes are found, no need to continue checking other nodes diff --git a/cognitive_architecture/modules/cognify/graph/add_semantic_search_connection.py b/cognitive_architecture/modules/cognify/graph/add_semantic_search_connection.py index 71cff166a..e69de29bb 100644 --- a/cognitive_architecture/modules/cognify/graph/add_semantic_search_connection.py +++ b/cognitive_architecture/modules/cognify/graph/add_semantic_search_connection.py @@ -1,101 +0,0 @@ - - - - - -async def process_items(grouped_data, unique_layer_uuids): - results_to_check = [] # This will hold results excluding self comparisons - tasks = [] # List to hold all tasks - task_to_info = {} # Dictionary to map tasks to their corresponding group id and item info - - # Iterate through each group in grouped_data - for group_id, items in grouped_data.items(): - # Filter unique_layer_uuids to exclude the current group_id - target_uuids = [uuid for uuid in unique_layer_uuids if uuid != group_id] - - # Process each item in the group - for item in items: - # For each target UUID, create an async task for the item's embedding retrieval - for target_id in target_uuids: - task = asyncio.create_task \ - (async_get_embedding_with_backoff(item['description'], "text-embedding-3-large")) - tasks.append(task) - # Map the task to the target id, item's node_id, and description for later retrieval - task_to_info[task] = (target_id, item['node_id'], group_id, item['description']) - - # Await all tasks to complete and gather results - results = await asyncio.gather(*tasks) - - # Process the results, associating them with their target id, node id, and description - for task, embedding in zip(tasks, results): - - target_id, node_id ,group_id, description = task_to_info[task] - results_to_check.append([target_id, embedding, description, node_id, group_id]) - - return results_to_check - -async def graph_ready_output(results): - relationship_dict ={} - - for result_tuple in results: - - uuid, scored_points_list, desc, node_id = result_tuple - # Unpack the tuple - - # Ensure there's a list to collect related items for this uuid - if uuid not in relationship_dict: - relationship_dict[uuid] = [] - - for scored_points in scored_points_list: # Iterate over the list of ScoredPoint lists - for scored_point in scored_points: # Iterate over each ScoredPoint object - if scored_point.score > 0.9: # Check the score condition - # Append a new dictionary to the list associated with the uuid - relationship_dict[uuid].append({ - 'collection_name_uuid': uuid, - 'searched_node_id': scored_point.id, - 'score': scored_point.score, - 'score_metadata': scored_point.payload, - 'original_id_for_search': node_id, - }) - return relationship_dict - -async def connect_nodes_in_graph(graph, relationship_dict): - """ - For each relationship in relationship_dict, check if both nodes exist in the graph based on node attributes. - If they do, create a connection (edge) between them. - - :param graph: A NetworkX graph object - :param relationship_dict: A dictionary containing relationships between nodes - """ - for id, relationships in relationship_dict.items(): - for relationship in relationships: - searched_node_attr_id = relationship['searched_node_id'] - print(searched_node_attr_id) - score_attr_id = relationship['original_id_for_search'] - score = relationship['score'] - - - # Initialize node keys for both searched_node and score_node - searched_node_key, score_node_key = None, None - - # Find nodes in the graph that match the searched_node_id and score_id from their attributes - for node, attrs in graph.nodes(data=True): - if 'id' in attrs: # Ensure there is an 'id' attribute - if attrs['id'] == searched_node_attr_id: - searched_node_key = node - elif attrs['id'] == score_attr_id: - score_node_key = node - - # If both nodes are found, no need to continue checking other nodes - if searched_node_key and score_node_key: - break - - # Check if both nodes were found in the graph - if searched_node_key is not None and score_node_key is not None: - print(searched_node_key) - print(score_node_key) - # If both nodes exist, create an edge between them - # You can customize the edge attributes as needed, here we use 'score' as an attribute - graph.add_edge(searched_node_key, score_node_key, weight=score, score_metadata=relationship.get('score_metadata')) - - return graph diff --git a/cognitive_architecture/modules/cognify/llm/add_node_connection_embeddings.py b/cognitive_architecture/modules/cognify/llm/add_node_connection_embeddings.py index ec86d7e71..549195123 100644 --- a/cognitive_architecture/modules/cognify/llm/add_node_connection_embeddings.py +++ b/cognitive_architecture/modules/cognify/llm/add_node_connection_embeddings.py @@ -15,8 +15,7 @@ async def process_items(grouped_data, unique_layer_uuids, llm_client): for item in items: # For each target UUID, create an async task for the item's embedding retrieval for target_id in target_uuids: - task = asyncio.create_task( - llm_client.async_get_embedding_with_backoff(item['description'], "text-embedding-3-large")) + task = asyncio.create_task(llm_client.async_get_embedding_with_backoff(item['description'], "text-embedding-3-large")) tasks.append(task) # Map the task to the target id, item's node_id, and description for later retrieval task_to_info[task] = (target_id, item['node_id'], group_id, item['description']) @@ -24,6 +23,7 @@ async def process_items(grouped_data, unique_layer_uuids, llm_client): # Await all tasks to complete and gather results results = await asyncio.gather(*tasks) + # Process the results, associating them with their target id, node id, and description for task, embedding in zip(tasks, results): target_id, node_id, group_id, description = task_to_info[task] diff --git a/cognitive_architecture/modules/cognify/vector/batch_search.py b/cognitive_architecture/modules/cognify/vector/batch_search.py index 752a18254..f51586e08 100644 --- a/cognitive_architecture/modules/cognify/vector/batch_search.py +++ b/cognitive_architecture/modules/cognify/vector/batch_search.py @@ -29,4 +29,6 @@ async def adapted_qdrant_batch_search(results_to_check,vector_client): if __name__ == '__main__': - client = get_vector_database() \ No newline at end of file + client = get_vector_database() + + adapted_qdrant_batch_search() \ No newline at end of file diff --git a/cognitive_architecture/modules/cognify/vector/load_propositions.py b/cognitive_architecture/modules/cognify/vector/load_propositions.py index 79b53d01d..2a7d401f2 100644 --- a/cognitive_architecture/modules/cognify/vector/load_propositions.py +++ b/cognitive_architecture/modules/cognify/vector/load_propositions.py @@ -4,19 +4,19 @@ from cognitive_architecture.infrastructure.llm.get_llm_client import get_llm_cli from qdrant_client import models from cognitive_architecture.infrastructure.databases.vector.get_vector_database import get_vector_database -async def get_embeddings(texts): +async def get_embeddings(texts:list): + """ Get embeddings for a list of texts""" client = get_llm_client() tasks = [ client.async_get_embedding_with_backoff(text, "text-embedding-3-large") for text in texts] results = await asyncio.gather(*tasks) return results -async def upload_embedding(id, metadata, some_embeddings, collection_name, client): - print(id) - # if some_embeddings and isinstance(some_embeddings[0], list): - # some_embeddings = [item for sublist in some_embeddings for item in sublist] - - client.upload_points( +async def upload_embedding(id, metadata, some_embeddings, collection_name): + """ Upload a single embedding to a collection in Qdrant.""" + client = get_vector_database() + # print("Uploading embeddings") + await client.create_data_points( collection_name=collection_name, - points=[ + data_points=[ models.PointStruct( id=id, vector={"content" :some_embeddings}, payload=metadata ) @@ -25,10 +25,11 @@ async def upload_embedding(id, metadata, some_embeddings, collection_name, clien ) -async def add_propositions(node_descriptions, client): +async def add_propositions(node_descriptions): for item in node_descriptions: - print(item['node_id']) - embeddings = await get_embeddings(item['description']) - await upload_embedding(id = item['node_id'], metadata = {"meta":item['description']}, some_embeddings = embeddings[0], collection_name= item['layer_decomposition_uuid'],client= client) + embeddings = await get_embeddings([item['description']]) + await upload_embedding(id = item['node_id'], metadata = {"meta":item['description']}, + some_embeddings = embeddings[0], + collection_name= item['layer_decomposition_uuid'])