gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refacotr: Add current development status

Browse source 
Save current development status

Refactor
This commit is contained in:
Igor Ilic 2024-12-18 17:10:08 +01:00
parent f6800b979e
commit ee6bc17032
2 changed files with 99 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
cognee/infrastructure/databases/relational/sqlalchemy/SqlAlchemyAdapter.py
View file

@ -5,10 +5,12 @@ from typing import AsyncGenerator, List
from contextlib import asynccontextmanager
from sqlalchemy import text, select, MetaData, Table, delete
from sqlalchemy.orm import joinedload
from sqlalchemy.exc import NoResultFound
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
from cognee.infrastructure.databases.exceptions import EntityNotFoundError
from cognee.modules.data.models.Data import Data
from ..ModelBase import Base
class SQLAlchemyAdapter():
@ -119,12 +121,37 @@ class SQLAlchemyAdapter():
# so must be enabled for each database connection/session separately.
await session.execute(text("PRAGMA foreign_keys = ON;"))
data_entity = await session.execute(select(Data).where(Data.id == data_id))
data_entity = await session.scalars(select(Data).where(Data.id == data_id)).one()
# Check if other data objects point to the same raw data location
raw_data_location_entities= await session.execute(
select(Data).where(Data.raw_data_location == data_entity.raw_data_location)).all()
# Don't delete local file unless this is the only reference to the file in the database
if len(raw_data_location_entities) == 1:
# delete local file
from cognee.base_config import get_base_config
config get_base_config()
await session.execute(delete(Data).where(Data.id == data_id))
await session.commit()
else:
async with self.get_async_session() as session:
try:
data_entity = (await session.scalars(select(Data).where(Data.id == data_id))).one()
except (ValueError, NoResultFound) as e:
raise EntityNotFoundError(message=f"Entity not found: {str(e)}")
# Check if other data objects point to the same raw data location
raw_data_location_entities = (await session.execute(
select(Data).where(Data.raw_data_location == data_entity.raw_data_location))).all()
# Don't delete local file unless this is the only reference to the file in the database
if len(raw_data_location_entities) == 1:
# delete local file
pass
await session.execute(delete(Data).where(Data.id == data_id))
await session.commit()

137
cognee/tests/test_pgvector.py
View file

@ -42,76 +42,81 @@ async def main():
)
cognee.config.system_root_directory(cognee_directory_path)
await cognee.prune.prune_data()
await cognee.prune.prune_system(metadata = True)
from cognee.infrastructure.databases.relational.get_relational_engine import get_relational_engine
dataset_name_1 = "natural_language"
dataset_name_2 = "quantum"
engine = get_relational_engine()
await engine.delete_data_entity("2cba57c3-d7ec-5746-b819-d89f87f05c18")
explanation_file_path = os.path.join(
pathlib.Path(__file__).parent, "test_data/Natural_language_processing.txt"
)
await cognee.add([explanation_file_path], dataset_name_1)
# await cognee.prune.prune_data()
# await cognee.prune.prune_system(metadata = True)
text = """A quantum computer is a computer that takes advantage of quantum mechanical phenomena.
At small scales, physical matter exhibits properties of both particles and waves, and quantum computing leverages this behavior, specifically quantum superposition and entanglement, using specialized hardware that supports the preparation and manipulation of quantum states.
Classical physics cannot explain the operation of these quantum devices, and a scalable quantum computer could perform some calculations exponentially faster (with respect to input size scaling) than any modern "classical" computer. In particular, a large-scale quantum computer could break widely used encryption schemes and aid physicists in performing physical simulations; however, the current state of the technology is largely experimental and impractical, with several obstacles to useful applications. Moreover, scalable quantum computers do not hold promise for many practical tasks, and for many important tasks quantum speedups are proven impossible.
The basic unit of information in quantum computing is the qubit, similar to the bit in traditional digital electronics. Unlike a classical bit, a qubit can exist in a superposition of its two "basis" states. When measuring a qubit, the result is a probabilistic output of a classical bit, therefore making quantum computers nondeterministic in general. If a quantum computer manipulates the qubit in a particular way, wave interference effects can amplify the desired measurement results. The design of quantum algorithms involves creating procedures that allow a quantum computer to perform calculations efficiently and quickly.
Physically engineering high-quality qubits has proven challenging. If a physical qubit is not sufficiently isolated from its environment, it suffers from quantum decoherence, introducing noise into calculations. Paradoxically, perfectly isolating qubits is also undesirable because quantum computations typically need to initialize qubits, perform controlled qubit interactions, and measure the resulting quantum states. Each of those operations introduces errors and suffers from noise, and such inaccuracies accumulate.
In principle, a non-quantum (classical) computer can solve the same computational problems as a quantum computer, given enough time. Quantum advantage comes in the form of time complexity rather than computability, and quantum complexity theory shows that some quantum algorithms for carefully selected tasks require exponentially fewer computational steps than the best known non-quantum algorithms. Such tasks can in theory be solved on a large-scale quantum computer whereas classical computers would not finish computations in any reasonable amount of time. However, quantum speedup is not universal or even typical across computational tasks, since basic tasks such as sorting are proven to not allow any asymptotic quantum speedup. Claims of quantum supremacy have drawn significant attention to the discipline, but are demonstrated on contrived tasks, while near-term practical use cases remain limited.
"""
# dataset_name_1 = "natural_language"
# dataset_name_2 = "quantum"
#
# explanation_file_path = os.path.join(
# pathlib.Path(__file__).parent, "test_data/Natural_language_processing.txt"
# )
# await cognee.add([explanation_file_path], dataset_name_1)
#
# text = """A quantum computer is a computer that takes advantage of quantum mechanical phenomena.
# At small scales, physical matter exhibits properties of both particles and waves, and quantum computing leverages this behavior, specifically quantum superposition and entanglement, using specialized hardware that supports the preparation and manipulation of quantum states.
# Classical physics cannot explain the operation of these quantum devices, and a scalable quantum computer could perform some calculations exponentially faster (with respect to input size scaling) than any modern "classical" computer. In particular, a large-scale quantum computer could break widely used encryption schemes and aid physicists in performing physical simulations; however, the current state of the technology is largely experimental and impractical, with several obstacles to useful applications. Moreover, scalable quantum computers do not hold promise for many practical tasks, and for many important tasks quantum speedups are proven impossible.
# The basic unit of information in quantum computing is the qubit, similar to the bit in traditional digital electronics. Unlike a classical bit, a qubit can exist in a superposition of its two "basis" states. When measuring a qubit, the result is a probabilistic output of a classical bit, therefore making quantum computers nondeterministic in general. If a quantum computer manipulates the qubit in a particular way, wave interference effects can amplify the desired measurement results. The design of quantum algorithms involves creating procedures that allow a quantum computer to perform calculations efficiently and quickly.
# Physically engineering high-quality qubits has proven challenging. If a physical qubit is not sufficiently isolated from its environment, it suffers from quantum decoherence, introducing noise into calculations. Paradoxically, perfectly isolating qubits is also undesirable because quantum computations typically need to initialize qubits, perform controlled qubit interactions, and measure the resulting quantum states. Each of those operations introduces errors and suffers from noise, and such inaccuracies accumulate.
# In principle, a non-quantum (classical) computer can solve the same computational problems as a quantum computer, given enough time. Quantum advantage comes in the form of time complexity rather than computability, and quantum complexity theory shows that some quantum algorithms for carefully selected tasks require exponentially fewer computational steps than the best known non-quantum algorithms. Such tasks can in theory be solved on a large-scale quantum computer whereas classical computers would not finish computations in any reasonable amount of time. However, quantum speedup is not universal or even typical across computational tasks, since basic tasks such as sorting are proven to not allow any asymptotic quantum speedup. Claims of quantum supremacy have drawn significant attention to the discipline, but are demonstrated on contrived tasks, while near-term practical use cases remain limited.
# """
#
# await cognee.add([text], dataset_name_2)
#
# await cognee.cognify([dataset_name_2, dataset_name_1])
#
# from cognee.infrastructure.databases.vector import get_vector_engine
#
# # Test getting of documents for search per dataset
# from cognee.modules.users.permissions.methods import get_document_ids_for_user
# user = await get_default_user()
# document_ids = await get_document_ids_for_user(user.id, [dataset_name_1])
# assert len(document_ids) == 1, f"Number of expected documents doesn't match {len(document_ids)} != 1"
#
# # Test getting of documents for search when no dataset is provided
# user = await get_default_user()
# document_ids = await get_document_ids_for_user(user.id)
# assert len(document_ids) == 2, f"Number of expected documents doesn't match {len(document_ids)} != 2"
#
# vector_engine = get_vector_engine()
# random_node = (await vector_engine.search("entity_name", "Quantum computer"))[0]
# random_node_name = random_node.payload["text"]
#
# search_results = await cognee.search(SearchType.INSIGHTS, query_text = random_node_name)
# assert len(search_results) != 0, "The search results list is empty."
# print("\n\nExtracted sentences are:\n")
# for result in search_results:
# print(f"{result}\n")
#
# search_results = await cognee.search(SearchType.CHUNKS, query_text = random_node_name, datasets=[dataset_name_2])
# assert len(search_results) != 0, "The search results list is empty."
# print("\n\nExtracted chunks are:\n")
# for result in search_results:
# print(f"{result}\n")
#
# search_results = await cognee.search(SearchType.SUMMARIES, query_text = random_node_name)
# assert len(search_results) != 0, "Query related summaries don't exist."
# print("\n\nExtracted summaries are:\n")
# for result in search_results:
# print(f"{result}\n")
#
# history = await cognee.get_search_history()
# assert len(history) == 6, "Search history is not correct."
#
# results = await brute_force_triplet_search('What is a quantum computer?')
# assert len(results) > 0
await cognee.add([text], dataset_name_2)
await cognee.cognify([dataset_name_2, dataset_name_1])
from cognee.infrastructure.databases.vector import get_vector_engine
# Test getting of documents for search per dataset
from cognee.modules.users.permissions.methods import get_document_ids_for_user
user = await get_default_user()
document_ids = await get_document_ids_for_user(user.id, [dataset_name_1])
assert len(document_ids) == 1, f"Number of expected documents doesn't match {len(document_ids)} != 1"
# Test getting of documents for search when no dataset is provided
user = await get_default_user()
document_ids = await get_document_ids_for_user(user.id)
assert len(document_ids) == 2, f"Number of expected documents doesn't match {len(document_ids)} != 2"
vector_engine = get_vector_engine()
random_node = (await vector_engine.search("entity_name", "Quantum computer"))[0]
random_node_name = random_node.payload["text"]
search_results = await cognee.search(SearchType.INSIGHTS, query_text = random_node_name)
assert len(search_results) != 0, "The search results list is empty."
print("\n\nExtracted sentences are:\n")
for result in search_results:
print(f"{result}\n")
search_results = await cognee.search(SearchType.CHUNKS, query_text = random_node_name, datasets=[dataset_name_2])
assert len(search_results) != 0, "The search results list is empty."
print("\n\nExtracted chunks are:\n")
for result in search_results:
print(f"{result}\n")
search_results = await cognee.search(SearchType.SUMMARIES, query_text = random_node_name)
assert len(search_results) != 0, "Query related summaries don't exist."
print("\n\nExtracted summaries are:\n")
for result in search_results:
print(f"{result}\n")
history = await cognee.get_search_history()
assert len(history) == 6, "Search history is not correct."
results = await brute_force_triplet_search('What is a quantum computer?')
assert len(results) > 0
await cognee.prune.prune_data()
assert not os.path.isdir(data_directory_path), "Local data files are not deleted"
await cognee.prune.prune_system(metadata=True)
tables_in_database = await vector_engine.get_table_names()
assert len(tables_in_database) == 0, "PostgreSQL database is not empty"
# await cognee.prune.prune_data()
# assert not os.path.isdir(data_directory_path), "Local data files are not deleted"
#
# await cognee.prune.prune_system(metadata=True)
# tables_in_database = await vector_engine.get_table_names()
# assert len(tables_in_database) == 0, "PostgreSQL database is not empty"
if __name__ == "__main__":
import asyncio