gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: remove unused/obsolete code

Browse source
This commit is contained in:
Boris Arzentar 2024-03-16 12:08:12 +01:00
parent f024e08b11
commit d96c51aeaf
56 changed files with 595 additions and 5611 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Demo_graph.ipynb
View file

@ -496,7 +496,7 @@
"metadata": {},
"outputs": [],
"source": [
"cognitive_layers_one = await content_to_cog_layers(\"generate_cog_layers.txt\", transformed_dict_1, response_model=DefaultCognitiveLayer)"
"cognitive_layers_one = await content_to_cog_layers(transformed_dict_1, response_model=DefaultCognitiveLayer)"
]
},
{
@ -506,7 +506,7 @@
"metadata": {},
"outputs": [],
"source": [
"cognitive_layers_two = await content_to_cog_layers(\"generate_cog_layers.txt\", transformed_dict_2, response_model=DefaultCognitiveLayer)"
"cognitive_layers_two = await content_to_cog_layers(transformed_dict_2, response_model=DefaultCognitiveLayer)"
]
},
{
@ -4708,4 +4708,4 @@
},
"nbformat": 4,
"nbformat_minor": 5
}
}

0
__init__.py
View file

432
api.py
View file

@ -1,432 +0,0 @@
import os
import json
import uvicorn
from fastapi import Depends
import logging
# Set up logging
logging.basicConfig(
level=logging.INFO, # Set the logging level (e.g., DEBUG, INFO, WARNING, ERROR, CRITICAL)
format="%(asctime)s [%(levelname)s] %(message)s", # Set the log message format
)
logger = logging.getLogger(__name__)
from cognee.config import Config
config = Config()
config.load()
from typing import Dict, Any, List
from fastapi import FastAPI, BackgroundTasks, HTTPException
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from cognee.database.relationaldb.database import AsyncSessionLocal
from cognee.database.relationaldb.database_crud import session_scope
from cognee.vectorstore_manager import Memory
from main import add_documents_to_graph_db, user_context_enrichment
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
app = FastAPI(debug=True)
#
# from auth.cognito.JWTBearer import JWTBearer
# from auth.auth import jwks
#
# auth = JWTBearer(jwks)
@app.get("/")
async def root():
"""
Root endpoint that returns a welcome message.
"""
return {"message": "Hello, World, I am alive!"}
@app.get("/health")
def health_check():
"""
Health check endpoint that returns the server status.
"""
return {"status": "OK"}
class Payload(BaseModel):
payload: Dict[str, Any]
from cognee.api.v1.memory.create_memory import MemoryType
class CreateMemoryPayload(BaseModel):
user_id: str
memory_name: str
memory_type: MemoryType
@app.post("/create-memory", response_model=dict)
async def create_memory(payload: CreateMemoryPayload):
from cognee.api.v1.memory.create_memory import create_memory as create_memory_v1, MemoryException
try:
await create_memory_v1(
payload.user_id,
payload.memory_name,
payload.memory_type or MemoryType.VECTOR,
)
except MemoryException as error:
return JSONResponse(
status_code = 409,
content = { "error": error.message }
)
return JSONResponse(
status_code = 200,
content = { "memory_name": payload.memory_name }
)
class RememberPayload(BaseModel):
user_id: str
memory_name: str
payload: List[str]
@app.post("/remember", response_model=dict)
async def remember(payload: RememberPayload):
from cognee.api.v1.memory.remember import remember as remember_v1
await remember_v1(
payload.user_id,
payload.memory_name,
payload.payload
)
return JSONResponse(
status_code = 200,
content = { "message": "ok" }
)
@app.post("/add-memory", response_model=dict)
async def add_memory(
payload: Payload,
# files: List[UploadFile] = File(...),
):
try:
logging.info(" Adding to Memory ")
decoded_payload = payload.payload
async with session_scope(session=AsyncSessionLocal()) as session:
from main import load_documents_to_vectorstore
if (
"settings" in decoded_payload
and decoded_payload["settings"] is not None
):
settings_for_loader = decoded_payload["settings"]
else:
settings_for_loader = None
if "content" in decoded_payload and decoded_payload["content"] is not None:
content = decoded_payload["content"]
else:
content = None
output = await load_documents_to_vectorstore(
session = session,
content = content,
user_id = decoded_payload["user_id"],
loader_settings = settings_for_loader,
)
return JSONResponse(content={"response": output}, status_code=200)
except Exception as e:
return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
@app.post("/add-architecture-public-memory", response_model=dict)
async def add_memory(
payload: Payload,
# files: List[UploadFile] = File(...),
):
try:
logging.info(" Adding to Memory ")
decoded_payload = payload.payload
async with session_scope(session=AsyncSessionLocal()) as session:
from main import load_documents_to_vectorstore
if "content" in decoded_payload and decoded_payload["content"] is not None:
content = decoded_payload["content"]
else:
content = None
user_id = "system_user"
loader_settings = {"format": "PDF", "source": "DEVICE", "path": [".data"]}
output = await load_documents_to_vectorstore(
user_id = user_id,
content = content,
session = session,
loader_settings = loader_settings,
)
return JSONResponse(content={"response": output}, status_code=200)
except Exception as e:
return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
@app.post("/user-query-to-graph")
async def user_query_to_graph(payload: Payload):
try:
from main import user_query_to_graph_db
decoded_payload = payload.payload
# Execute the query - replace this with the actual execution method
async with session_scope(session=AsyncSessionLocal()) as session:
# Assuming you have a method in Neo4jGraphDB to execute the query
result = await user_query_to_graph_db(
session=session,
user_id=decoded_payload["user_id"],
query_input=decoded_payload["query"],
)
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/document-to-graph-db")
async def document_to_graph_db(payload: Payload):
logging.info("Adding documents to graph db")
try:
decoded_payload = payload.payload
if "settings" in decoded_payload and decoded_payload["settings"] is not None:
settings_for_loader = decoded_payload["settings"]
else:
settings_for_loader = None
if (
"memory_type" in decoded_payload
and decoded_payload["memory_type"] is not None
):
memory_type = decoded_payload["memory_type"]
else:
memory_type = None
async with session_scope(session=AsyncSessionLocal()) as session:
result = await add_documents_to_graph_db(
session=session,
user_id=decoded_payload["user_id"],
document_memory_types=memory_type,
)
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/cognitive-context-enrichment")
async def cognitive_context_enrichment(payload: Payload):
try:
decoded_payload = payload.payload
async with session_scope(session=AsyncSessionLocal()) as session:
result = await user_context_enrichment(
session,
user_id=decoded_payload["user_id"],
query=decoded_payload["query"],
generative_response=decoded_payload["generative_response"],
memory_type=decoded_payload["memory_type"],
)
return JSONResponse(content={"response": result}, status_code=200)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/classify-user-query")
async def classify_user_query(payload: Payload):
try:
decoded_payload = payload.payload
async with session_scope(session=AsyncSessionLocal()) as session:
from main import relevance_feedback
result = await relevance_feedback(
query=decoded_payload["query"],
input_type=decoded_payload["knowledge_type"],
)
return JSONResponse(content={"response": result}, status_code=200)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/user-query-classifier")
async def user_query_classfier(payload: Payload):
try:
decoded_payload = payload.payload
# Execute the query - replace this with the actual execution method
async with session_scope(session=AsyncSessionLocal()) as session:
from cognee.classifiers.classify_user_input import (
classify_user_query,
)
# Assuming you have a method in Neo4jGraphDB to execute the query
result = await classify_user_query(
session, decoded_payload["user_id"], decoded_payload["query"]
)
return JSONResponse(content={"response": result}, status_code=200)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/drop-db")
async def drop_db(payload: Payload):
try:
decoded_payload = payload.payload
if decoded_payload["operation"] == "drop":
if os.environ.get("AWS_ENV") == "dev":
host = os.environ.get("POSTGRES_HOST")
username = os.environ.get("POSTGRES_USER")
password = os.environ.get("POSTGRES_PASSWORD")
database_name = os.environ.get("POSTGRES_DB")
else:
pass
from cognee.database.create_database import (
drop_database,
create_admin_engine,
)
engine = create_admin_engine(username, password, host, database_name)
connection = engine.raw_connection()
drop_database(connection, database_name)
return JSONResponse(content={"response": "DB dropped"}, status_code=200)
else:
if os.environ.get("AWS_ENV") == "dev":
host = os.environ.get("POSTGRES_HOST")
username = os.environ.get("POSTGRES_USER")
password = os.environ.get("POSTGRES_PASSWORD")
database_name = os.environ.get("POSTGRES_DB")
else:
pass
from cognee.database.create_database import (
create_database,
create_admin_engine,
)
engine = create_admin_engine(username, password, host, database_name)
connection = engine.raw_connection()
create_database(connection, database_name)
return JSONResponse(content={"response": " DB drop"}, status_code=200)
except Exception as e:
return HTTPException(status_code=500, detail=str(e))
@app.post("/create-public-memory")
async def create_public_memory(payload: Payload):
try:
decoded_payload = payload.payload
if "user_id" in decoded_payload and decoded_payload["user_id"] is not None:
user_id = decoded_payload["user_id"]
else:
user_id = None
if "labels" in decoded_payload and decoded_payload["labels"] is not None:
labels = decoded_payload["labels"]
else:
labels = None
if "topic" in decoded_payload and decoded_payload["topic"] is not None:
topic = decoded_payload["topic"]
else:
topic = None
# Execute the query - replace this with the actual execution method
# async with session_scope(session=AsyncSessionLocal()) as session:
# from main import create_public_memory
# Assuming you have a method in Neo4jGraphDB to execute the query
result = await create_public_memory(user_id=user_id, labels=labels, topic=topic)
return JSONResponse(content={"response": result}, status_code=200)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/attach-user-to-public-memory")
async def attach_user_to_public_memory(payload: Payload):
try:
decoded_payload = payload.payload
if "topic" in decoded_payload and decoded_payload["topic"] is not None:
topic = decoded_payload["topic"]
else:
topic = None
if "labels" in decoded_payload and decoded_payload["labels"] is not None:
labels = decoded_payload["labels"]
else:
labels = ["sr"]
# Execute the query - replace this with the actual execution method
async with session_scope(session=AsyncSessionLocal()) as session:
from main import attach_user_to_memory, create_public_memory
# Assuming you have a method in Neo4jGraphDB to execute the query
await create_public_memory(
user_id=decoded_payload["user_id"], topic=topic, labels=labels
)
result = await attach_user_to_memory(
user_id=decoded_payload["user_id"], topic=topic, labels=labels
)
return JSONResponse(content={"response": result}, status_code=200)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/unlink-user-from-public-memory")
async def unlink_user_from_public_memory(payload: Payload):
try:
decoded_payload = payload.payload
if "topic" in decoded_payload and decoded_payload["topic"] is not None:
topic = decoded_payload["topic"]
else:
topic = None
# Execute the query - replace this with the actual execution method
async with session_scope(session=AsyncSessionLocal()) as session:
from main import unlink_user_from_memory
# Assuming you have a method in Neo4jGraphDB to execute the query
result = await unlink_user_from_memory(
user_id=decoded_payload["user_id"],
topic=topic,
labels=decoded_payload["labels"],
)
return JSONResponse(content={"response": result}, status_code=200)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
def start_api_server(host: str = "0.0.0.0", port: int = 8000):
"""
Start the API server using uvicorn.
Parameters:
host (str): The host for the server.
port (int): The port for the server.
"""
try:
logger.info(f"Starting server at {host}:{port}")
uvicorn.run(app, host=host, port=port)
except Exception as e:
logger.exception(f"Failed to start server: {e}")
# Here you could add any cleanup code or error recovery code.
if __name__ == "__main__":
start_api_server()

692
cognee - Get Started.ipynb
View file

@ -194,7 +194,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 1,
"id": "5b3954c1-f537-4be7-a578-1d5037c21374",
"metadata": {},
"outputs": [
@ -202,10 +202,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Pipeline file_load_from_filesystem load step completed in 0.29 seconds\n",
"Pipeline file_load_from_filesystem load step completed in 0.34 seconds\n",
"1 load package(s) were loaded to destination duckdb and into dataset izmene\n",
"The duckdb destination used duckdb:///:external: location to store data\n",
"Load package 1710408302.8475971 is LOADED and contains no failed jobs\n"
"Load package 1710586934.8904011 is LOADED and contains no failed jobs\n"
]
}
],
@ -223,7 +223,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 2,
"id": "39df49ca-06f0-4b86-ae27-93c68ddceac3",
"metadata": {},
"outputs": [
@ -231,7 +231,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"/Users/vasa/Projects/cognee/cognee/data/cognee/cognee.duckdb\n",
"/Users/borisarzentar/Projects/Topoteretes/cognee/cognee/data/cognee/cognee.duckdb\n",
"['izmene']\n"
]
}
@ -264,7 +264,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 3,
"id": "97e8647a-052c-4689-b1de-8d81765462e0",
"metadata": {},
"outputs": [
@ -277,9 +277,8 @@
"Document (881ecb36-2819-54c3-8147-ed80293084d6) categorized: ['Laws, regulations, and legal case documents']\n",
"Document (881ecb36-2819-54c3-8147-ed80293084d6) layer graphs created\n",
"Document (881ecb36-2819-54c3-8147-ed80293084d6) layers connected\n",
"Error during batch search for ID 6b81d9c2-4121-4366-ab53-71511ce61371: \n",
"Document (881ecb36-2819-54c3-8147-ed80293084d6) processed\n",
"Graph is visualized at: https://hub.graphistry.com/graph/graph.html?dataset=2c27d75e0d754e5c95c59d39dd2fe1ee&type=arrow&viztoken=3ddceed7-539d-49fd-ab33-2ca0691028d1&usertag=0b63180a-pygraphistry-0.33.5&splashAfter=1710408590&info=true\n",
"Graph is visualized at: https://hub.graphistry.com/graph/graph.html?dataset=037fedd82ee3490aaf52ee8e535cc3fc&type=arrow&viztoken=f05741c9-99ea-4397-a99e-34c57949c17b&usertag=e15d5b35-pygraphistry-0.33.5&splashAfter=1710587061&info=true\n",
"None\n"
]
}
@ -289,14 +288,9 @@
"from cognee import cognify, list_datasets\n",
"from cognee.utils import render_graph\n",
"\n",
"# text = \"\"\"In the nicest possible way, Britons have always been a bit silly about animals. “Keeping pets, for the English, is not so much a leisure activity as it is an entire way of life,” wrote the anthropologist Kate Fox in Watching the English, nearly 20 years ago. Our dogs, in particular, have been an acceptable outlet for emotions and impulses we otherwise keep strictly controlled – our latent desire to be demonstratively affectionate, to be silly and chat to strangers. If this seems like an exaggeration, consider the different reactions you’d get if you struck up a conversation with someone in a park with a dog, versus someone on the train.\n",
"# Indeed, British society has been set up to accommodate these four-legged ambassadors. In the UK – unlike Australia, say, or New Zealand – dogs are not just permitted on public transport but often openly encouraged. Many pubs and shops display waggish signs, reading, “Dogs welcome, people tolerated”, and have treat jars on their counters. The other day, as I was waiting outside a cafe with a friend’s dog, the barista urged me to bring her inside.\n",
"# For years, Britons’ non-partisan passion for animals has been consistent amid dwindling common ground. But lately, rather than bringing out the best in us, our relationship with dogs is increasingly revealing us at our worst – and our supposed “best friends” are paying the price.\n",
"# As with so many latent traits in the national psyche, it all came unleashed with the pandemic, when many people thought they might as well make the most of all that time at home and in local parks with a dog. Between 2019 and 2022, the number of pet dogs in the UK rose from about nine million to 13 million. But there’s long been a seasonal surge around this time of year, substantial enough for the Dogs Trust charity to coin its famous slogan back in 1978: “A dog is for life, not just for Christmas.”\"\"\"\n",
"\n",
"print(list_datasets())\n",
"\n",
"graph = await cognify(\"izmene\")\n",
"graph = await cognify()\n",
"\n",
"graph_url = await render_graph(graph, graph_type = \"networkx\")\n",
"print(graph_url)\n"
@ -304,36 +298,18 @@
},
{
"cell_type": "code",
"execution_count": 11,
"id": "2487d12b-570c-424c-8b6c-4c23f4073e05",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"hello\n"
]
}
],
"source": [
"print('hello')"
]
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 4,
"id": "a228fb2c-5bbc-48b4-af3d-4a26e840a79e",
"metadata": {},
"outputs": [],
"source": [
"from cognee import search\n",
"from cognee.api.v1.search.search import SearchType"
"from cognee.api.v1.search import SearchType"
]
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 8,
"id": "4ee34d29-f75d-4766-8906-b851b57bd5ef",
"metadata": {},
"outputs": [
@ -341,200 +317,496 @@
"name": "stdout",
"output_type": "stream",
"text": [
"b006ca75-d5aa-4006-9c9a-4d2b700670e3\n",
"Ministry responsible for construction, transport, and infrastructure regulations in Serbia\n",
"Regulation that amends the procedure for creating and controlling technical documentation for buildings based on their class and purpose\n",
"Publication of the Amendment Regulation in the Official Gazette of the Republic of Serbia, No. 77/2015, on September 9, 2015\n",
"Regulations for the main project in construction as per the Planning and Construction Law and its amendments\n",
"Minister of the Ministry of Construction, Transport, and Infrastructure\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Official Gazette of RS\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Publication date of the amendments and supplements rule in the Official Gazette of RS\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Zakon o planiranju i izgradnji\n",
"Službeni glasnik RS references for Zakon o planiranju i izgradnji: br. 72/09, 81/09 - ispravka, 64/10 - US, 24/11, 121/12, 42/13 - US, 50/13, 98/13 - US\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the Content, Method, and Procedure of Preparation and Method of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlović\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Official Gazette of the Republic of Serbia, number 77/2015\n",
"September 9, 2015\n",
"Law on Planning and Construction\n",
"Official Gazette references for Law on Planning and Construction\n",
"Ministarstvo građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"Zorana Mihajlović\n",
"Minister of Construction, Transport and Infrastructure\n",
"Rulebook on Amendments and Supplements\n",
"Rulebook on the content, method, and procedure for the preparation and control of technical documentation according to the class and purpose of objects\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlovic, prof. dr\n",
"14cff510-4ce9-44fe-a826-aea5349d0a38\n",
"Ministry responsible for construction, transport, and infrastructure regulations in Serbia\n",
"Regulation that amends the procedure for creating and controlling technical documentation for buildings based on their class and purpose\n",
"Publication of the Amendment Regulation in the Official Gazette of the Republic of Serbia, No. 77/2015, on September 9, 2015\n",
"Regulations for the main project in construction as per the Planning and Construction Law and its amendments\n",
"Minister of the Ministry of Construction, Transport, and Infrastructure\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Official Gazette of RS\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Publication date of the amendments and supplements rule in the Official Gazette of RS\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Zakon o planiranju i izgradnji\n",
"Službeni glasnik RS references for Zakon o planiranju i izgradnji: br. 72/09, 81/09 - ispravka, 64/10 - US, 24/11, 121/12, 42/13 - US, 50/13, 98/13 - US\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the Content, Method, and Procedure of Preparation and Method of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlović\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Official Gazette of the Republic of Serbia, number 77/2015\n",
"September 9, 2015\n",
"Law on Planning and Construction\n",
"Official Gazette references for Law on Planning and Construction\n",
"Ministarstvo građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"Zorana Mihajlović\n",
"Minister of Construction, Transport and Infrastructure\n",
"Rulebook on Amendments and Supplements\n",
"Rulebook on the content, method, and procedure for the preparation and control of technical documentation according to the class and purpose of objects\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlovic, prof. dr\n",
"56f0f95f-28c0-4cdf-8b53-3a0d271f2a9a\n",
"Ministry responsible for construction, transport, and infrastructure regulations in Serbia\n",
"Regulation that amends the procedure for creating and controlling technical documentation for buildings based on their class and purpose\n",
"Publication of the Amendment Regulation in the Official Gazette of the Republic of Serbia, No. 77/2015, on September 9, 2015\n",
"Regulations for the main project in construction as per the Planning and Construction Law and its amendments\n",
"Minister of the Ministry of Construction, Transport, and Infrastructure\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Official Gazette of RS\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Publication date of the amendments and supplements rule in the Official Gazette of RS\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Zakon o planiranju i izgradnji\n",
"Službeni glasnik RS references for Zakon o planiranju i izgradnji: br. 72/09, 81/09 - ispravka, 64/10 - US, 24/11, 121/12, 42/13 - US, 50/13, 98/13 - US\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the Content, Method, and Procedure of Preparation and Method of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlović\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Official Gazette of the Republic of Serbia, number 77/2015\n",
"September 9, 2015\n",
"Law on Planning and Construction\n",
"Official Gazette references for Law on Planning and Construction\n",
"Ministarstvo građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"Zorana Mihajlović\n",
"Minister of Construction, Transport and Infrastructure\n",
"Rulebook on Amendments and Supplements\n",
"Rulebook on the content, method, and procedure for the preparation and control of technical documentation according to the class and purpose of objects\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlovic, prof. dr\n",
"065288b5-94a9-4c8d-9ec7-339fe0b21226\n",
"Ministry responsible for construction, transport, and infrastructure regulations in Serbia\n",
"Regulation that amends the procedure for creating and controlling technical documentation for buildings based on their class and purpose\n",
"Publication of the Amendment Regulation in the Official Gazette of the Republic of Serbia, No. 77/2015, on September 9, 2015\n",
"Regulations for the main project in construction as per the Planning and Construction Law and its amendments\n",
"Minister of the Ministry of Construction, Transport, and Infrastructure\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Official Gazette of RS\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Publication date of the amendments and supplements rule in the Official Gazette of RS\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Zakon o planiranju i izgradnji\n",
"Službeni glasnik RS references for Zakon o planiranju i izgradnji: br. 72/09, 81/09 - ispravka, 64/10 - US, 24/11, 121/12, 42/13 - US, 50/13, 98/13 - US\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the Content, Method, and Procedure of Preparation and Method of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlović\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Official Gazette of the Republic of Serbia, number 77/2015\n",
"September 9, 2015\n",
"Law on Planning and Construction\n",
"Official Gazette references for Law on Planning and Construction\n",
"Ministarstvo građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"Zorana Mihajlović\n",
"Minister of Construction, Transport and Infrastructure\n",
"Rulebook on Amendments and Supplements\n",
"Rulebook on the content, method, and procedure for the preparation and control of technical documentation according to the class and purpose of objects\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlovic, prof. dr\n",
"272b9058-de0c-4b56-bff4-29aba1ca5251\n",
"Ministry responsible for construction, transport, and infrastructure regulations in Serbia\n",
"Regulation that amends the procedure for creating and controlling technical documentation for buildings based on their class and purpose\n",
"Publication of the Amendment Regulation in the Official Gazette of the Republic of Serbia, No. 77/2015, on September 9, 2015\n",
"Regulations for the main project in construction as per the Planning and Construction Law and its amendments\n",
"Minister of the Ministry of Construction, Transport, and Infrastructure\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Official Gazette of RS\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Publication date of the amendments and supplements rule in the Official Gazette of RS\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Zakon o planiranju i izgradnji\n",
"Službeni glasnik RS references for Zakon o planiranju i izgradnji: br. 72/09, 81/09 - ispravka, 64/10 - US, 24/11, 121/12, 42/13 - US, 50/13, 98/13 - US\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the Content, Method, and Procedure of Preparation and Method of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlović\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Official Gazette of the Republic of Serbia, number 77/2015\n",
"September 9, 2015\n",
"Law on Planning and Construction\n",
"Official Gazette references for Law on Planning and Construction\n",
"Ministarstvo građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"Zorana Mihajlović\n",
"Minister of Construction, Transport and Infrastructure\n",
"Rulebook on Amendments and Supplements\n",
"Rulebook on the content, method, and procedure for the preparation and control of technical documentation according to the class and purpose of objects\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Zorana Mihajlovic, prof. dr\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/var/folders/d_/1x0yyl7n5g5cc8vlgchdr06m0000gn/T/ipykernel_56167/1760472791.py:4: RuntimeWarning: coroutine 'search' was never awaited\n",
" out = await search(graph, query_params)\n",
"RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n"
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n",
"Ministry of Construction, Transport, and Infrastructure\n",
"Rulebook on Amendments and Supplements to the Rulebook on the Content, Manner, and Procedure of Preparation and the Method of Technical Documentation Control According to the Class and Purpose of Buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Construction Permit\n",
"Conceptual Project\n",
"Execution Project\n",
"Completed building project\n",
"Publication Date\n",
"Effective Date\n",
"Law Publication Dates\n",
"Prof. Dr. Zorana Mihajlovic\n",
"Ministry of Construction, Transport and Infrastructure\n",
"The amendment of the Rule on the Content, Mode, and Procedure of Preparation and Means of Control of Technical Documentation According to the Class and Purpose of Buildings\n",
"Publication in the Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Main project for construction permit, conceptual project, and execution project\n",
"Amendment and supplementation regulation to the technical documentation process and control according to the class and purpose of buildings\n",
"Official Gazette of the Republic of Serbia\n",
"Law on Planning and Construction\n",
"Minister of Construction, Transport, and Infrastructure\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Službeni glasnik RS, br. 77/2015\n",
"9.9.2015\n",
"prof. dr Zorana Mihajlović\n",
"4.9.2015\n",
"Ministry of Construction, Transport and Infrastructure\n",
"Regulation on Amendments and Supplements to the Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Regulation on the content, method, and procedure for preparing and controlling technical documentation according to class and purpose of buildings\n",
"Main Project\n",
"Law on Planning and Construction\n",
"Regulation on amendments and supplements to the Regulation on the content, method, and procedure for the preparation and manner of controlling technical documentation according to the class and purpose of objects\n",
"Published in Official Gazette of RS, No. 77/2015 on September 9, 2015\n",
"Law on Planning and Construction (\"Official Gazette of RS\", Nos. 72/09, 81/09 - correction, 64/10 - Constitutional Court, 24/11, 121/12, 42/13 - Constitutional Court, 50/13, 98/13 - Constitutional Court)\n",
"Minister, Prof. Dr. Zorana Mihajlovic\n",
"Ministar građevinarstva, saobraćaja i infrastrukture\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Publication in Službeni glasnik RS, No. 77/2015\n",
"Glavni projekat\n",
"Zakon o planiranju i izgradnji\n",
"građevinska dozvola\n",
"idejni projekat\n",
"projekat za izvođenje\n",
"projekat izvedenog objekta\n",
"upotrebna dozvola\n",
"Pravilnik o izmenama i dopunama Pravilnika o sadržini, načinu i postupku izrade i način vršenja kontrole tehničke dokumentacije prema klasi i nameni objekata\n",
"Zakon o planiranju i izgradnji\n",
"\"Službeni glasnik RS\"\n",
"prof. dr Zorana Mihajlovic\n"
]
}
],
@ -542,7 +814,9 @@
"query_params = {\n",
" SearchType.SIMILARITY: {'query': 'your search query here'}\n",
"}\n",
"out = await search(graph, query_params)"
"results = await search(graph, query_params)\n",
"for result in results[0]:\n",
" print(result)"
]
},
{
@ -625,46 +899,10 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": null,
"id": "f087510e-6df7-4a03-945d-70f8c91c8d65",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Deleting collection: 94a675a8-4ab4-4c85-bb06-26b8674b9ac0\n",
"Collection '94a675a8-4ab4-4c85-bb06-26b8674b9ac0' deleted successfully.\n",
"Deleting collection: 4299de57-f252-4524-b42f-1bcbf94af122\n",
"Collection '4299de57-f252-4524-b42f-1bcbf94af122' deleted successfully.\n",
"Deleting collection: 45ad96db-b3d3-4d38-9aae-c02dc63bba15\n",
"Collection '45ad96db-b3d3-4d38-9aae-c02dc63bba15' deleted successfully.\n",
"Deleting collection: e3702209-2c4b-47c2-834f-25b3f92c41a7\n",
"Collection 'e3702209-2c4b-47c2-834f-25b3f92c41a7' deleted successfully.\n",
"Deleting collection: 53f96999-912a-4873-bba9-aa545903f5fd\n",
"Collection '53f96999-912a-4873-bba9-aa545903f5fd' deleted successfully.\n",
"Deleting collection: c726b009-fecc-45ac-bf16-68ceec0c9f29\n",
"Collection 'c726b009-fecc-45ac-bf16-68ceec0c9f29' deleted successfully.\n",
"Deleting collection: c4f28a0a-95b2-415d-a127-c78a142a068d\n",
"Collection 'c4f28a0a-95b2-415d-a127-c78a142a068d' deleted successfully.\n",
"Deleting collection: 6a6d69d6-16b3-4c1a-935b-739d51051b5a\n",
"Collection '6a6d69d6-16b3-4c1a-935b-739d51051b5a' deleted successfully.\n",
"Deleting collection: f78c2fbd-a6bc-40d1-a771-2e17a47c6ce3\n",
"Collection 'f78c2fbd-a6bc-40d1-a771-2e17a47c6ce3' deleted successfully.\n",
"Deleting collection: fc869dd2-e60b-4783-b617-63377690d748\n",
"Collection 'fc869dd2-e60b-4783-b617-63377690d748' deleted successfully.\n",
"Deleting collection: 2649b0a7-23b9-4e7d-9a96-f3b4eee3d3fe\n",
"Collection '2649b0a7-23b9-4e7d-9a96-f3b4eee3d3fe' deleted successfully.\n",
"Deleting collection: 71a7da39-ad23-47c7-8f5c-32f625308d3b\n",
"Collection '71a7da39-ad23-47c7-8f5c-32f625308d3b' deleted successfully.\n",
"Deleting collection: 94a73201-001a-4296-ab4f-cd4c4d98c44a\n",
"Collection '94a73201-001a-4296-ab4f-cd4c4d98c44a' deleted successfully.\n",
"Deleting collection: cad2276f-bdc2-497a-a75f-067a162f2bab\n",
"Collection 'cad2276f-bdc2-497a-a75f-067a162f2bab' deleted successfully.\n",
"All collections have been deleted.\n"
]
}
],
"outputs": [],
"source": [
"from qdrant_client import models, QdrantClient\n",
"import os\n",

20
cognee/api/v1/cognify/cognify.py
View file

@ -1,6 +1,6 @@
import asyncio
# import logging
from typing import List
from typing import List, Union
from qdrant_client import models
import instructor
from openai import OpenAI
@ -35,11 +35,24 @@ aclient = instructor.patch(OpenAI())
USER_ID = "default_user"
async def cognify(dataset_name: str = "root"):
async def cognify(datasets: Union[str, List[str]] = None):
"""This function is responsible for the cognitive processing of the content."""
db = DuckDBAdapter()
files_metadata = db.get_files_metadata(dataset_name)
if datasets is None or len(datasets) == 0:
datasets = db.get_datasets()
awaitables = []
if isinstance(datasets, list):
for dataset in datasets:
awaitables.append(cognify(dataset))
graphs = await asyncio.gather(*awaitables)
return graphs[0]
files_metadata = db.get_files_metadata(datasets)
awaitables = []
@ -77,7 +90,6 @@ async def process_text(input_text: str, file_metadata: dict):
print(f"Document ({file_metadata['id']}) categorized: {file_metadata['categories']}")
cognitive_layers = await content_to_cog_layers(
"generate_cog_layers.txt",
classified_categories[0],
response_model = DefaultCognitiveLayer
)

1
cognee/api/v1/search/__init__.py Normal file
View file

@ -0,0 +1 @@
from .search import search, SearchType

0
cognee/classifiers/__init__.py
View file

65
cognee/classifiers/classify_documents.py
View file

@ -1,65 +0,0 @@
""" This module contains the classifiers for the documents. """
import json
import logging
from langchain.prompts import ChatPromptTemplate
from langchain.document_loaders import TextLoader
from langchain.document_loaders import DirectoryLoader
from langchain.chains import create_extraction_chain
from langchain.chat_models import ChatOpenAI
from ..config import Config
config = Config()
config.load()
OPENAI_API_KEY = config.openai_key
async def classify_documents(query: str, document_id: str, content: str):
"""Classify the documents based on the query and content."""
document_context = content
logging.info("This is the document context %s", document_context)
llm = ChatOpenAI(temperature=0, model=config.model)
prompt_classify = ChatPromptTemplate.from_template(
"""You are a summarizer and classifier.
Determine what book this is and where does it belong in the output :
{query}, Id: {d_id} Document context is: {context}"""
)
json_structure = [
{
"name": "summarizer",
"description": "Summarization and classification",
"parameters": {
"type": "object",
"properties": {
"DocumentCategory": {
"type": "string",
"description": "The classification of documents "
"in groups such as legal, medical, etc.",
},
"Title": {
"type": "string",
"description": "The title of the document",
},
"Summary": {
"type": "string",
"description": "The summary of the document",
},
"d_id": {"type": "string", "description": "The id of the document"},
},
"required": ["DocumentCategory", "Title", "Summary", "d_id"],
},
}
]
chain_filter = prompt_classify | llm.bind(
function_call={"name": "summarizer"}, functions=json_structure
)
classifier_output = await chain_filter.ainvoke(
{"query": query, "d_id": document_id, "context": str(document_context)}
)
arguments_str = classifier_output.additional_kwargs["function_call"]["arguments"]
logging.info("This is the arguments string %s", arguments_str)
arguments_dict = json.loads(arguments_str)
return arguments_dict

56
cognee/classifiers/classify_summary.py
View file

@ -1,56 +0,0 @@
""" This module contains the function to classify a summary of a document. """
import json
import logging
from langchain.prompts import ChatPromptTemplate
from langchain.chains import create_extraction_chain
from langchain.chat_models import ChatOpenAI
from ..config import Config
config = Config()
config.load()
OPENAI_API_KEY = config.openai_key
async def classify_summary(query, document_summaries):
"""Classify the documents based on the query and content."""
llm = ChatOpenAI(temperature=0, model=config.model)
prompt_classify = ChatPromptTemplate.from_template(
"""You are a classifier. Determine what document
are relevant for the given query: {query},
Document summaries and ids:{document_summaries}"""
)
json_structure = [
{
"name": "classifier",
"description": "Classification",
"parameters": {
"type": "object",
"properties": {
"DocumentSummary": {
"type": "string",
"description": "The summary of the document "
"and the topic it deals with.",
},
"d_id": {"type": "string", "description": "The id of the document"},
},
"required": ["DocumentSummary"],
},
}
]
chain_filter = prompt_classify | llm.bind(
function_call={"name": "classifier"}, functions=json_structure
)
classifier_output = await chain_filter.ainvoke(
{"query": query, "document_summaries": document_summaries}
)
arguments_str = classifier_output.additional_kwargs["function_call"]["arguments"]
logging.info("This is the arguments string %s", arguments_str)
arguments_dict = json.loads(arguments_str)
logging.info("Relevant summary is %s", arguments_dict.get("DocumentSummary", None))
classfier_id = arguments_dict.get("d_id", None)
logging.info("This is the classifier id %s", classfier_id)
return classfier_id

52
cognee/classifiers/classify_user_input.py
View file

@ -1,52 +0,0 @@
""" This module contains the classifiers for the documents. """
import json
import logging
from langchain.prompts import ChatPromptTemplate
from langchain.chains import create_extraction_chain
from langchain.chat_models import ChatOpenAI
from ..config import Config
config = Config()
config.load()
OPENAI_API_KEY = config.openai_key
async def classify_user_input(query, input_type):
""" Classify the user input based on the query and input type."""
llm = ChatOpenAI(temperature=0, model=config.model)
prompt_classify = ChatPromptTemplate.from_template(
"""You are a classifier.
Determine with a True or False if the following input: {query},
is relevant for the following memory category: {input_type}"""
)
json_structure = [
{
"name": "classifier",
"description": "Classification",
"parameters": {
"type": "object",
"properties": {
"InputClassification": {
"type": "boolean",
"description": "The classification of the input",
}
},
"required": ["InputClassification"],
},
}
]
chain_filter = prompt_classify | llm.bind(
function_call={"name": "classifier"}, functions=json_structure
)
classifier_output = await chain_filter.ainvoke(
{"query": query, "input_type": input_type}
)
arguments_str = classifier_output.additional_kwargs["function_call"]["arguments"]
logging.info("This is the arguments string %s", arguments_str)
arguments_dict = json.loads(arguments_str)
logging.info("Relevant summary is %s", arguments_dict.get("DocumentSummary", None))
InputClassification = arguments_dict.get("InputClassification", None)
logging.info("This is the classification %s", InputClassification)
return InputClassification

53
cognee/classifiers/classify_user_query.py
View file

@ -1,53 +0,0 @@
""" This module contains the function to classify the user query. """
import json
from langchain.prompts import ChatPromptTemplate
from langchain.chains import create_extraction_chain
from langchain.chat_models import ChatOpenAI
from langchain.document_loaders import TextLoader
from langchain.document_loaders import DirectoryLoader
from ..config import Config
config = Config()
config.load()
OPENAI_API_KEY = config.openai_key
async def classify_user_query(query, context, document_types):
"""Classify the user query based on the context and document types."""
llm = ChatOpenAI(temperature=0, model=config.model)
prompt_classify = ChatPromptTemplate.from_template(
"""You are a classifier.
You store user memories, thoughts and feelings.
Determine if you need to use them to answer this query : {query}"""
)
json_structure = [
{
"name": "classifier",
"description": "Classification",
"parameters": {
"type": "object",
"properties": {
"UserQueryClassifier": {
"type": "bool",
"description": "The classification of documents "
"in groups such as legal, medical, etc.",
}
},
"required": ["UserQueryClassifier"],
},
}
]
chain_filter = prompt_classify | llm.bind(
function_call={"name": "classifier"}, functions=json_structure
)
classifier_output = await chain_filter.ainvoke(
{"query": query, "context": context, "document_types": document_types}
)
arguments_str = classifier_output.additional_kwargs["function_call"]["arguments"]
print("This is the arguments string", arguments_str)
arguments_dict = json.loads(arguments_str)
classfier_value = arguments_dict.get("UserQueryClassifier", None)
print("This is the classifier value", classfier_value)
return classfier_value

0
cognee/database/__init__.py
View file

63
cognee/database/database_manager.py
View file

@ -1,63 +0,0 @@
"""This module provides functionalities for creating and managing databases."""
import os
from contextlib import asynccontextmanager
from sqlalchemy import text
from sqlalchemy.ext.asyncio import create_async_engine
from cognee.config import Config
from cognee.database.relationaldb.database import Base, get_sqlalchemy_database_url
globalConfig = Config()
class DatabaseManager:
"""Manages database creation, deletion, and table initialization."""
def __init__(self):
"""Initialize the Database Url with a given configuration."""
self.engine = create_async_engine(get_sqlalchemy_database_url(globalConfig.db_type), echo = True)
self.db_type = globalConfig.db_type
@asynccontextmanager
async def get_connection(self):
"""Initialize the DatabaseManager with a given configuration."""
if self.db_type in ["sqlite", "duckdb"]:
# For SQLite and DuckDB, the engine itself manages connections
yield self.engine
else:
async with self.engine.connect() as connection:
yield connection
async def database_exists(self, db_name):
"""Check if a database exists."""
if self.db_type in ["sqlite", "duckdb"]:
# For SQLite and DuckDB, check if the database file exists
return os.path.exists(db_name)
else:
query = text(f"SELECT 1 FROM pg_database WHERE datname='{db_name}'")
async with self.get_connection() as connection:
result = await connection.execute(query)
return result.fetchone() is not None
async def create_database(self, db_name):
"""Create a new database."""
if self.db_type not in ["sqlite", "duckdb"]:
# For databases like PostgreSQL, create the database explicitly
async with self.get_connection() as connection:
await connection.execute(text(f"CREATE DATABASE {db_name}"))
async def drop_database(self, db_name):
"""Drop an existing database."""
if self.db_type in ["sqlite", "duckdb"]:
# For SQLite and DuckDB, simply remove the database file
os.remove(db_name)
else:
async with self.get_connection() as connection:
await connection.execute(text(f"DROP DATABASE IF EXISTS {db_name}"))
async def create_tables(self):
"""Create tables based on the SQLAlchemy Base metadata."""
try:
async with self.engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as e:
print(e)
raise e

0
cognee/database/graphdb/__init__.py
View file

1002
cognee/database/graphdb/graph.py
View file

File diff suppressed because it is too large Load diff

128
cognee/database/graphdb/networkx/networkx_graph.py
View file

@ -1,128 +0,0 @@
import pickle
from pathlib import Path
from cognee.config import Config
import networkx as nx
config = Config()
config = config.load()
class NetworkXGraphDB:
"""A class to manage a graph database using NetworkX"""
# graph_path = (Path(config.db_path) / config.graph_name).absolute()
def __init__(self, filename="cognee_graph.pkl"):
self.filename = filename
try:
self.graph = self.load_graph() # Attempt to load an existing graph
except (FileNotFoundError, EOFError, pickle.UnpicklingError):
self.graph = nx.Graph() # Create a new graph if loading failed
def save_graph(self):
"""Save the graph to a file using pickle"""
with open(self.filename, "wb") as f:
pickle.dump(self.graph, f)
def load_graph(self):
"""Load the graph from a file using pickle"""
with open(self.filename, "rb") as f:
return pickle.load(f)
def create_base_cognitive_architecture(self, user_id: str):
# Add nodes for user and memory types if they don't exist
self.graph.add_node(user_id, type="User")
self.graph.add_node(f"{user_id}_semantic", type="SemanticMemory")
self.graph.add_node(f"{user_id}_episodic", type="EpisodicMemory")
self.graph.add_node(f"{user_id}_buffer", type="Buffer")
# Add edges to connect user to memory types
self.graph.add_edge(
user_id, f"{user_id}_semantic", relation="HAS_SEMANTIC_MEMORY"
)
self.graph.add_edge(
user_id, f"{user_id}_episodic", relation="HAS_EPISODIC_MEMORY"
)
self.graph.add_edge(user_id, f"{user_id}_buffer", relation="HAS_BUFFER")
self.save_graph() # Save the graph after modifying it
def delete_all_user_memories(self, user_id: str):
# Remove nodes and edges related to the user's memories
for memory_type in ["semantic", "episodic", "buffer"]:
memory_node = f"{user_id}_{memory_type}"
self.graph.remove_node(memory_node)
self.save_graph() # Save the graph after modifying it
def delete_specific_memory_type(self, user_id: str, memory_type: str):
# Remove a specific type of memory node and its related edges
memory_node = f"{user_id}_{memory_type.lower()}"
if memory_node in self.graph:
self.graph.remove_node(memory_node)
self.save_graph() # Save the graph after modifying it
def retrieve_semantic_memory(self, user_id: str):
return [n for n in self.graph.neighbors(f"{user_id}_semantic")]
def retrieve_episodic_memory(self, user_id: str):
return [n for n in self.graph.neighbors(f"{user_id}_episodic")]
def retrieve_buffer_memory(self, user_id: str):
return [n for n in self.graph.neighbors(f"{user_id}_buffer")]
def generate_graph_semantic_memory_document_summary(
self,
document_summary,
unique_graphdb_mapping_values,
document_namespace,
user_id,
):
for node, attributes in unique_graphdb_mapping_values.items():
self.graph.add_node(node, **attributes)
self.graph.add_edge(f"{user_id}_semantic", node, relation="HAS_KNOWLEDGE")
self.save_graph()
def generate_document_summary(
self,
document_summary,
unique_graphdb_mapping_values,
document_namespace,
user_id,
):
self.generate_graph_semantic_memory_document_summary(
document_summary, unique_graphdb_mapping_values, document_namespace, user_id
)
async def get_document_categories(self, user_id):
return [
self.graph.nodes[n]["category"]
for n in self.graph.neighbors(f"{user_id}_semantic")
if "category" in self.graph.nodes[n]
]
async def get_document_ids(self, user_id, category):
return [
n
for n in self.graph.neighbors(f"{user_id}_semantic")
if self.graph.nodes[n].get("category") == category
]
def create_document_node(self, document_summary, user_id):
d_id = document_summary["d_id"]
self.graph.add_node(d_id, **document_summary)
self.graph.add_edge(f"{user_id}_semantic", d_id, relation="HAS_DOCUMENT")
self.save_graph()
def update_document_node_with_namespace(
self, user_id, vectordb_namespace, document_id
):
if self.graph.has_node(document_id):
self.graph.nodes[document_id]["vectordbNamespace"] = vectordb_namespace
self.save_graph()
def get_namespaces_by_document_category(self, user_id, category):
return [
self.graph.nodes[n].get("vectordbNamespace")
for n in self.graph.neighbors(f"{user_id}_semantic")
if self.graph.nodes[n].get("category") == category
]

0
cognee/database/relationaldb/__init__.py
View file

69
cognee/database/relationaldb/database.py
View file

@ -1,69 +0,0 @@
"""Database configuration and connection."""
from pathlib import Path
# from contextlib import asynccontextmanager
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
from sqlalchemy.orm import declarative_base
from cognee.config import Config
globalConfig = Config()
# in seconds
MAX_RETRIES = 3
RETRY_DELAY = 5
def get_sqlalchemy_database_url(
db_type = globalConfig.db_type,
db_name = globalConfig.db_name,
db_path = globalConfig.db_path,
user = globalConfig.db_user,
password = globalConfig.db_password,
host = globalConfig.db_host,
port = globalConfig.db_port,
):
"""Get the SQLAlchemy database URL based on parameters."""
if db_type == "sqlite":
db_path = (Path(db_path) / db_name).absolute()
return f"sqlite+aiosqlite:///{db_path}" # SQLite uses file path
elif db_type == "duckdb":
db_path = (Path(db_path) / db_name).absolute()
return f"duckdb+aiosqlite:///{db_path}"
elif db_type == "postgresql":
# Ensure optional parameters are handled gracefully
port_str = f":{port}" if port else ""
password_str = f":{password}" if password else ""
if not all([user, host]):
raise ValueError("User and host are required for PostgreSQL connections.")
return f"postgresql+asyncpg://{user}{password_str}@{host}{port_str}/{db_name}"
else:
raise ValueError(f"Unsupported DB_TYPE: {db_type}")
SQLALCHEMY_DATABASE_URL = get_sqlalchemy_database_url()
engine = create_async_engine(
SQLALCHEMY_DATABASE_URL,
pool_recycle=3600,
echo=globalConfig.sqlalchemy_logging,
)
AsyncSessionLocal = async_sessionmaker(
bind=engine,
class_=AsyncSession,
expire_on_commit=False,
)
Base = declarative_base()
# @asynccontextmanager
# async def get_db():
# """Provide a database session to the context."""
# db = AsyncSessionLocal()
# try:
# yield db
# finally:
# await db.close()

71
cognee/database/relationaldb/database_crud.py
View file

@ -1,71 +0,0 @@
from contextlib import asynccontextmanager
import logging
from .models.sessions import Session
from .models.memory import MemoryModel
from .models.user import User
from .models.operation import Operation
from .models.metadatas import MetaDatas
from .models.docs import DocsModel
logger = logging.getLogger(__name__)
@asynccontextmanager
async def session_scope(session):
"""Provide a transactional scope around a series of operations."""
# session = AsyncSessionLocal()
try:
yield session
await session.commit()
except Exception as e:
await session.rollback()
logger.error(f"Session rollback due to: {str(e)}")
raise
finally:
await session.close()
async def add_entity(session, entity):
async with session_scope(session) as s: # Use your async session_scope
s.add(entity) # No need to commit; session_scope takes care of it
return "Successfully added entity"
def update_entity_graph_summary(session, model, entity_id, new_value):
with session_scope(session) as s:
# Retrieve the entity from the database
entity = s.query(model).filter_by(id=entity_id).first()
if entity:
entity.graph_summary = new_value
s.commit()
return "Successfully updated entity"
else:
return "Entity not found"
async def update_entity(session, model, entity_id, new_value):
async with session_scope(session) as s:
# Retrieve the entity from the database
entity = await s.get(model, entity_id)
if entity:
entity.operation_status = new_value
await s.commit()
else:
return "Entity not found"
async def fetch_job_id(session, user_id=None, memory_id=None, job_id=None):
try:
result = await session.execute(
session.query(Session.id)
.filter_by(user_id=user_id, id=job_id)
.order_by(Session.created_at)
.first()
)
return result.scalar_one_or_none()
except Exception as e:
logging.error(f"An error occurred: {str(e)}")
return None

0
cognee/database/relationaldb/models/__init__.py
View file

19
cognee/database/relationaldb/models/docs.py
View file

@ -1,19 +0,0 @@
""" This module contains the MemoryModel class, which is a SQLAlchemy model for the memory table in the relational database. """
from datetime import datetime
from sqlalchemy import Column, String, DateTime, ForeignKey, Boolean
from sqlalchemy.orm import relationship
from ..database import Base
class DocsModel(Base):
""" Docs model"""
__tablename__ = "docs"
id = Column(String, primary_key=True)
operation_id = Column(String, ForeignKey("operations.id"), index=True)
doc_name = Column(String, nullable=True)
graph_summary = Column(Boolean, nullable=True)
memory_category = Column(String, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, onupdate=datetime.utcnow)
operations = relationship("Operation", back_populates="docs")

29
cognee/database/relationaldb/models/memory.py
View file

@ -1,29 +0,0 @@
""" This module contains the MemoryModel class, which is a SQLAlchemy model for the memory table in the relational database. """
from datetime import datetime
from sqlalchemy import Column, String, DateTime, ForeignKey, UUID
from sqlalchemy.orm import relationship
from ..database import Base
class MemoryModel(Base):
""" Memory model"""
__tablename__ = "memories"
id = Column(UUID, primary_key=True)
user_id = Column(String, ForeignKey("users.id"), index=True)
operation_id = Column(String, ForeignKey("operations.id"), index=True)
memory_name = Column(String, nullable=True)
memory_category = Column(String, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, onupdate=datetime.utcnow)
methods_list = Column(String, nullable=True)
attributes_list = Column(String, nullable=True)
user = relationship("User", back_populates="memories")
operation = relationship("Operation", back_populates="memories")
metadatas = relationship(
"MetaDatas", back_populates="memory", cascade="all, delete-orphan"
)
def __repr__(self):
return f"<Memory(id={self.id}, user_id={self.user_id}, created_at={self.created_at}, updated_at={self.updated_at})>"

27
cognee/database/relationaldb/models/metadatas.py
View file

@ -1,27 +0,0 @@
# metadata.py
""" MetaData model """
from datetime import datetime
from sqlalchemy import Column, String, DateTime, ForeignKey
from sqlalchemy.orm import relationship
import os
import sys
from ..database import Base
class MetaDatas(Base):
""" MetaData model"""
__tablename__ = "metadatas"
id = Column(String, primary_key=True)
user_id = Column(String, ForeignKey("users.id"), index=True)
version = Column(String, nullable=False)
contract_metadata = Column(String, nullable=False)
memory_id = Column(String, ForeignKey("memories.id"), index=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, onupdate=datetime.utcnow)
user = relationship("User", back_populates="metadatas")
memory = relationship("MemoryModel", back_populates="metadatas")
def __repr__(self):
return f"<MetaData(id={self.id}, version={self.version}, field={self.field}, memory_id={self.memory_id}, created_at={self.created_at}, updated_at={self.updated_at})>"

28
cognee/database/relationaldb/models/operation.py
View file

@ -1,28 +0,0 @@
# operation.py
""" Operation model """
from datetime import datetime
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey
from sqlalchemy.orm import relationship
import os
import sys
from ..database import Base
class Operation(Base):
""" Operation model"""
__tablename__ = "operations"
id = Column(String, primary_key=True)
user_id = Column(String, ForeignKey("users.id"), index=True) # Link to User
operation_type = Column(String, nullable=True)
operation_status = Column(String, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, onupdate=datetime.utcnow)
memories = relationship("MemoryModel", back_populates="operation")
# Relationships
user = relationship("User", back_populates="operations")
docs = relationship("DocsModel", back_populates="operations")
def __repr__(self):
return f"<Operation(id={self.id}, user_id={self.user_id}, created_at={self.created_at}, updated_at={self.updated_at})>"

27
cognee/database/relationaldb/models/sessions.py
View file

@ -1,27 +0,0 @@
# session.py
""" Session model """
from datetime import datetime
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey
from sqlalchemy.orm import relationship
import os
import sys
from ..database import Base
class Session(Base):
""" Session model"""
__tablename__ = "sessions"
id = Column(String, primary_key=True)
user_id = Column(String, ForeignKey("users.id"), index=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, onupdate=datetime.utcnow)
# Corrected relationship name
user = relationship("User", back_populates="sessions")
# operations = relationship("Operation", back_populates="session", cascade="all, delete-orphan")
def __repr__(self):
return f"<Session(id={self.id}, user_id={self.user_id}, created_at={self.created_at}, updated_at={self.updated_at})>"

39
cognee/database/relationaldb/models/user.py
View file

@ -1,39 +0,0 @@
# user.py
""" User model """
from datetime import datetime
from sqlalchemy import Column, String, DateTime
from sqlalchemy.orm import relationship
import os
import sys
from .memory import MemoryModel
from .operation import Operation
from .sessions import Session
from .metadatas import MetaDatas
from .docs import DocsModel
from ..database import Base
class User(Base):
""" User model"""
__tablename__ = "users"
id = Column(String, primary_key=True, index=True)
session_id = Column(String, nullable=True, unique=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, onupdate=datetime.utcnow)
# Relationships
memories = relationship(
"MemoryModel", back_populates="user", cascade="all, delete-orphan"
)
operations = relationship(
"Operation", back_populates="user", cascade="all, delete-orphan"
)
sessions = relationship(
"Session", back_populates="user", cascade="all, delete-orphan"
)
metadatas = relationship("MetaDatas", back_populates="user")
def __repr__(self):
return f"<User(id={self.id}, created_at={self.created_at}, updated_at={self.updated_at})>"

0
cognee/database/vectordb/__init__.py
View file

230
cognee/database/vectordb/basevectordb.py
View file

@ -1,230 +0,0 @@
import logging
from io import BytesIO
import os, sys
# Add the parent directory to sys.path
# sys.path.append(os.path.dirname(os.path.abspath(__file__)))
import sqlalchemy as sa
print(os.getcwd())
logging.basicConfig(level=logging.INFO)
# import marvin
import requests
from langchain.document_loaders import PyPDFLoader
from langchain.retrievers import WeaviateHybridSearchRetriever
from weaviate.gql.get import HybridFusion
from cognee.database.relationaldb.models.sessions import Session
from cognee.database.relationaldb.models.metadatas import MetaDatas
from cognee.database.relationaldb.models.operation import Operation
from cognee.database.relationaldb.models.docs import DocsModel
from sqlalchemy.orm import sessionmaker
from cognee.database.relationaldb.database import engine
from typing import Optional
import time
import tracemalloc
tracemalloc.start()
from datetime import datetime
from langchain.embeddings.openai import OpenAIEmbeddings
from cognee.database.vectordb.vectordb import (
PineconeVectorDB,
WeaviateVectorDB,
LanceDB,
)
from langchain.schema import Document
import uuid
import weaviate
from marshmallow import Schema, fields
import json
from cognee.database.vectordb.vector_db_type import VectorDBType
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
# marvin.settings.openai.api_key = os.environ.get("OPENAI_API_KEY")
class VectorDBFactory:
def __init__(self):
self.db_map = {
VectorDBType.PINECONE.value: PineconeVectorDB,
VectorDBType.WEAVIATE.value: WeaviateVectorDB,
VectorDBType.LANCEDB.value: LanceDB,
# Add more database types and their corresponding classes here
}
def create_vector_db(
self,
user_id: str,
index_name: str,
memory_id: str,
db_type: str,
namespace: str = None,
embeddings=None,
):
logging.info(f"db_type: {db_type}")
logging.info(f"embeddings: {self.db_map}")
if db_type in self.db_map:
return self.db_map[db_type](
user_id, index_name, memory_id, namespace, embeddings
)
raise ValueError(f"Unsupported database type: {db_type}")
class BaseMemory:
def __init__(
self,
user_id: str,
memory_id: Optional[str],
index_name: Optional[str],
db_type: str,
namespace: str,
embeddings: Optional[None],
):
self.user_id = user_id
self.memory_id = memory_id
self.index_name = index_name
self.namespace = namespace
self.embeddings = embeddings
self.db_type = db_type
factory = VectorDBFactory()
self.vector_db = factory.create_vector_db(
self.user_id,
self.index_name,
self.memory_id,
db_type=self.db_type,
namespace=self.namespace,
embeddings=self.embeddings,
)
def init_client(self, embeddings, namespace: str):
return self.vector_db.init_client(embeddings, namespace)
def create_field(self, field_type, **kwargs):
field_mapping = {
"Str": fields.Str,
"Int": fields.Int,
"Float": fields.Float,
"Bool": fields.Bool,
}
return field_mapping[field_type](**kwargs)
def create_dynamic_schema(self, params):
"""Create a dynamic schema based on provided parameters."""
dynamic_fields = {field_name: fields.Str() for field_name in params.keys()}
# Create a Schema instance with the dynamic fields
dynamic_schema_instance = Schema.from_dict(dynamic_fields)()
return dynamic_schema_instance
async def get_version_from_db(self, user_id, memory_id):
# Logic to retrieve the version from the database.
Session = sessionmaker(bind=engine)
session = Session()
try:
# Querying both fields: contract_metadata and created_at
result = (
session.query(MetaDatas.contract_metadata, MetaDatas.created_at)
.filter_by(user_id=user_id) # using parameter, not self.user_id
.order_by(MetaDatas.created_at.desc())
.first()
)
if result:
version_in_db, created_at = result
logging.info(f"version_in_db: {version_in_db}")
from ast import literal_eval
version_in_db = literal_eval(version_in_db)
version_in_db = version_in_db.get("version")
return [version_in_db, created_at]
else:
return None
finally:
session.close()
async def update_metadata(self, user_id, memory_id, version_in_params, params):
version_from_db = await self.get_version_from_db(user_id, memory_id)
Session = sessionmaker(bind=engine)
session = Session()
# If there is no metadata, insert it.
if version_from_db is None:
session.add(
MetaDatas(
id=str(uuid.uuid4()),
user_id=self.user_id,
version=str(int(time.time())),
memory_id=self.memory_id,
contract_metadata=params,
)
)
session.commit()
return params
# If params version is higher, update the metadata.
elif version_in_params > version_from_db[0]:
session.add(
MetaDatas(
id=str(uuid.uuid4()),
user_id=self.user_id,
memory_id=self.memory_id,
contract_metadata=params,
)
)
session.commit()
return params
else:
return params
async def add_memories(
self,
observation: Optional[str] = None,
loader_settings: dict = None,
params: Optional[dict] = None,
namespace: Optional[str] = None,
custom_fields: Optional[str] = None,
embeddings: Optional[str] = None,
):
return await self.vector_db.add_memories(
observation=observation,
loader_settings=loader_settings,
params=params,
namespace=namespace,
metadata_schema_class=None,
embeddings=embeddings,
)
# Add other db_type conditions if necessary
async def fetch_memories(
self,
observation: str,
search_type: Optional[str] = None,
params: Optional[str] = None,
namespace: Optional[str] = None,
n_of_observations: Optional[int] = 2,
):
logging.info(namespace)
logging.info(params)
logging.info(observation)
return await self.vector_db.fetch_memories(
observation=observation,
search_type=search_type,
params=params,
namespace=namespace,
n_of_observations=n_of_observations,
)
async def delete_memories(self, namespace: str, params: Optional[str] = None):
return await self.vector_db.delete_memories(namespace, params)
async def count_memories(self, namespace: str, params: Optional[str] = None):
return await self.vector_db.count_memories(namespace, params)

0
cognee/database/vectordb/chunkers/__init__.py
View file

10
cognee/database/vectordb/chunkers/chunk_strategy.py
View file

@ -1,10 +0,0 @@
from enum import Enum
class ChunkStrategy(Enum):
"""Chunking strategies for the vector database."""
EXACT = "exact"
PARAGRAPH = "paragraph"
SENTENCE = "sentence"
VANILLA = "vanilla"
SUMMARY = "summary"

93
cognee/database/vectordb/chunkers/chunkers.py
View file

@ -1,93 +0,0 @@
"""Module for chunking text data based on various strategies."""
import re
import logging
from cognee.database.vectordb.chunkers.chunk_strategy import ChunkStrategy
from langchain.text_splitter import RecursiveCharacterTextSplitter
def chunk_data(chunk_strategy=None, source_data=None, chunk_size=None, chunk_overlap=None):
"""Chunk the given source data into smaller parts based on the specified strategy."""
if chunk_strategy == ChunkStrategy.VANILLA:
chunked_data = vanilla_chunker(source_data, chunk_size, chunk_overlap)
elif chunk_strategy == ChunkStrategy.PARAGRAPH:
chunked_data = chunk_data_by_paragraph(source_data, chunk_size, chunk_overlap)
elif chunk_strategy == ChunkStrategy.SENTENCE:
chunked_data = chunk_by_sentence(source_data, chunk_size, chunk_overlap)
elif chunk_strategy == ChunkStrategy.EXACT:
chunked_data = chunk_data_exact(source_data, chunk_size, chunk_overlap)
elif chunk_strategy == ChunkStrategy.SUMMARY:
chunked_data = summary_chunker(source_data, chunk_size, chunk_overlap)
else:
chunked_data = vanilla_chunker(source_data, chunk_size, chunk_overlap)
return chunked_data
def vanilla_chunker(source_data, chunk_size=100, chunk_overlap=20):
"""Chunk the given source data into smaller parts using a vanilla strategy."""
text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size
, chunk_overlap=chunk_overlap
, length_function=len)
pages = text_splitter.create_documents([source_data])
return pages
def summary_chunker(source_data, chunk_size=400, chunk_overlap=20):
"""Chunk the given source data into smaller parts, focusing on summarizing content."""
text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size
, chunk_overlap=chunk_overlap
, length_function=len)
try:
pages = text_splitter.create_documents([source_data])
except Exception as e:
pages = text_splitter.create_documents(source_data.content)
logging.error(f"An error occurred: %s {str(e)}")
if len(pages) > 10:
return pages[:5] + pages[-5:]
return pages
def chunk_data_exact(data_chunks, chunk_size, chunk_overlap):
"""Chunk the data into exact sizes as specified, without considering content."""
data = "".join(data_chunks)
chunks = [data[i:i + chunk_size] for i in range(0, len(data), chunk_size - chunk_overlap)]
return chunks
def chunk_by_sentence(data_chunks, chunk_size, overlap):
"""Chunk the data by sentences, ensuring each chunk does not exceed the specified size."""
data = "".join(data_chunks)
sentence_endings = r"(?<=[.!?…]) +"
sentences = re.split(sentence_endings, data)
sentence_chunks = []
for sentence in sentences:
if len(sentence) > chunk_size:
chunks = chunk_data_exact([sentence], chunk_size, overlap)
sentence_chunks.extend(chunks)
else:
sentence_chunks.append(sentence)
return sentence_chunks
def chunk_data_by_paragraph(data_chunks, chunk_size, overlap, bound=0.75):
"""Chunk the data by paragraphs, with consideration for chunk size and overlap."""
data = "".join(data_chunks)
total_length = len(data)
chunks = []
check_bound = int(bound * chunk_size)
start_idx = 0
while start_idx < total_length:
end_idx = min(start_idx + chunk_size, total_length)
next_paragraph_index = data.find("\n\n", start_idx + check_bound, end_idx)
if next_paragraph_index != -1:
end_idx = next_paragraph_index + 2
chunks.append(data[start_idx:end_idx + overlap])
start_idx = end_idx
return chunks

161
cognee/database/vectordb/cognee_manager.py
View file

@ -1,161 +0,0 @@
import os
import requests
import json
from .embeddings import Embeddings
from .vector_db import VectorDB
from .response import Response
class CogneeManager:
def __init__(
self,
embeddings: Embeddings = None,
vector_db: VectorDB = None,
vector_db_key: str = None,
embedding_api_key: str = None,
webhook_url: str = None,
lines_per_batch: int = 1000,
webhook_key: str = None,
document_id: str = None,
chunk_validation_url: str = None,
internal_api_key: str = "test123",
base_url="http://localhost:8000",
):
self.embeddings = embeddings if embeddings else Embeddings()
self.vector_db = vector_db if vector_db else VectorDB()
self.webhook_url = webhook_url
self.lines_per_batch = lines_per_batch
self.webhook_key = webhook_key
self.document_id = document_id
self.chunk_validation_url = chunk_validation_url
self.vector_db_key = vector_db_key
self.embeddings_api_key = embedding_api_key
self.internal_api_key = internal_api_key
self.base_url = base_url
def serialize(self):
data = {
"EmbeddingsMetadata": json.dumps(self.embeddings.serialize()),
"VectorDBMetadata": json.dumps(self.vector_db.serialize()),
"WebhookURL": self.webhook_url,
"LinesPerBatch": self.lines_per_batch,
"DocumentID": self.document_id,
"ChunkValidationURL": self.chunk_validation_url,
}
return {k: v for k, v in data.items() if v is not None}
def upload(self, file_paths: list[str], base_url=None):
if base_url:
url = base_url + "/jobs"
else:
url = self.base_url + "/jobs"
data = self.serialize()
headers = self.generate_headers()
multipart_form_data = [
(
"file",
(
os.path.basename(filepath),
open(filepath, "rb"),
"application/octet-stream",
),
)
for filepath in file_paths
]
print(f"embedding {len(file_paths)} documents at {url}")
response = requests.post(
url, files=multipart_form_data, headers=headers, stream=True, data=data
)
if response.status_code == 500:
print(response.text)
return Response(error=response.text, status_code=response.status_code)
response_json = response.json()
if response.status_code >= 400 and response.status_code < 500:
print(f"Error: {response_json['error']}")
return Response.from_json(response_json, response.status_code)
def get_job_statuses(self, job_ids: list[int], base_url=None):
if base_url:
url = base_url + "/jobs/status"
else:
url = self.base_url + "/jobs/status"
headers = {
"Authorization": self.internal_api_key,
}
data = {"JobIDs": job_ids}
print(f"retrieving job statuses for {len(job_ids)} jobs at {url}")
response = requests.post(url, headers=headers, json=data)
if response.status_code == 500:
print(response.text)
return Response(error=response.text, status_code=response.status_code)
response_json = response.json()
if response.status_code >= 400 and response.status_code < 500:
print(f"Error: {response_json['error']}")
return Response.from_json(response_json, response.status_code)
def embed(self, filepath, base_url=None):
if base_url:
url = base_url + "/embed"
else:
url = self.base_url + "/embed"
data = self.serialize()
headers = self.generate_headers()
files = {"SourceData": open(filepath, "rb")}
print(f"embedding document at file path {filepath} at {url}")
response = requests.post(url, headers=headers, data=data, files=files)
if response.status_code == 500:
print(response.text)
return Response(error=response.text, status_code=response.status_code)
response_json = response.json()
if response.status_code >= 400 and response.status_code < 500:
print(f"Error: {response_json['error']}")
return Response.from_json(response_json, response.status_code)
def get_job_status(self, job_id, base_url=None):
if base_url:
url = base_url + "/jobs/" + str(job_id) + "/status"
else:
url = self.base_url + "/jobs/" + str(job_id) + "/status"
headers = {
"Authorization": self.internal_api_key,
}
print(f"retrieving job status for job {job_id} at {url}")
response = requests.get(url, headers=headers)
if response.status_code == 500:
print(response.text)
return Response(error=response.text, status_code=response.status_code)
response_json = response.json()
if response.status_code >= 400 and response.status_code < 500:
print(f"Error: {response_json['error']}")
return Response.from_json(response_json, response.status_code)
def generate_headers(self):
headers = {
"Authorization": self.internal_api_key,
"X-EmbeddingAPI-Key": self.embeddings_api_key,
"X-VectorDB-Key": self.vector_db_key,
"X-Webhook-Key": self.webhook_key,
}
return {k: v for k, v in headers.items() if v is not None}

34
cognee/database/vectordb/embeddings/embeddings.py
View file

@ -1,34 +0,0 @@
from .embeddings_type import EmbeddingsType
from ..chunkers.chunk_strategy import ChunkStrategy
class Embeddings:
def __init__(
self,
embeddings_type: EmbeddingsType = EmbeddingsType.OPEN_AI,
chunk_size: int = 256,
chunk_overlap: int = 128,
chunk_strategy: ChunkStrategy = ChunkStrategy.EXACT,
docker_image: str = None,
hugging_face_model_name: str = None,
):
self.embeddings_type = embeddings_type
self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap
self.chunk_strategy = chunk_strategy
self.docker_image = docker_image
self.hugging_face_model_name = hugging_face_model_name
def serialize(self):
data = {
"embeddings_type": self.embeddings_type.name
if self.embeddings_type
else None,
"chunk_size": self.chunk_size,
"chunk_overlap": self.chunk_overlap,
"chunk_strategy": self.chunk_strategy.name if self.chunk_strategy else None,
"docker_image": self.docker_image,
"hugging_face_model_name": self.hugging_face_model_name,
}
return {k: v for k, v in data.items() if v is not None}

9
cognee/database/vectordb/embeddings/embeddings_type.py
View file

@ -1,9 +0,0 @@
from enum import Enum
class EmbeddingsType(Enum):
OPEN_AI = "open_ai"
COHERE = "cohere"
SELF_HOSTED = "self_hosted"
HUGGING_FACE = "hugging_face"
IMAGE = "image"

18
cognee/database/vectordb/job.py
View file

@ -1,18 +0,0 @@
class Job:
def __init__(self, job_id, job_status=None, filename=None):
self.job_id = job_id
self.job_status = job_status
self.filename = filename
def __str__(self):
attributes = []
if self.job_id is not None:
attributes.append(f"job_id: {self.job_id}")
if self.job_status is not None:
attributes.append(f"job_status: {self.job_status}")
if self.filename is not None:
attributes.append(f"filename: {self.filename}")
return "Job(" + ", ".join(attributes) + ")"
def __repr__(self):
return self.__str__()

0
cognee/database/vectordb/loaders/__init__.py
View file

226
cognee/database/vectordb/loaders/loaders.py
View file

@ -1,226 +0,0 @@
from io import BytesIO
import fitz
import os
import sys
from cognee.database.vectordb.chunkers.chunkers import chunk_data
from cognee.shared.language_processing import (
translate_text,
detect_language,
)
from langchain.document_loaders import UnstructuredURLLoader
from langchain.document_loaders import DirectoryLoader
import logging
import os
from langchain.document_loaders import TextLoader
import requests
async def fetch_pdf_content(file_url):
response = requests.get(file_url)
pdf_stream = BytesIO(response.content)
with fitz.open(stream=pdf_stream, filetype="pdf") as doc:
return "".join(page.get_text() for page in doc)
async def fetch_text_content(file_url):
loader = UnstructuredURLLoader(urls=file_url)
return loader.load()
async def process_content(
content, metadata, loader_strategy, chunk_size, chunk_overlap
):
pages = chunk_data(
chunk_strategy=loader_strategy,
source_data=content,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
)
if metadata is None:
metadata = {"metadata": "None"}
chunk_count = 0
for chunk in pages:
chunk_count += 1
chunk.metadata = metadata
chunk.metadata["chunk_count"] = chunk_count
if detect_language(pages) != "en":
logging.info("Translating Page")
for page in pages:
if detect_language(page.page_content) != "en":
page.page_content = translate_text(page.page_content)
return pages
async def _document_loader(observation: str, loader_settings: dict):
document_format = loader_settings.get("format", "text")
loader_strategy = loader_settings.get("strategy", "VANILLA")
chunk_size = loader_settings.get("chunk_size", 500)
chunk_overlap = loader_settings.get("chunk_overlap", 20)
logging.info("LOADER SETTINGS %s", loader_settings)
list_of_docs = loader_settings["path"]
chunked_doc = []
if loader_settings.get("source") == "URL":
for file in list_of_docs:
if document_format == "PDF":
content = await fetch_pdf_content(file)
elif document_format == "TEXT":
content = await fetch_text_content(file)
else:
raise ValueError(f"Unsupported document format: {document_format}")
pages = await process_content(
content,
metadata=None,
loader_strategy=loader_strategy,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
)
chunked_doc.append(pages)
elif loader_settings.get("source") == "DEVICE":
if loader_settings.get("bulk_load", False) == True:
current_directory = os.getcwd()
logging.info("Current Directory: %s", current_directory)
loader = DirectoryLoader(".data", recursive=True)
documents = loader.load()
for document in documents:
# print ("Document: ", document.page_content)
pages = await process_content(
content=str(document.page_content),
metadata=document.metadata,
loader_strategy=loader_strategy,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
)
chunked_doc.append(pages)
else:
from langchain.document_loaders import PyPDFLoader
loader = PyPDFLoader(loader_settings.get("single_document_path"))
documents = loader.load()
for document in documents:
pages = await process_content(
content=str(document.page_content),
metadata=document.metadata,
loader_strategy=loader_strategy,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
)
chunked_doc.append(pages)
else:
raise ValueError(f"Unsupported source type: {loader_settings.get('source')}")
return chunked_doc
# async def _document_loader( observation: str, loader_settings: dict):
#
# document_format = loader_settings.get("format", "text")
# loader_strategy = loader_settings.get("strategy", "VANILLA")
# chunk_size = loader_settings.get("chunk_size", 500)
# chunk_overlap = loader_settings.get("chunk_overlap", 20)
#
#
# logging.info("LOADER SETTINGS %s", loader_settings)
#
# list_of_docs = loader_settings["path"]
# chunked_doc = []
#
# if loader_settings.get("source") == "URL":
# for file in list_of_docs:
# if document_format == "PDF":
# logging.info("File is %s", file)
# pdf_response = requests.get(file)
# pdf_stream = BytesIO(pdf_response.content)
# with fitz.open(stream=pdf_stream, filetype='pdf') as doc:
# file_content = ""
# for page in doc:
# file_content += page.get_text()
# pages = chunk_data(chunk_strategy=loader_strategy, source_data=file_content, chunk_size=chunk_size,
# chunk_overlap=chunk_overlap)
# from cognee.shared.language_processing import translate_text,detect_language
#
# if detect_language(pages) != "en":
# logging.info("Current Directory 3")
# for page in pages:
# if detect_language(page.page_content) != "en":
# logging.info("Translating Page")
# page.page_content = translate_text(page.page_content)
#
# chunked_doc.append(pages)
#
# logging.info("Document translation complete. Proceeding...")
#
# chunked_doc.append(pages)
#
# elif document_format == "TEXT":
# loader = UnstructuredURLLoader(urls=file)
# file_content = loader.load()
# pages = chunk_data(chunk_strategy=loader_strategy, source_data=file_content, chunk_size=chunk_size,
# chunk_overlap=chunk_overlap)
#
# from cognee.shared.language_processing import translate_text, detect_language
#
# if detect_language(pages) != "en":
# logging.info("Current Directory 3")
# for page in pages:
# if detect_language(page.page_content) != "en":
# logging.info("Translating Page")
# page.page_content = translate_text(page.page_content)
#
# chunked_doc.append(pages)
#
# logging.info("Document translation complete. Proceeding...")
#
# chunked_doc.append(pages)
#
# elif loader_settings.get("source") == "DEVICE":
#
# current_directory = os.getcwd()
# logging.info("Current Directory: %s", current_directory)
#
# loader = DirectoryLoader(".data", recursive=True)
# if document_format == "PDF":
# # loader = SimpleDirectoryReader(".data", recursive=True, exclude_hidden=True)
# documents = loader.load()
# pages = chunk_data(chunk_strategy=loader_strategy, source_data=str(documents), chunk_size=chunk_size,
# chunk_overlap=chunk_overlap)
# logging.info("Documents: %s", documents)
# from cognee.shared.language_processing import translate_text, detect_language
#
# if detect_language(pages) != "en":
# logging.info("Current Directory 3")
# for page in pages:
# if detect_language(page.page_content) != "en":
# logging.info("Translating Page")
# page.page_content = translate_text(page.page_content)
#
# chunked_doc.append(pages)
#
# logging.info("Document translation complete. Proceeding...")
#
# # pages = documents.load_and_split()
# chunked_doc.append(pages)
#
#
# elif document_format == "TEXT":
# documents = loader.load()
# pages = chunk_data(chunk_strategy=loader_strategy, source_data=str(documents), chunk_size=chunk_size,
# chunk_overlap=chunk_overlap)
# logging.info("Documents: %s", documents)
# # pages = documents.load_and_split()
# chunked_doc.append(pages)
#
# else:
# raise ValueError(f"Error: ")
# return chunked_doc

86
cognee/database/vectordb/response.py
View file

@ -1,86 +0,0 @@
from .job import Job
class Response:
def __init__(
self,
error=None,
message=None,
successful_uploads=None,
failed_uploads=None,
empty_files_count=None,
duplicate_files_count=None,
job_id=None,
jobs=None,
job_status=None,
status_code=None,
):
self.error = error
self.message = message
self.successful_uploads = successful_uploads
self.failed_uploads = failed_uploads
self.empty_files_count = empty_files_count
self.duplicate_files_count = duplicate_files_count
self.job_id = job_id
self.jobs = jobs
self.job_status = job_status
self.status_code = status_code
@classmethod
def from_json(cls, json_dict, status_code):
successful_uploads = cls._convert_successful_uploads_to_jobs(
json_dict.get("successful_uploads", None)
)
jobs = cls._convert_to_jobs(json_dict.get("Jobs", None))
return cls(
error=json_dict.get("error"),
message=json_dict.get("message"),
successful_uploads=successful_uploads,
failed_uploads=json_dict.get("failed_uploads"),
empty_files_count=json_dict.get("empty_files_count"),
duplicate_files_count=json_dict.get("duplicate_files_count"),
job_id=json_dict.get("JobID"),
jobs=jobs,
job_status=json_dict.get("JobStatus"),
status_code=status_code,
)
@classmethod
def _convert_successful_uploads_to_jobs(cls, successful_uploads):
if not successful_uploads:
return None
return [
Job(filename=key, job_id=val) for key, val in successful_uploads.items()
]
@classmethod
def _convert_to_jobs(cls, jobs):
if not jobs:
return None
return [Job(job_id=job["JobID"], job_status=job["JobStatus"]) for job in jobs]
def __str__(self):
attributes = []
if self.error is not None:
attributes.append(f"error: {self.error}")
if self.message is not None:
attributes.append(f"message: {self.message}")
if self.successful_uploads is not None:
attributes.append(f"successful_uploads: {str(self.successful_uploads)}")
if self.failed_uploads is not None:
attributes.append(f"failed_uploads: {self.failed_uploads}")
if self.empty_files_count is not None:
attributes.append(f"empty_files_count: {self.empty_files_count}")
if self.duplicate_files_count is not None:
attributes.append(f"duplicate_files_count: {self.duplicate_files_count}")
if self.job_id is not None:
attributes.append(f"job_id: {self.job_id}")
if self.jobs is not None:
attributes.append(f"jobs: {str(self.jobs)}")
if self.job_status is not None:
attributes.append(f"job_status: {self.job_status}")
if self.status_code is not None:
attributes.append(f"status_code: {self.status_code}")
return "Response(" + ", ".join(attributes) + ")"

15
cognee/database/vectordb/vector_db_type.py
View file

@ -1,15 +0,0 @@
from enum import Enum
class VectorDBType(Enum):
PINECONE = "pinecone"
WEAVIATE = "weaviate"
MILVUS = "milvus"
QDRANT = "qdrant"
DEEPLAKE = "deeplake"
VESPA = "vespa"
PGVECTOR = "pgvector"
REDIS = "redis"
LANCEDB = "lancedb"
MONGODB = "mongodb"
FAISS = "faiss"

515
cognee/database/vectordb/vectordb.py
View file

@ -1,515 +0,0 @@
# Make sure to install the following packages: dlt, langchain, duckdb, python-dotenv, openai, weaviate-client
import logging
from langchain.text_splitter import RecursiveCharacterTextSplitter
from marshmallow import Schema, fields
from cognee.database.vectordb.loaders.loaders import _document_loader
# Add the parent directory to sys.path
logging.basicConfig(level=logging.INFO)
from langchain.retrievers import WeaviateHybridSearchRetriever, ParentDocumentRetriever
from weaviate.gql.get import HybridFusion
import tracemalloc
tracemalloc.start()
import os
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.schema import Document
import weaviate
LTM_MEMORY_ID_DEFAULT = "00000"
ST_MEMORY_ID_DEFAULT = "0000"
BUFFER_ID_DEFAULT = "0000"
class VectorDB:
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
def __init__(
self,
user_id: str,
index_name: str,
memory_id: str,
namespace: str = None,
embeddings=None,
):
self.user_id = user_id
self.index_name = index_name
self.namespace = namespace
self.memory_id = memory_id
self.embeddings = embeddings
class PineconeVectorDB(VectorDB):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.init_pinecone(self.index_name)
def init_pinecone(self, index_name):
# Pinecone initialization logic
pass
import langchain.embeddings
class WeaviateVectorDB(VectorDB):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.init_weaviate(embeddings=self.embeddings, namespace=self.namespace)
def init_weaviate(
self,
embeddings=OpenAIEmbeddings(openai_api_key=os.getenv("OPENAI_API_KEY", "")),
namespace=None,
retriever_type="",
):
# Weaviate initialization logic
auth_config = weaviate.auth.AuthApiKey(
api_key=os.environ.get("WEAVIATE_API_KEY")
)
client = weaviate.Client(
url=os.environ.get("WEAVIATE_URL"),
auth_client_secret=auth_config,
additional_headers={"X-OpenAI-Api-Key": os.environ.get("OPENAI_API_KEY")},
)
if retriever_type == "single_document_context":
retriever = WeaviateHybridSearchRetriever(
client=client,
index_name=namespace,
text_key="text",
attributes=[],
embedding=embeddings,
create_schema_if_missing=True,
)
return retriever
elif retriever_type == "multi_document_context":
retriever = WeaviateHybridSearchRetriever(
client=client,
index_name=namespace,
text_key="text",
attributes=[],
embedding=embeddings,
create_schema_if_missing=True,
)
return retriever
else:
return client
# child_splitter = RecursiveCharacterTextSplitter(chunk_size=400)
# store = InMemoryStore()
# retriever = ParentDocumentRetriever(
# vectorstore=vectorstore,
# docstore=store,
# child_splitter=child_splitter,
# )
from marshmallow import Schema, fields
def create_document_structure(observation, params, metadata_schema_class=None):
"""
Create and validate a document structure with optional custom fields.
:param observation: Content of the document.
:param params: Metadata information.
:param metadata_schema_class: Custom metadata schema class (optional).
:return: A list containing the validated document data.
"""
document_data = {"metadata": params, "page_content": observation}
def get_document_schema():
class DynamicDocumentSchema(Schema):
metadata = fields.Nested(metadata_schema_class, required=True)
page_content = fields.Str(required=True)
return DynamicDocumentSchema
# Validate and deserialize, defaulting to "1.0" if not provided
CurrentDocumentSchema = get_document_schema()
loaded_document = CurrentDocumentSchema().load(document_data)
return [loaded_document]
def _stuct(self, observation, params, metadata_schema_class=None):
"""Utility function to create the document structure with optional custom fields."""
# Construct document data
document_data = {"metadata": params, "page_content": observation}
def get_document_schema():
class DynamicDocumentSchema(Schema):
metadata = fields.Nested(metadata_schema_class, required=True)
page_content = fields.Str(required=True)
return DynamicDocumentSchema
# Validate and deserialize # Default to "1.0" if not provided
CurrentDocumentSchema = get_document_schema()
loaded_document = CurrentDocumentSchema().load(document_data)
return [loaded_document]
async def add_memories(
self,
observation,
loader_settings=None,
params=None,
namespace=None,
metadata_schema_class=None,
embeddings="hybrid",
):
# Update Weaviate memories here
if namespace is None:
namespace = self.namespace
params["user_id"] = self.user_id
logging.info("User id is %s", self.user_id)
retriever = self.init_weaviate(
embeddings=OpenAIEmbeddings(),
namespace=namespace,
retriever_type="single_document_context",
)
if loader_settings:
# Assuming _document_loader returns a list of documents
documents = await _document_loader(observation, loader_settings)
logging.info("here are the docs %s", str(documents))
chunk_count = 0
for doc_list in documents:
for doc in doc_list:
chunk_count += 1
params["chunk_count"] = doc.metadata.get("chunk_count", "None")
logging.info(
"Loading document with provided loader settings %s", str(doc)
)
params["source"] = doc.metadata.get("source", "None")
logging.info("Params are %s", str(params))
retriever.add_documents(
[Document(metadata=params, page_content=doc.page_content)]
)
else:
chunk_count = 0
from cognee.database.vectordb.chunkers.chunkers import (
chunk_data,
)
documents = [
chunk_data(
chunk_strategy="VANILLA",
source_data=observation,
chunk_size=300,
chunk_overlap=20,
)
]
for doc in documents[0]:
chunk_count += 1
params["chunk_order"] = chunk_count
params["source"] = "User loaded"
logging.info(
"Loading document with default loader settings %s", str(doc)
)
logging.info("Params are %s", str(params))
retriever.add_documents(
[Document(metadata=params, page_content=doc.page_content)]
)
async def fetch_memories(
self,
observation: str,
namespace: str = None,
search_type: str = "hybrid",
params=None,
**kwargs,
):
"""
Fetch documents from weaviate.
Parameters:
- observation (str): User query.
- namespace (str, optional): Type of memory accessed.
- search_type (str, optional): Type of search ('text', 'hybrid', 'bm25', 'generate', 'generate_grouped'). Defaults to 'hybrid'.
- **kwargs: Additional parameters for flexibility.
Returns:
List of documents matching the query or an empty list in case of error.
Example:
fetch_memories(query="some query", search_type='text', additional_param='value')
"""
client = self.init_weaviate(namespace=self.namespace)
if search_type is None:
search_type = "hybrid"
if not namespace:
namespace = self.namespace
logging.info("Query on namespace %s", namespace)
params_user_id = {
"path": ["user_id"],
"operator": "Like",
"valueText": self.user_id,
}
def list_objects_of_class(class_name, schema):
return [
prop["name"]
for class_obj in schema["classes"]
if class_obj["class"] == class_name
for prop in class_obj["properties"]
]
base_query = (
client.query.get(
namespace, list(list_objects_of_class(namespace, client.schema.get()))
)
.with_additional(
["id", "creationTimeUnix", "lastUpdateTimeUnix", "score", "distance"]
)
.with_where(params_user_id)
.with_limit(10)
)
n_of_observations = kwargs.get("n_of_observations", 2)
# try:
if search_type == "text":
query_output = (
base_query.with_near_text({"concepts": [observation]})
.with_autocut(n_of_observations)
.do()
)
elif search_type == "hybrid":
query_output = (
base_query.with_hybrid(
query=observation, fusion_type=HybridFusion.RELATIVE_SCORE
)
.with_autocut(n_of_observations)
.do()
)
elif search_type == "bm25":
query_output = (
base_query.with_bm25(query=observation)
.with_autocut(n_of_observations)
.do()
)
elif search_type == "summary":
filter_object = {
"operator": "And",
"operands": [
{
"path": ["user_id"],
"operator": "Equal",
"valueText": self.user_id,
},
{
"path": ["chunk_order"],
"operator": "LessThan",
"valueNumber": 30,
},
],
}
base_query = (
client.query.get(
namespace,
list(list_objects_of_class(namespace, client.schema.get())),
)
.with_additional(
[
"id",
"creationTimeUnix",
"lastUpdateTimeUnix",
"score",
"distance",
]
)
.with_where(filter_object)
.with_limit(30)
)
query_output = (
base_query
# .with_hybrid(query=observation, fusion_type=HybridFusion.RELATIVE_SCORE)
.do()
)
elif search_type == "summary_filter_by_object_name":
filter_object = {
"operator": "And",
"operands": [
{
"path": ["user_id"],
"operator": "Equal",
"valueText": self.user_id,
},
{
"path": ["doc_id"],
"operator": "Equal",
"valueText": params,
},
],
}
base_query = (
client.query.get(
namespace,
list(list_objects_of_class(namespace, client.schema.get())),
)
.with_additional(
[
"id",
"creationTimeUnix",
"lastUpdateTimeUnix",
"score",
"distance",
]
)
.with_where(filter_object)
.with_limit(30)
.with_hybrid(query=observation, fusion_type=HybridFusion.RELATIVE_SCORE)
)
query_output = base_query.do()
return query_output
elif search_type == "generate":
generate_prompt = kwargs.get("generate_prompt", "")
query_output = (
base_query.with_generate(single_prompt=observation)
.with_near_text({"concepts": [observation]})
.with_autocut(n_of_observations)
.do()
)
elif search_type == "generate_grouped":
generate_prompt = kwargs.get("generate_prompt", "")
query_output = (
base_query.with_generate(grouped_task=observation)
.with_near_text({"concepts": [observation]})
.with_autocut(n_of_observations)
.do()
)
else:
logging.error(f"Invalid search_type: {search_type}")
return []
# except Exception as e:
# logging.error(f"Error executing query: {str(e)}")
# return []
return query_output
async def delete_memories(self, namespace: str, params: dict = None):
if namespace is None:
namespace = self.namespace
client = self.init_weaviate(namespace=self.namespace)
if params:
where_filter = {
"path": ["id"],
"operator": "Equal",
"valueText": params.get("id", None),
}
return client.batch.delete_objects(
class_name=self.namespace,
# Same `where` filter as in the GraphQL API
where=where_filter,
)
else:
# Delete all objects
return client.batch.delete_objects(
class_name=namespace,
where={
"path": ["version"],
"operator": "Equal",
"valueText": "1.0",
},
)
async def count_memories(self, namespace: str = None, params: dict = None) -> int:
"""
Count memories in a Weaviate database.
Args:
namespace (str, optional): The Weaviate namespace to count memories in. If not provided, uses the default namespace.
Returns:
int: The number of memories in the specified namespace.
"""
if namespace is None:
namespace = self.namespace
client = self.init_weaviate(namespace=namespace)
try:
object_count = client.query.aggregate(namespace).with_meta_count().do()
return object_count
except Exception as e:
logging.info(f"Error counting memories: {str(e)}")
# Handle the error or log it
return 0
def update_memories(self, observation, namespace: str, params: dict = None):
client = self.init_weaviate(namespace=self.namespace)
client.data_object.update(
data_object={
# "text": observation,
"user_id": str(self.user_id),
"version": params.get("version", None) or "",
"agreement_id": params.get("agreement_id", None) or "",
"privacy_policy": params.get("privacy_policy", None) or "",
"terms_of_service": params.get("terms_of_service", None) or "",
"format": params.get("format", None) or "",
"schema_version": params.get("schema_version", None) or "",
"checksum": params.get("checksum", None) or "",
"owner": params.get("owner", None) or "",
"license": params.get("license", None) or "",
"validity_start": params.get("validity_start", None) or "",
"validity_end": params.get("validity_end", None) or ""
# **source_metadata,
},
class_name="Test",
uuid=params.get("id", None),
consistency_level=weaviate.data.replication.ConsistencyLevel.ALL, # default QUORUM
)
return
import os
import lancedb
from pydantic import BaseModel
from typing import List, Optional
import pandas as pd
import pyarrow as pa
class LanceDB(VectorDB):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.db = self.init_lancedb()
def init_lancedb(self):
# Initialize LanceDB connection
# Adjust the URI as needed for your LanceDB setup
uri = "s3://my-bucket/lancedb" if self.namespace else "~/.lancedb"
db = lancedb.connect(uri, api_key=os.getenv("LANCEDB_API_KEY"))
return db
def create_table(
self,
name: str,
schema: Optional[pa.Schema] = None,
data: Optional[pd.DataFrame] = None,
):
# Create a table in LanceDB. If schema is not provided, it will be inferred from the data.
if data is not None and schema is None:
schema = pa.Schema.from_pandas(data)
table = self.db.create_table(name, schema=schema)
if data is not None:
table.add(data.to_dict("records"))
return table
def add_memories(self, table_name: str, data: pd.DataFrame):
# Add data to an existing table in LanceDB
table = self.db.open_table(table_name)
table.add(data.to_dict("records"))
def fetch_memories(
self, table_name: str, query_vector: List[float], top_k: int = 10
):
# Perform a vector search in the specified table
table = self.db.open_table(table_name)
results = table.search(query_vector).limit(top_k).to_pandas()
return results

2
cognee/infrastructure/llm/llm_interface.py
View file

@ -35,6 +35,6 @@ class LLMInterface(Protocol):
raise NotImplementedError
@abstractmethod
def show_prompt(self, text_input: str, system_prompt_path: str) -> str:
def show_prompt(self, text_input: str, system_prompt: str) -> str:
"""To get structured output, import/call this function"""
raise NotImplementedError

18
cognee/infrastructure/llm/openai/adapter.py
View file

@ -6,8 +6,8 @@ import instructor
from openai import AsyncOpenAI
from pydantic import BaseModel
from tenacity import retry, stop_after_attempt
from cognee.utils import read_query_prompt
from ..llm_interface import LLMInterface
from cognee.infrastructure.llm.llm_interface import LLMInterface
from cognee.infrastructure.llm.prompts import read_query_prompt
class OpenAIAdapter(LLMInterface):
"""Adapter for OpenAI's GPT-3, GPT=4 API"""
@ -31,13 +31,13 @@ class OpenAIAdapter(LLMInterface):
async def acreate_embedding_with_backoff(self, input: List[str], model: str = "text-embedding-3-large"):
"""Wrapper around Embedding.acreate w/ backoff"""
return await self.aclient.embeddings.create(input=input, model=model)
return await self.aclient.embeddings.create(input = input, model = model)
async def async_get_embedding_with_backoff(self, text, model="text-embedding-3-large"):
"""To get text embeddings, import/call this function
It specifies defaults + handles rate-limiting + is async"""
text = text.replace("\n", " ")
response = await self.aclient.embeddings.create(input =text, model= model)
response = await self.aclient.embeddings.create(input = text, model = model)
embedding = response.data[0].embedding
return embedding
@ -85,13 +85,13 @@ class OpenAIAdapter(LLMInterface):
response_model = response_model,
)
def show_prompt(self, text_input: str, system_prompt_path: str) -> str:
def show_prompt(self, text_input: str, system_prompt: str) -> str:
"""Format and display the prompt for a user query."""
if not text_input:
text_input= "No user input provided."
if not system_prompt_path:
text_input = "No user input provided."
if not system_prompt:
raise ValueError("No system prompt path provided.")
system_prompt = read_query_prompt(system_prompt_path)
system_prompt = read_query_prompt(system_prompt)
formatted_prompt = f"""System Prompt:\n{system_prompt}\n\nUser Input:\n{text_input}\n"""
formatted_prompt = f"""System Prompt:\n{system_prompt}\n\nUser Input:\n{text_input}\n""" if system_prompt else None
return formatted_prompt

33
cognee/infrastructure/llm/openai/queries.py
View file

@ -1,12 +1,10 @@
""" This module contains the functions that are used to query the language model. """
import os
import logging
import instructor
from openai import OpenAI
import logging
from cognee.shared.data_models import KnowledgeGraph, MemorySummary
from cognee.config import Config
from cognee.infrastructure.llm.prompts import read_query_prompt
from cognee.shared.data_models import KnowledgeGraph, MemorySummary
config = Config()
config.load()
@ -15,29 +13,11 @@ OPENAI_API_KEY = config.openai_key
aclient = instructor.patch(OpenAI())
# Function to read query prompts from files
def read_query_prompt(filename):
"""Read a query prompt from a file."""
try:
with open(filename, "r") as file:
return file.read()
except FileNotFoundError:
logging.info(f"Error: File not found. Attempted to read: %s {filename}")
logging.info(f"Current working directory: %s {os.getcwd()}")
return None
except Exception as e:
logging.info(f"An error occurred: %s {e}")
return None
def generate_graph(input) -> KnowledgeGraph:
"""Generate a knowledge graph from a user query."""
model = "gpt-4-1106-preview"
user_prompt = f"Use the given format to extract information from the following input: {input}."
system_prompt = read_query_prompt(
"cognee/llm/prompts/generate_graph_prompt.txt"
)
system_prompt = read_query_prompt("generate_graph_prompt.txt")
out = aclient.chat.completions.create(
model=model,
@ -86,9 +66,8 @@ async def generate_summary(input) -> MemorySummary:
def user_query_to_edges_and_nodes(input: str) -> KnowledgeGraph:
"""Generate a knowledge graph from a user query."""
system_prompt = read_query_prompt(
"cognee/llm/prompts/generate_graph_prompt.txt"
)
system_prompt = read_query_prompt("generate_graph_prompt.txt")
return aclient.chat.completions.create(
model=config.model,
messages=[

2
cognee/infrastructure/llm/prompts/__init__.py Normal file
View file

@ -0,0 +1,2 @@
from .render_prompt import render_prompt
from .read_query_prompt import read_query_prompt

17
cognee/infrastructure/llm/prompts/read_query_prompt.py Normal file
View file

@ -0,0 +1,17 @@
from os import path
import logging
from cognee.root_dir import get_absolute_path
def read_query_prompt(prompt_file_name: str):
"""Read a query prompt from a file."""
try:
file_path = path.join(get_absolute_path("./infrastructure/llm/prompts"), prompt_file_name)
with open(file_path, "r", encoding = "utf-8") as file:
return file.read()
except FileNotFoundError:
logging.error(f"Error: Prompt file not found. Attempted to read: %s {file_path}")
return None
except Exception as e:
logging.error(f"An error occurred: %s {e}")
return None

25
cognee/infrastructure/llm/prompts/render_prompt.py Normal file
View file

@ -0,0 +1,25 @@
from jinja2 import Environment, FileSystemLoader, select_autoescape
from cognee.root_dir import get_absolute_path
def render_prompt(filename: str, context: dict) -> str:
"""Render a Jinja2 template asynchronously.
:param filename: The name of the template file to render.
:param context: The context to render the template with.
:return: The rendered template as a string."""
# Set the base directory relative to the cognee root directory
base_directory = get_absolute_path("./infrastructure/llm/prompts")
# Initialize the Jinja2 environment to load templates from the filesystem
env = Environment(
loader = FileSystemLoader(base_directory),
autoescape = select_autoescape(["html", "xml", "txt"])
)
# Load the template by name
template = env.get_template(filename)
# Render the template with the provided context
rendered_template = template.render(context)
return rendered_template

6
cognee/modules/cognify/llm/classify_content.py
View file

@ -1,13 +1,13 @@
""" This module contains the code to classify content into categories using the LLM API. """
from typing import Type, List
from pydantic import BaseModel
from cognee.infrastructure.llm.prompts import read_query_prompt
from cognee.infrastructure.llm.get_llm_client import get_llm_client
from cognee.utils import read_query_prompt
async def classify_into_categories(text_input: str, system_prompt_path: str, response_model: Type[BaseModel]):
async def classify_into_categories(text_input: str, system_prompt_file: str, response_model: Type[BaseModel]):
llm_client = get_llm_client()
system_prompt = await read_query_prompt(system_prompt_path)
system_prompt = read_query_prompt(system_prompt_file)
llm_output = await llm_client.acreate_structured_output(text_input, system_prompt, response_model)

6
cognee/modules/cognify/llm/content_to_cog_layers.py
View file

@ -1,12 +1,12 @@
""" This module contains the code to classify content into categories using the LLM API. """
from typing import Type
from pydantic import BaseModel
from cognee.infrastructure.llm.prompts import render_prompt
from cognee.infrastructure.llm.get_llm_client import get_llm_client
from cognee.utils import async_render_template
async def content_to_cog_layers(filename: str, context, response_model: Type[BaseModel]):
async def content_to_cog_layers(context, response_model: Type[BaseModel]):
llm_client = get_llm_client()
formatted_text_input = await async_render_template(filename, context)
formatted_text_input = render_prompt("generate_cog_layers.txt", context)
return await llm_client.acreate_structured_output(formatted_text_input, formatted_text_input, response_model)

4
cognee/modules/cognify/llm/generate_graph.py
View file

@ -4,12 +4,12 @@ from typing import Type
from pydantic import BaseModel
from cognee.infrastructure.llm.get_llm_client import get_llm_client
from cognee.shared.data_models import KnowledgeGraph
from cognee.utils import async_render_template
from cognee.infrastructure.llm.prompts import render_prompt
async def generate_graph(text_input: str, filename: str, context, response_model: Type[BaseModel]):
llm_client = get_llm_client()
formatted_text_input = await async_render_template(filename, context)
formatted_text_input = render_prompt(filename, context)
output = await llm_client.acreate_structured_output(text_input, formatted_text_input, response_model)
context_key = json.dumps(context, sort_keys = True)

1
cognee/modules/search/graph/search_neighbour.py
View file

@ -8,7 +8,6 @@ async def search_neighbour(CONNECTED_GRAPH, id):
relevant_layer = attr['layer_uuid']
if attr_.get('layer_uuid') == relevant_layer:
print(attr_['description'])
relevant_context.append(attr_['description'])
return relevant_context

9
cognee/modules/search/vector/search_similarity.py
View file

@ -1,21 +1,18 @@
from cognee.infrastructure.llm.get_llm_client import get_llm_client
from cognee.modules.cognify.graph.add_node_connections import extract_node_descriptions
from cognee.infrastructure.databases.vector.get_vector_database import get_vector_database
async def search_similarity(query:str ,graph):
node_descriptions = await extract_node_descriptions(graph.nodes(data = True))
# print(node_descriptions)
unique_layer_uuids = set(node["layer_decomposition_uuid"] for node in node_descriptions)
client = get_llm_client()
out = []
query = await client.async_get_embedding_with_backoff(query)
# print(query)
for id in unique_layer_uuids:
from cognee.infrastructure.databases.vector.get_vector_database import get_vector_database
vector_client = get_vector_database()
result = await vector_client.search(id, query,10)
@ -29,14 +26,12 @@ async def search_similarity(query:str ,graph):
relevant_context = []
for proposition_id in out[0][0]:
print(proposition_id)
for n,attr in graph.nodes(data=True):
if proposition_id in n:
for n_, attr_ in graph.nodes(data=True):
relevant_layer = attr['layer_uuid']
if attr_.get('layer_uuid') == relevant_layer:
print(attr_['description'])
relevant_context.append(attr_['description'])
return relevant_context
return relevant_context

8
cognee/presets.py
View file

@ -1,8 +0,0 @@
DEFAULT_PRESET = "cognitive_architecture_chat"
preset_options = [DEFAULT_PRESET]
def use_preset():
"""Placeholder for different present options"""
pass

27
cognee/setup_database.py
View file

@ -1,27 +0,0 @@
"""This module is used to create the database and tables for the cognitive architecture."""
import logging
logger = logging.getLogger(__name__)
async def main():
"""Runs as a part of startup docker scripts to create the database and tables."""
from cognee.config import Config
config = Config()
config.load()
from cognee.database.database_manager import DatabaseManager
db_manager = DatabaseManager()
database_name = config.db_name
if not await db_manager.database_exists(database_name):
print(f"Database {database_name} does not exist. Creating...")
await db_manager.create_database(database_name)
print(f"Database {database_name} created successfully.")
await db_manager.create_tables()
if __name__ == "__main__":
import asyncio
asyncio.run(main())

347
cognee/utils.py
View file

@ -1,51 +1,20 @@
""" This module contains utility functions for the cognitive architecture. """
""" This module contains utility functions for the cognee. """
import os
import uuid
import random
import string
import logging
import graphistry
from pathlib import Path
from jinja2 import Environment, FileSystemLoader, select_autoescape
from sqlalchemy import or_
from sqlalchemy.future import select
from sqlalchemy.orm import contains_eager
from sqlalchemy.ext.asyncio import AsyncSession
from cognee.database.relationaldb.models.docs import DocsModel
from cognee.database.relationaldb.models.memory import MemoryModel
from cognee.database.relationaldb.models.operation import Operation
from cognee.config import Config
config = Config()
config.load()
class Node:
def __init__(self, id, description, color):
self.id = id
self.description = description
self.color = color
class Edge:
def __init__(self, source, target, label, color):
self.source = source
self.target = target
self.label = label
self.color = color
from cognee.root_dir import get_absolute_path
def get_document_names(doc_input):
"""
Get a list of document names.
This function takes doc_input, which can be a folder path, a single document file path, or a document name as a string.
This function takes doc_input, which can be a folder path,
a single document file path, or a document name as a string.
It returns a list of document names based on the doc_input.
Args:
doc_input (str): The doc_input can be a folder path, a single document file path, or a document name as a string.
doc_input (str): The doc_input can be a folder path, a single document file path,
or a document name as a string.
Returns:
list: A list of document names.
@ -78,7 +47,7 @@ def get_document_names(doc_input):
def format_dict(d):
""" Format a dictionary as a string."""
"""Format a dictionary as a string."""
# Initialize an empty list to store formatted items
formatted_items = []
@ -99,286 +68,50 @@ def format_dict(d):
return formatted_string
def append_uuid_to_variable_names(variable_mapping):
""" Append a UUID to the variable names to make them unique."""
unique_variable_mapping = {}
for original_name in variable_mapping.values():
unique_name = f"{original_name}_{uuid.uuid4().hex}"
unique_variable_mapping[original_name] = unique_name
return unique_variable_mapping
# Update the functions to use the unique variable names
def create_node_variable_mapping(nodes):
""" Create a mapping of node identifiers to unique variable names."""
mapping = {}
for node in nodes:
variable_name = f"{node['category']}{node['id']}".lower()
mapping[node["id"]] = variable_name
return mapping
def create_edge_variable_mapping(edges):
""" Create a mapping of edge identifiers to unique variable names."""
mapping = {}
for edge in edges:
# Construct a unique identifier for the edge
variable_name = f"edge{edge['source']}to{edge['target']}".lower()
mapping[(edge["source"], edge["target"])] = variable_name
return mapping
def generate_letter_uuid(length=8):
"""Generate a random string of uppercase letters with the specified length."""
letters = string.ascii_uppercase # A-Z
return "".join(random.choice(letters) for _ in range(length))
async def get_vectordb_namespace(session: AsyncSession, user_id: str):
""" Asynchronously retrieves the latest memory names for a given user."""
try:
result = await session.execute(
select(MemoryModel.memory_name)
.where(MemoryModel.user_id == user_id)
.order_by(MemoryModel.created_at.desc())
)
namespace = [row[0] for row in result.fetchall()]
return namespace
except Exception as e:
logging.error(
f"An error occurred while retrieving the Vectordb_namespace: {str(e)}"
)
return None
async def get_vectordb_document_name(session: AsyncSession, user_id: str):
""" Asynchronously retrieves the latest memory names for a given user."""
try:
result = await session.execute(
select(DocsModel.doc_name)
.where(DocsModel.user_id == user_id)
.order_by(DocsModel.created_at.desc())
)
doc_names = [row[0] for row in result.fetchall()]
return doc_names
except Exception as e:
logging.error(
f"An error occurred while retrieving the Vectordb_namespace: {str(e)}"
)
return None
async def get_model_id_name(session: AsyncSession, id: str):
""" Asynchronously retrieves the latest memory names for a given user."""
try:
result = await session.execute(
select(MemoryModel.memory_name)
.where(MemoryModel.id == id)
.order_by(MemoryModel.created_at.desc())
)
doc_names = [row[0] for row in result.fetchall()]
return doc_names
except Exception as e:
logging.error(
f"An error occurred while retrieving the Vectordb_namespace: {str(e)}"
)
return None
async def get_unsumarized_vector_db_namespace(session: AsyncSession, user_id: str):
"""
Asynchronously retrieves the latest memory names and document details for a given user.
This function executes a database query to fetch memory names and document details
associated with operations performed by a specific user. It leverages explicit joins
with the 'docs' and 'memories' tables and applies eager loading to optimize performance.
Parameters:
- session (AsyncSession): The database session for executing the query.
- user_id (str): The unique identifier of the user.
Returns:
- Tuple[List[str], List[Tuple[str, str]]]: A tuple containing a list of memory names and
a list of tuples with document names and their corresponding IDs.
Returns None if an exception occurs.
Raises:
- Exception: Propagates any exceptions that occur during query execution.
Example Usage:
"""
# try:
result = await session.execute(
select(Operation)
.join(Operation.docs) # Explicit join with docs table
.join(Operation.memories) # Explicit join with memories table
.options(
contains_eager(Operation.docs), # Informs ORM of the join for docs
contains_eager(Operation.memories), # Informs ORM of the join for memories
)
.where(
(Operation.user_id == user_id)
& or_( # Filter by user_id
DocsModel.graph_summary == False, # Condition 1: graph_summary is False
DocsModel.graph_summary == None, # Condition 3: graph_summary is None
) # Filter by user_id
)
.order_by(Operation.created_at.desc()) # Order by creation date
)
operations = result.unique().scalars().all()
# Extract memory names and document names and IDs
# memory_names = [memory.memory_name for op in operations for memory in op.memories]
memory_details = [
(memory.memory_name, memory.memory_category)
for op in operations
for memory in op.memories
]
docs = [(doc.doc_name, doc.id) for op in operations for doc in op.docs]
return memory_details, docs
async def get_memory_name_by_doc_id(session: AsyncSession, docs_id: str):
"""
Asynchronously retrieves memory names associated with a specific document ID.
This function executes a database query to fetch memory names linked to a document
through operations. The query is filtered based on a given document ID and retrieves
only the memory names without loading the entire Operation entity.
Parameters:
- session (AsyncSession): The database session for executing the query.
- docs_id (str): The unique identifier of the document.
Returns:
- List[str]: A list of memory names associated with the given document ID.
Returns None if an exception occurs.
Raises:
- Exception: Propagates any exceptions that occur during query execution.
"""
try:
result = await session.execute(
select(MemoryModel.memory_name)
.join(
Operation, Operation.id == MemoryModel.operation_id
) # Join with Operation
.join(
DocsModel, DocsModel.operation_id == Operation.id
) # Join with DocsModel
.where(DocsModel.id == docs_id) # Filtering based on the passed document ID
.distinct() # To avoid duplicate memory names
)
memory_names = [row[0] for row in result.fetchall()]
return memory_names
except Exception as e:
# Handle the exception as needed
print(f"An error occurred: {e}")
return None
async def read_query_prompt(filename: str) -> str:
"""Read a query prompt from a file.
:param filename: The name of the file to read.
:return: The content of the file as a string.
"""
script_directory = Path(__file__).parent
# Set the base directory relative to the script's directory
base_directory = script_directory.parent / "cognee/infrastructure/llm/prompts"
# Construct the full file path
file_path = base_directory / filename
try:
return file_path.read_text()
except FileNotFoundError:
logging.error(f"File not found: {file_path.absolute()}")
except Exception as e:
logging.error(f"An error of type {type(e).__name__} occurred while reading file: {file_path.absolute()}. Error message: {e}")
return None
async def print_file_content(file_path):
# Create a Path object for the file path
path = Path(file_path)
# Check if the file exists
if path.is_file():
# Open and read the file, then print its content
with path.open('r') as file:
print(file.read())
else:
# Print an error message if the file does not exist
print(f"The file '{file_path}' does not exist.")
async def async_render_template(filename: str, context: dict) -> str:
"""Render a Jinja2 template asynchronously.
:param filename: The name of the template file to render.
:param context: The context to render the template with.
:return: The rendered template as a string."""
# Initialize the Jinja2 environment to load templates from the filesystem
script_directory = Path(__file__).parent
# Set the base directory relative to the script's directory
base_directory = script_directory.parent / "cognee/infrastructure/llm/prompts"
# Construct the full file path
file_path = base_directory / filename
env = Environment(
loader=FileSystemLoader(base_directory),
autoescape=select_autoescape(['html', 'xml', 'txt'])
)
# Load the template by name
template = env.get_template(filename)
# Render the template with the provided context
rendered_template = template.render(context)
return rendered_template
async def render_graph(graph, graph_type):
# Authenticate with your Graphistry API key
import networkx as nx
import pandas as pd
graphistry.register(api=3, username=config.graphistry_username, password=config.graphistry_password)
# Convert the NetworkX graph to a Pandas DataFrame representing the edge list
edges = nx.to_pandas_edgelist(graph)
# Visualize the graph using Graphistry
plotter = graphistry.edges(edges, 'source', 'target')
# Visualize the graph (this will open a URL in your default web browser)
url = plotter.plot(render=False, as_files=True)
print(f"Graph is visualized at: {url}")
#
# How to render a graph
#
# import networkx as nx
# # Create a simple NetworkX graph
#
# Create a simple NetworkX graph
# G = nx.Graph()
#
# # Add nodes
# G.add_node(1)
# G.add_node(2)
#
# # Add an edge between nodes
# Add an edge between nodes
# G.add_edge(1, 2)
#
# import asyncio
#
# # Define the graph type (for this example, it's just a placeholder as the function doesn't use it yet)
# graph_type = "simple"
# Define the graph type (for this example, it's just a placeholder as the function doesn't use it yet)
# graph_type = "networkx"
#
# # Call the render_graph function
# Call the render_graph function
# asyncio.run(render_graph(G, graph_type))
#
async def render_graph(graph, graph_type):
# Authenticate with your Graphistry API key
import networkx as nx
from cognee.config import Config
config = Config()
config.load()
graphistry.register(
api = 3,
username = config.graphistry_username,
password = config.graphistry_password
)
# Convert the NetworkX graph to a Pandas DataFrame representing the edge list
edges = nx.to_pandas_edgelist(graph)
# Visualize the graph using Graphistry
plotter = graphistry.edges(edges, "source", "target")
# Visualize the graph (this will open a URL in your default web browser)
url = plotter.plot(render = False, as_files = True)
print(f"Graph is visualized at: {url}")

509
cognee/vectorstore_manager.py
View file

@ -1,509 +0,0 @@
import logging
logging.basicConfig(level=logging.INFO)
from sqlalchemy.future import select
from cognee.database.relationaldb.models.user import User
from cognee.database.relationaldb.models.memory import MemoryModel
import ast
import tracemalloc
from cognee.database.relationaldb.database_crud import add_entity
tracemalloc.start()
import uuid
from cognee.database.vectordb.basevectordb import BaseMemory
from cognee.config import Config
globalConfig = Config()
class DynamicBaseMemory(BaseMemory):
def __init__(
self,
name: str,
user_id: str,
memory_id: str,
index_name: str,
db_type: str,
namespace: str=None,
embeddings=None,
):
super().__init__(user_id, memory_id, index_name, db_type, namespace, embeddings)
self.name = name
self.attributes = set()
self.methods = set()
self.inheritance = None
self.associations = []
async def add_method(self, method_name):
"""
Add a method to the memory class.
Args:
- method_name (str): The name of the method to be added.
Returns:
None
"""
self.methods.add(method_name)
async def add_attribute(self, attribute_name):
"""
Add an attribute to the memory class.
Args:
- attribute_name (str): The name of the attribute to be added.
Returns:
None
"""
self.attributes.add(attribute_name)
async def get_attribute(self, attribute_name):
"""
Check if the attribute is in the memory class.
Args:
- attribute_name (str): The name of the attribute to be checked.
Returns:
bool: True if attribute exists, False otherwise.
"""
return attribute_name in self.attributes
async def add_association(self, associated_memory):
"""
Add an association to another memory class.
Args:
- associated_memory (MemoryClass): The memory class to be associated with.
Returns:
None
"""
if associated_memory not in self.associations:
self.associations.append(associated_memory)
# Optionally, establish a bidirectional association
associated_memory.associations.append(self)
class Attribute:
def __init__(self, name):
"""
Initialize the Attribute class.
Args:
- name (str): The name of the attribute.
Attributes:
- name (str): Stores the name of the attribute.
"""
self.name = name
class Method:
def __init__(self, name):
"""
Initialize the Method class.
Args:
- name (str): The name of the method.
Attributes:
- name (str): Stores the name of the method.
"""
self.name = name
class Memory:
def __init__(
self,
user_id: str = "676",
session=None,
index_name: str = None,
db_type: str = globalConfig.vectordb,
namespace: str = None,
memory_id: str = None,
memory_class=None,
job_id: str = None,
) -> None:
self.load_environment_variables()
self.memory_id = memory_id
self.user_id = user_id
self.session = session
self.index_name = index_name
self.db_type = db_type
self.namespace = namespace
self.memory_instances = []
self.memory_class = memory_class
self.job_id = job_id
# self.memory_class = DynamicBaseMemory(
# "Memory", user_id, str(self.memory_id), index_name, db_type, namespace
# )
def load_environment_variables(self) -> None:
self.OPENAI_TEMPERATURE = globalConfig.openai_temperature
self.OPENAI_API_KEY = globalConfig.openai_key
@classmethod
async def create_memory(
cls,
user_id: str,
session,
job_id: str = None,
memory_label: str = None,
**kwargs,
):
"""
Class method that acts as a factory method for creating Memory instances.
It performs necessary DB checks or updates before instance creation.
"""
existing_user = await cls.check_existing_user(user_id, session)
logging.info(f"Existing user: {existing_user}")
if existing_user:
# Handle existing user scenario...
memory_id = await cls.check_existing_memory(user_id, memory_label, session)
if memory_id is None:
memory_id = await cls.handle_new_memory(
user_id=user_id,
session=session,
job_id=job_id,
memory_name=memory_label,
)
logging.info(
f"Existing user {user_id} found in the DB. Memory ID: {memory_id}"
)
else:
# Handle new user scenario...
await cls.handle_new_user(user_id, session)
memory_id = await cls.handle_new_memory(
user_id=user_id,
session=session,
job_id=job_id,
memory_name=memory_label,
)
logging.info(
f"New user {user_id} created in the DB. Memory ID: {memory_id}"
)
memory_class = DynamicBaseMemory(
memory_label,
user_id,
str(memory_id),
index_name=memory_label,
db_type=globalConfig.vectordb,
**kwargs,
)
return cls(
user_id=user_id,
session=session,
memory_id=memory_id,
job_id=job_id,
memory_class=memory_class,
**kwargs,
)
async def list_memory_classes(self):
"""
Lists all available memory classes in the memory instance.
"""
# Use a list comprehension to filter attributes that end with '_class'
return [attr for attr in dir(self) if attr.endswith("_class")]
@staticmethod
async def check_existing_user(user_id: str, session):
"""Check if a user exists in the DB and return it."""
result = await session.execute(select(User).where(User.id == user_id))
return result.scalar_one_or_none()
@staticmethod
async def check_existing_memory(user_id: str, memory_label: str, session):
"""Check if a user memory exists in the DB and return it. Filters by user and label"""
try:
result = await session.execute(
select(MemoryModel.id)
.where(MemoryModel.user_id == user_id)
.filter_by(memory_name=memory_label)
.order_by(MemoryModel.created_at)
)
return result.scalar_one_or_none()
except Exception as e:
logging.error(f"An error occurred: {str(e)}")
return None
@staticmethod
async def handle_new_user(user_id: str, session):
"""
Handle new user creation in the database.
Args:
user_id (str): The unique identifier for the new user.
session: The database session for the operation.
Returns:
str: A success message or an error message.
Raises:
Exception: If any error occurs during the user creation process.
"""
try:
new_user = User(id=user_id)
await add_entity(session, new_user)
return "User creation successful."
except Exception as e:
return f"Error creating user: {str(e)}"
@staticmethod
async def handle_new_memory(
user_id: str,
session,
job_id: str = None,
memory_name: str = None,
memory_category: str = "PUBLIC",
):
"""
Handle new memory creation associated with a user.
Args:
user_id (str): The user's unique identifier.
session: The database session for the operation.
job_id (str, optional): The identifier of the associated job, if any.
memory_name (str, optional): The name of the memory.
Returns:
str: The unique memory ID if successful, or an error message.
Raises:
Exception: If any error occurs during memory creation.
"""
try:
memory_id = str(uuid.uuid4())
logging.info("Job id %s", job_id)
memory = MemoryModel(
id=memory_id,
user_id=user_id,
operation_id=job_id,
memory_name=memory_name,
memory_category=memory_category,
methods_list=str(["Memory", "SemanticMemory", "EpisodicMemory"]),
attributes_list=str(
[
"user_id",
"index_name",
"db_type",
"knowledge_source",
"knowledge_type",
"memory_id",
"long_term_memory",
"short_term_memory",
"namespace",
]
),
)
await add_entity(session, memory)
return memory_id
except Exception as e:
return f"Error creating memory: {str(e)}"
async def add_memory_instance(self, memory_class_name: str):
"""Add a new memory instance to the memory_instances list."""
instance = DynamicBaseMemory(
memory_class_name,
self.user_id,
self.memory_id,
self.index_name,
self.db_type,
self.namespace,
)
print("The following instance was defined", instance)
self.memory_instances.append(instance)
async def query_method(self):
methods_list = await self.session.execute(
select(MemoryModel.methods_list).where(MemoryModel.id == self.memory_id)
)
methods_list = methods_list.scalar_one_or_none()
return methods_list
async def manage_memory_attributes(self, existing_user):
"""Manage memory attributes based on the user existence."""
if existing_user:
print(f"ID before query: {self.memory_id}, type: {type(self.memory_id)}")
# attributes_list = await self.session.query(MemoryModel.attributes_list).filter_by(id=self.memory_id[0]).scalar()
attributes_list = await self.query_method()
logging.info(f"Attributes list: {attributes_list}")
if attributes_list is not None:
attributes_list = ast.literal_eval(attributes_list)
await self.handle_attributes(attributes_list)
else:
logging.warning("attributes_list is None!")
else:
attributes_list = [
"user_id",
"index_name",
"db_type",
"knowledge_source",
"knowledge_type",
"memory_id",
"long_term_memory",
"short_term_memory",
"namespace",
]
await self.handle_attributes(attributes_list)
async def handle_attributes(self, attributes_list):
"""Handle attributes for existing memory instances."""
for attr in attributes_list:
await self.memory_class.add_attribute(attr)
async def manage_memory_methods(self, existing_user):
"""
Manage memory methods based on the user existence.
"""
if existing_user:
# Fetch existing methods from the database
# methods_list = await self.session.query(MemoryModel.methods_list).filter_by(id=self.memory_id).scalar()
methods_list = await self.session.execute(
select(MemoryModel.methods_list).where(
MemoryModel.id == self.memory_id[0]
)
)
methods_list = methods_list.scalar_one_or_none()
methods_list = ast.literal_eval(methods_list)
else:
# Define default methods for a new user
methods_list = [
"async_create_long_term_memory",
"async_init",
"add_memories",
"fetch_memories",
"delete_memories",
"async_create_short_term_memory",
"_create_buffer_context",
"_get_task_list",
"_run_main_buffer",
"_available_operations",
"_provide_feedback",
]
# Apply methods to memory instances
for class_instance in self.memory_instances:
for method in methods_list:
class_instance.add_method(method)
async def dynamic_method_call(
self, dynamic_base_memory_instance, method_name: str, *args, **kwargs
):
if method_name in dynamic_base_memory_instance.methods:
method = getattr(dynamic_base_memory_instance, method_name, None)
if method:
return await method(*args, **kwargs)
raise AttributeError(
f"{dynamic_base_memory_instance.name} object has no attribute {method_name}"
)
async def add_dynamic_memory_class(self, class_name: str, namespace: str):
logging.info("Here is the memory id %s", self.memory_id[0])
new_memory_class = DynamicBaseMemory(
class_name,
self.user_id,
self.memory_id[0],
self.index_name,
self.db_type,
namespace,
)
setattr(self, f"{class_name.lower()}_class", new_memory_class)
return new_memory_class
async def add_attribute_to_class(self, class_instance, attribute_name: str):
# add this to database for a particular user and load under memory id
await class_instance.add_attribute(attribute_name)
async def add_method_to_class(self, class_instance, method_name: str):
# add this to database for a particular user and load under memory id
await class_instance.add_method(method_name)
async def main():
# if you want to run the script as a standalone script, do so with the examples below
# memory = Memory(user_id="TestUser")
# await memory.async_init()
params = {
"version": "1.0",
"agreement_id": "AG123456",
"privacy_policy": "https://example.com/privacy",
"terms_of_service": "https://example.com/terms",
"format": "json",
"schema_version": "1.1",
"checksum": "a1b2c3d4e5f6",
"owner": "John Doe",
"license": "MIT",
"validity_start": "2023-08-01",
"validity_end": "2024-07-31",
}
loader_settings = {
"format": "PDF",
"source": "URL",
"path": ["https://www.ibiblio.org/ebooks/London/Call%20of%20Wild.pdf"],
}
# memory_instance = Memory(namespace='SEMANTICMEMORY')
# sss = await memory_instance.dynamic_method_call(memory_instance.semantic_memory_class, 'fetch_memories', observation='some_observation')
from database.relationaldb.database_crud import session_scope
from database.relationaldb.database import AsyncSessionLocal
async with session_scope(AsyncSessionLocal()) as session:
memory = await Memory.create_memory(
"677", session, "SEMANTICMEMORY", namespace="SEMANTICMEMORY"
)
ff = memory.memory_instances
logging.info("ssss %s", ff)
# Adding a memory instance
# await memory.add_memory_instance("ExampleMemory")
# Managing memory attributes
existing_user = await Memory.check_existing_user("677", session)
print("here is the existing user", existing_user)
await memory.manage_memory_attributes(existing_user)
# aeehuvyq_semanticememory_class
await memory.add_dynamic_memory_class("semanticmemory", "SEMANTICMEMORY")
await memory.add_method_to_class(memory.semanticmemory_class, "add_memories")
await memory.add_method_to_class(memory.semanticmemory_class, "fetch_memories")
sss = await memory.dynamic_method_call(
memory.semanticmemory_class,
"add_memories",
observation="some_observation",
params=params,
loader_settings=loader_settings,
)
# susu = await memory.dynamic_method_call(
# memory.semanticmemory_class,
# "fetch_memories",
# observation="document summary",
# )
# print(susu)
# Adding a dynamic memory class
# dynamic_memory = memory.add_dynamic_memory_class("DynamicMemory", "ExampleNamespace")
# memory_instance = Memory(namespace='PROCEDURALMEMORY', session=session)
# procedural_memory_class = memory_instance.add_dynamic_memory_class('ProceduralMemory', 'PROCEDURALMEMORY')
# memory_instance.add_method_to_class(procedural_memory_class, 'add_memories')
#
# print(sss)
# load_jack_london = await memory._add_semantic_memory(observation = "bla", loader_settings=loader_settings, params=params)
# print(load_jack_london)
modulator = {"relevance": 0.1, "frequency": 0.1}
if __name__ == "__main__":
import asyncio
asyncio.run(main())

916
main.py
View file

@ -1,916 +0,0 @@
from typing import Optional, List
from neo4j.exceptions import Neo4jError
from pydantic import BaseModel, Field
from cognee.database.graphdb.graph import Neo4jGraphDB
from cognee.database.relationaldb.models.memory import MemoryModel
import os
from cognee.database.relationaldb.database_crud import (
session_scope,
update_entity_graph_summary,
)
from cognee.database.relationaldb.database import AsyncSessionLocal
from cognee.utils import generate_letter_uuid
import instructor
from openai import OpenAI
from cognee.vectorstore_manager import Memory
from cognee.database.relationaldb.database_crud import fetch_job_id
import uuid
from cognee.database.relationaldb.models.sessions import Session
from cognee.database.relationaldb.models.operation import Operation
from cognee.database.relationaldb.database_crud import (
session_scope,
add_entity,
update_entity,
fetch_job_id,
)
from cognee.database.relationaldb.models.metadatas import MetaDatas
from cognee.database.relationaldb.models.docs import DocsModel
from cognee.database.relationaldb.models.memory import MemoryModel
from cognee.database.relationaldb.models.user import User
from cognee.classifiers.classify_summary import classify_summary
from cognee.classifiers.classify_documents import classify_documents
from cognee.classifiers.classify_user_query import classify_user_query
from cognee.classifiers.classify_user_input import classify_user_input
aclient = instructor.patch(OpenAI())
DEFAULT_PRESET = "promethai_chat"
preset_options = [DEFAULT_PRESET]
PROMETHAI_DIR = os.path.join(os.path.expanduser("~"), ".")
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
from cognee.config import Config
config = Config()
config.load()
from cognee.utils import get_document_names
from sqlalchemy.orm import selectinload, joinedload, contains_eager
import logging
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from cognee.utils import (
get_document_names,
generate_letter_uuid,
get_memory_name_by_doc_id,
get_unsumarized_vector_db_namespace,
get_vectordb_namespace,
get_vectordb_document_name,
)
from cognee.shared.language_processing import (
translate_text,
detect_language,
)
async def fetch_document_vectordb_namespace(
session: AsyncSession, user_id: str, namespace_id: str, doc_id: str = None
):
logging.info("user id is", user_id)
memory = await Memory.create_memory(
user_id, session, namespace=namespace_id, memory_label=namespace_id
)
# Managing memory attributes
existing_user = await Memory.check_existing_user(user_id, session)
print("here is the existing user", existing_user)
await memory.manage_memory_attributes(existing_user)
print("Namespace id is %s", namespace_id)
await memory.add_dynamic_memory_class(namespace_id.lower(), namespace_id)
namespace_class = namespace_id + "_class"
dynamic_memory_class = getattr(memory, namespace_class.lower(), None)
methods_to_add = ["add_memories", "fetch_memories", "delete_memories"]
if dynamic_memory_class is not None:
for method_name in methods_to_add:
await memory.add_method_to_class(dynamic_memory_class, method_name)
print(f"Memory method {method_name} has been added")
else:
print(f"No attribute named in memory.")
print("Available memory classes:", await memory.list_memory_classes())
result = await memory.dynamic_method_call(
dynamic_memory_class,
"fetch_memories",
observation="placeholder",
search_type="summary_filter_by_object_name",
params=doc_id,
)
logging.info("Result is %s", str(result))
return result, namespace_id
async def load_documents_to_vectorstore(
session: AsyncSession,
user_id: str,
content: str = None,
job_id: str = None,
loader_settings: dict = None,
memory_type: str = "PRIVATE",
):
namespace_id = str(generate_letter_uuid()) + "_" + "SEMANTICMEMORY"
namespace_class = namespace_id + "_class"
logging.info("Namespace created with id %s", namespace_id)
try:
new_user = User(id=user_id)
await add_entity(session, new_user)
except:
pass
if job_id is None:
job_id = str(uuid.uuid4())
await add_entity(
session,
Operation(
id=job_id,
user_id=user_id,
operation_status="RUNNING",
operation_type="DATA_LOAD",
),
)
memory = await Memory.create_memory(
user_id,
session,
namespace=namespace_id,
job_id=job_id,
memory_label=namespace_id,
)
if content is not None:
document_names = [content[:30]]
if loader_settings is not None:
document_source = (
loader_settings.get("document_names")
if isinstance(loader_settings.get("document_names"), list)
else loader_settings.get("path", "None")
)
logging.info("Document source is %s", document_source)
# try:
document_names = get_document_names(document_source[0])
logging.info(str(document_names))
# except:
# document_names = document_source
for doc in document_names:
from cognee.shared.language_processing import (
translate_text,
detect_language,
)
# translates doc titles to english
if loader_settings is not None:
logging.info("Detecting language of document %s", doc)
loader_settings["single_document_path"] = (
loader_settings.get("path", "None")[0] + "/" + doc
)
logging.info(
"Document path is %s",
loader_settings.get("single_document_path", "None"),
)
memory_category = loader_settings.get("memory_category", "PUBLIC")
if loader_settings is None:
memory_category = "CUSTOM"
if detect_language(doc) != "en":
doc_ = doc.strip(".pdf").replace("-", " ")
doc_ = translate_text(doc_, "sr", "en")
else:
doc_ = doc
doc_id = str(uuid.uuid4())
logging.info("Document name is %s", doc_)
await add_entity(
session,
DocsModel(
id=doc_id,
operation_id=job_id,
graph_summary=False,
memory_category=memory_category,
doc_name=doc_,
),
)
# Managing memory attributes
existing_user = await Memory.check_existing_user(user_id, session)
await memory.manage_memory_attributes(existing_user)
params = {"doc_id": doc_id}
print("Namespace id is %s", namespace_id)
await memory.add_dynamic_memory_class(namespace_id.lower(), namespace_id)
dynamic_memory_class = getattr(memory, namespace_class.lower(), None)
methods_to_add = ["add_memories", "fetch_memories", "delete_memories"]
if dynamic_memory_class is not None:
for method_name in methods_to_add:
await memory.add_method_to_class(dynamic_memory_class, method_name)
print(f"Memory method {method_name} has been added")
else:
print(f"No attribute named in memory.")
print("Available memory classes:", await memory.list_memory_classes())
result = await memory.dynamic_method_call(
dynamic_memory_class,
"add_memories",
observation=content,
params=params,
loader_settings=loader_settings,
)
await update_entity(session, Operation, job_id, "SUCCESS")
return 1
async def user_query_to_graph_db(session: AsyncSession, user_id: str, query_input: str):
try:
new_user = User(id=user_id)
await add_entity(session, new_user)
except:
pass
job_id = str(uuid.uuid4())
await add_entity(
session,
Operation(
id=job_id,
user_id=user_id,
operation_status="RUNNING",
operation_type="USER_QUERY_TO_GRAPH_DB",
),
)
detected_language = detect_language(query_input)
if detected_language != "en":
translated_query = translate_text(query_input, detected_language, "en")
else:
translated_query = query_input
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
cypher_query = (
await neo4j_graph_db.generate_cypher_query_for_user_prompt_decomposition(
user_id, translated_query
)
)
result = await neo4j_graph_db.query(cypher_query)
await neo4j_graph_db.run_merge_query(
user_id=user_id, memory_type="SemanticMemory", similarity_threshold=0.8
)
await neo4j_graph_db.run_merge_query(
user_id=user_id, memory_type="EpisodicMemory", similarity_threshold=0.8
)
await neo4j_graph_db.close()
await update_entity(session, Operation, job_id, "SUCCESS")
return result
async def add_documents_to_graph_db(
session: AsyncSession, user_id: str = None, document_memory_types: list = None
):
""""""
if document_memory_types is None:
document_memory_types = ["PUBLIC"]
logging.info("Document memory types are", document_memory_types)
try:
# await update_document_vectordb_namespace(postgres_session, user_id)
memory_details, docs = await get_unsumarized_vector_db_namespace(
session, user_id
)
logging.info("Docs are", docs)
memory_details = [
detail for detail in memory_details if detail[1] in document_memory_types
]
logging.info("Memory details", memory_details)
for doc in docs:
logging.info("Memory names are", memory_details)
doc_name, doc_id = doc
logging.info("Doc id is", doc_id)
try:
classification_content = await fetch_document_vectordb_namespace(
session, user_id, memory_details[0][0], doc_id
)
retrieval_chunks = [
item["text"]
for item in classification_content[0]["data"]["Get"][
memory_details[0][0]
]
]
logging.info("Classification content is", classification_content)
except:
classification_content = ""
retrieval_chunks = ""
# retrieval_chunks = [item['text'] for item in classification_content[0]['data']['Get'][memory_details[0]]]
# Concatenating the extracted text values
concatenated_retrievals = " ".join(retrieval_chunks)
print(concatenated_retrievals)
logging.info("Retrieval chunks are", retrieval_chunks)
classification = await classify_documents(
doc_name, document_id=doc_id, content=concatenated_retrievals
)
logging.info("Classification is %s", str(classification))
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
if document_memory_types == ["PUBLIC"]:
await create_public_memory(
user_id=user_id, labels=["sr"], topic="PublicMemory"
)
ids = await neo4j_graph_db.retrieve_node_id_for_memory_type(
topic="PublicMemory"
)
await neo4j_graph_db.close()
print(ids)
else:
ids = await neo4j_graph_db.retrieve_node_id_for_memory_type(
topic="SemanticMemory"
)
await neo4j_graph_db.close()
print(ids)
for id in ids:
print(id.get("memoryId"))
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
if document_memory_types == ["PUBLIC"]:
rs = neo4j_graph_db.create_document_node_cypher(
classification, user_id, public_memory_id=id.get("memoryId")
)
await neo4j_graph_db.close()
else:
rs = neo4j_graph_db.create_document_node_cypher(
classification, user_id, memory_type="SemanticMemory"
)
await neo4j_graph_db.close()
logging.info("Cypher query is %s", str(rs))
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
await neo4j_graph_db.query(rs)
await neo4j_graph_db.close()
logging.info("WE GOT HERE")
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
if memory_details[0][1] == "PUBLIC":
await neo4j_graph_db.update_document_node_with_db_ids(
vectordb_namespace=memory_details[0][0], document_id=doc_id
)
await neo4j_graph_db.close()
else:
await neo4j_graph_db.update_document_node_with_db_ids(
vectordb_namespace=memory_details[0][0],
document_id=doc_id,
user_id=user_id,
)
await neo4j_graph_db.close()
# await update_entity_graph_summary(session, DocsModel, doc_id, True)
except Exception as e:
return e
class ResponseString(BaseModel):
response: str = Field(
default=None
) # Defaulting to None or you can use a default string like ""
quotation: str = Field(default=None) # Same here
def generate_graph(input) -> ResponseString:
out = aclient.chat.completions.create(
model="gpt-4-1106-preview",
messages=[
{
"role": "user",
"content": f"""Use the given context to answer query and use help of associated context: {input}. """,
},
{
"role": "system",
"content": """You are a top-tier algorithm
designed for using context summaries based on cognitive psychology to answer user queries, and provide a simple response.
Do not mention anything explicit about cognitive architecture, but use the context to answer the query. If you are using a document, reference document metadata field""",
},
],
response_model=ResponseString,
)
return out
async def user_context_enrichment(
session,
user_id: str,
query: str,
generative_response: bool = False,
memory_type: str = None,
) -> str:
"""
Asynchronously enriches the user context by integrating various memory systems and document classifications.
This function uses cognitive architecture to access and manipulate different memory systems (semantic, episodic, and procedural) associated with a user.
It fetches memory details from a Neo4j graph database, classifies document categories based on the user's query, and retrieves document IDs for relevant categories.
The function also dynamically manages memory attributes and methods, extending the context with document store information to enrich the user's query response.
Parameters:
- session (AsyncSession): The database session for executing queries.
- user_id (str): The unique identifier of the user.
- query (str): The original query from the user.
Returns:
- str: The final enriched context after integrating various memory systems and document classifications.
The function performs several key operations:
1. Retrieves semantic and episodic memory details for the user from the Neo4j graph database.
2. Logs and classifies document categories relevant to the user's query.
3. Fetches document IDs from Neo4j and corresponding memory names from a PostgreSQL database.
4. Dynamically manages memory attributes and methods, including the addition of methods like 'add_memories', 'fetch_memories', and 'delete_memories' to the memory class.
5. Extends the context with document store information relevant to the user's query.
6. Generates and logs the final result after processing and integrating all information.
Raises:
- Exception: Propagates any exceptions that occur during database operations or memory management.
Example Usage:
```python
enriched_context = await user_context_enrichment(session, "user123", "How does cognitive architecture work?")
```
"""
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
# await user_query_to_graph_db(session, user_id, query)
semantic_mem = await neo4j_graph_db.retrieve_semantic_memory(user_id=user_id)
await neo4j_graph_db.close()
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
episodic_mem = await neo4j_graph_db.retrieve_episodic_memory(user_id=user_id)
logging.info("Episodic memory is %s", episodic_mem)
await neo4j_graph_db.close()
# public_mem = neo4j_graph_db.retrieve_public_memory(user_id=user_id)
if detect_language(query) != "en":
query = translate_text(query, "sr", "en")
logging.info("Translated query is %s", str(query))
if memory_type == "PublicMemory":
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
summaries = await neo4j_graph_db.get_memory_linked_document_summaries(
user_id=user_id, memory_type=memory_type
)
await neo4j_graph_db.close()
logging.info("Summaries are is %s", summaries)
# logging.info("Context from graphdb is %s", context)
# result = neo4j_graph_db.query(document_categories_query)
# summaries = [record.get("summary") for record in result]
# logging.info('Possible document categories are', str(result))
# logging.info('Possible document categories are', str(categories))
max_attempts = 3
relevant_summary_id = None
for _ in range(max_attempts):
relevant_summary_id = await classify_summary(
query=query, document_summaries=str(summaries)
)
logging.info("Relevant summary id is %s", relevant_summary_id)
if relevant_summary_id is not None:
break
# logging.info("Relevant categories after the classifier are %s", relevant_categories)
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
postgres_id = await neo4j_graph_db.get_memory_linked_document_ids(
user_id, summary_id=relevant_summary_id, memory_type=memory_type
)
await neo4j_graph_db.close()
# postgres_id = neo4j_graph_db.query(get_doc_ids)
logging.info("Postgres ids are %s", postgres_id)
namespace_id = await get_memory_name_by_doc_id(session, postgres_id[0])
logging.info("Namespace ids are %s", namespace_id)
params = {"doc_id": postgres_id[0]}
namespace_id = namespace_id[0]
namespace_class = namespace_id + "_class"
if memory_type == "PublicMemory":
user_id = "system_user"
memory = await Memory.create_memory(
user_id,
session,
namespace=namespace_id,
job_id="23232",
memory_label=namespace_id,
)
existing_user = await Memory.check_existing_user(user_id, session)
print("here is the existing user", existing_user)
await memory.manage_memory_attributes(existing_user)
print("Namespace id is %s", namespace_id)
await memory.add_dynamic_memory_class(namespace_id.lower(), namespace_id)
dynamic_memory_class = getattr(memory, namespace_class.lower(), None)
methods_to_add = ["add_memories", "fetch_memories", "delete_memories"]
if dynamic_memory_class is not None:
for method_name in methods_to_add:
await memory.add_method_to_class(dynamic_memory_class, method_name)
print(f"Memory method {method_name} has been added")
else:
print(f"No attribute named in memory.")
print("Available memory classes:", await memory.list_memory_classes())
results = await memory.dynamic_method_call(
dynamic_memory_class,
"fetch_memories",
observation=query,
params=postgres_id[0],
search_type="summary_filter_by_object_name",
)
logging.info("Result is %s", str(results))
search_context = ""
for result in results["data"]["Get"][namespace_id]:
# Assuming 'result' is a dictionary and has keys like 'source', 'text'
source = (
result["source"]
.replace("-", " ")
.replace(".pdf", "")
.replace(".data/", "")
)
text = result["text"]
search_context += f"Document source: {source}, Document text: {text} \n"
else:
search_context = "No relevant documents found"
context = f""" You are a memory system that uses cognitive architecture to enrich the
LLM context and provide better query response.
You have access to the following information:
EPISODIC MEMORY: {episodic_mem}
SEMANTIC MEMORY: {semantic_mem}
PROCEDURAL MEMORY: NULL
SEARCH CONTEXT: The following documents provided with sources they were
extracted from could be used to provide an answer {search_context}
The original user query: {query}
"""
if generative_response is not True:
return context
else:
generative_result = generate_graph(context)
logging.info("Generative result is %s", generative_result.model_dump_json())
return generative_result.model_dump_json()
async def create_public_memory(
user_id: str = None, labels: list = None, topic: str = None
) -> Optional[int]:
"""
Create a public memory node associated with a user in a Neo4j graph database.
If Public Memory exists, it will return the id of the memory.
This is intended as standalone node that can be attached to any user.
It is not attached to any user by default.
Args:
user_id (str): The unique identifier for the user.
session (AsyncSession): An asynchronous session for database operations.
Returns:
Optional[int]: The ID of the created public memory node or None if an error occurs.
:param labels: Label for the memory, to help filter for different countries
:param topic: Topic for the memory, to help provide a name
"""
# Validate input parameters
if not labels:
labels = ["sr"] # Labels for the memory node
if not topic:
topic = "PublicMemory"
try:
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
# Assuming the topic for public memory is predefined, e.g., "PublicMemory"
# Create the memory node
memory_id = await neo4j_graph_db.create_memory_node(labels=labels, topic=topic)
await neo4j_graph_db.close()
return memory_id
except Neo4jError as e:
logging.error(f"Error creating public memory node: {e}")
return None
async def attach_user_to_memory(
user_id: str = None, labels: list = None, topic: str = None
) -> Optional[int]:
"""
Link user to public memory
Args:
user_id (str): The unique identifier for the user.
topic (str): Memory name
Returns:
Optional[int]: The ID of the created public memory node or None if an error occurs.
:param labels: Label for the memory, to help filter for different countries
:param topic: Topic for the memory, to help provide a name
"""
# Validate input parameters
if not user_id:
raise ValueError("User ID is required.")
if not labels:
labels = ["sr"] # Labels for the memory node
if not topic:
topic = "PublicMemory"
try:
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
# Assuming the topic for public memory is predefined, e.g., "PublicMemory"
ids = await neo4j_graph_db.retrieve_node_id_for_memory_type(topic=topic)
await neo4j_graph_db.close()
for id in ids:
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
linked_memory = await neo4j_graph_db.link_public_memory_to_user(
memory_id=id.get("memoryId"), user_id=user_id
)
await neo4j_graph_db.close()
return 1
except Neo4jError as e:
logging.error(f"Error creating public memory node: {e}")
return None
async def unlink_user_from_memory(
user_id: str = None, labels: list = None, topic: str = None
) -> Optional[int]:
"""
Unlink user from memory
Args:
user_id (str): The unique identifier for the user.
topic (str): Memory name
Returns:
Optional[int]: The ID of the created public memory node or None if an error occurs.
:param labels: Label for the memory, to help filter for different countries
:param topic: Topic for the memory, to help provide a name
"""
# Validate input parameters
if not user_id:
raise ValueError("User ID is required.")
if not labels:
labels = ["sr"] # Labels for the memory node
if not topic:
topic = "PublicMemory"
try:
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
# Assuming the topic for public memory is predefined, e.g., "PublicMemory"
ids = await neo4j_graph_db.retrieve_node_id_for_memory_type(topic=topic)
await neo4j_graph_db.close()
for id in ids:
neo4j_graph_db = Neo4jGraphDB(
url=config.graph_database_url,
username=config.graph_database_username,
password=config.graph_database_password,
)
linked_memory = neo4j_graph_db.unlink_memory_from_user(
memory_id=id.get("memoryId"), user_id=user_id
)
await neo4j_graph_db.close()
return 1
except Neo4jError as e:
logging.error(f"Error creating public memory node: {e}")
return None
async def relevance_feedback(query: str, input_type: str):
max_attempts = 6
result = None
for attempt in range(1, max_attempts + 1):
result = await classify_user_input(query, input_type=input_type)
if isinstance(result, bool):
break # Exit the loop if a result of type bool is obtained
return result
async def main():
user_id = "user_test_1_1"
async with session_scope(AsyncSessionLocal()) as session:
# await update_entity(session, DocsModel, "8cd9a022-5a7a-4af5-815a-f988415536ae", True)
# output = await get_unsumarized_vector_db_namespace(session, user_id)
class GraphQLQuery(BaseModel):
query: str
# gg = await user_query_to_graph_db(session, user_id, "How does cognitive architecture work?")
# print(gg)
# def cypher_statement_correcting( input: str) -> str:
# out = aclient.chat.completions.create(
# model=config.model,
# temperature=0,
# max_tokens=2000,
# messages=[
# {
# "role": "user",
# "content": f"""Check the cypher query for syntax issues, and fix any if found and return it as is: {input}. """,
#
# },
# {"role": "system", "content": """You are a top-tier algorithm
# designed for checking cypher queries for neo4j graph databases. You have to return input provided to you as is."""}
# ],
# response_model=GraphQLQuery,
# )
# return out
#
#
# query= """WITH person1_4f21b68c73e24d0497e1010eb747b892, location2_dc0c68a9651142d38b6e117bfdc5c227, object3_4c7ba47babd24be1b35c30c42c87a3e9, product4_c984d5f9695f48ee9a43f58f57cc6740, location5_5e43f4c45b3c44ea897c12220db4c051, object6_5cdb87ad488c450c9dbce07b7daf3d8d, information7_f756e3f3720c4fe5aeb01287badaf088, event8_da6334e744454264900296319e14b532, action9_48e45419604e4d66b3e718ee1d6c095f, action10_f48acb1db4da4934afbe17363e9e63a4, user , semantic, episodic, buffer
# CREATE (person1_4f21b68c73e24d0497e1010eb747b892)-[:EXPERIENCED]->(event8_da6334e744454264900296319e14b532)
# CREATE (person1_4f21b68c73e24d0497e1010eb747b892)-[:HAS]->(object3_4c7ba47babd24be1b35c30c42c87a3e9)
# CREATE (object3_4c7ba47babd24be1b35c30c42c87a3e9)-[:INCLUDES]->(product4_c984d5f9695f48ee9a43f58f57cc6740)
# CREATE (product4_c984d5f9695f48ee9a43f58f57cc6740)-[:TO_BE_PURCHASED_AT]->(location5_5e43f4c45b3c44ea897c12220db4c051)
# CREATE (person1_4f21b68c73e24d0497e1010eb747b892)-[:INTENDS_TO_PERFORM]->(action9_48e45419604e4d66b3e718ee1d6c095f)
# CREATE (object6_5cdb87ad488c450c9dbce07b7daf3d8d)-[:A_CLASSICAL_BOOK_TO_BE_SUMMARIZED]->(information7_f756e3f3720c4fe5aeb01287badaf088)
# CREATE (person1_4f21b68c73e24d0497e1010eb747b892)-[:NEEDS_TO_COMPLETE]->(action10_f48acb1db4da4934afbe17363e9e63a4)
# WITH person1_4f21b68c73e24d0497e1010eb747b892, location2_dc0c68a9651142d38b6e117bfdc5c227, object3_4c7ba47babd24be1b35c30c42c87a3e9, product4_c984d5f9695f48ee9a43f58f57cc6740, location5_5e43f4c45b3c44ea897c12220db4c051, object6_5cdb87ad488c450c9dbce07b7daf3d8d, information7_f756e3f3720c4fe5aeb01287badaf088, event8_da6334e744454264900296319e14b532, action9_48e45419604e4d66b3e718ee1d6c095f, action10_f48acb1db4da4934afbe17363e9e63a4, user, semantic, episodic, buffer
# CREATE (episodic)-[:HAS_EVENT]->(person1_4f21b68c73e24d0497e1010eb747b892)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(person1_4f21b68c73e24d0497e1010eb747b892)
# CREATE (episodic)-[:HAS_EVENT]->(location2_dc0c68a9651142d38b6e117bfdc5c227)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(location2_dc0c68a9651142d38b6e117bfdc5c227)
# CREATE (episodic)-[:HAS_EVENT]->(object3_4c7ba47babd24be1b35c30c42c87a3e9)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(object3_4c7ba47babd24be1b35c30c42c87a3e9)
# CREATE (episodic)-[:HAS_EVENT]->(product4_c984d5f9695f48ee9a43f58f57cc6740)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(product4_c984d5f9695f48ee9a43f58f57cc6740)
# CREATE (episodic)-[:HAS_EVENT]->(location5_5e43f4c45b3c44ea897c12220db4c051)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(location5_5e43f4c45b3c44ea897c12220db4c051)
# CREATE (episodic)-[:HAS_EVENT]->(object6_5cdb87ad488c450c9dbce07b7daf3d8d)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(object6_5cdb87ad488c450c9dbce07b7daf3d8d)
# CREATE (episodic)-[:HAS_EVENT]->(information7_f756e3f3720c4fe5aeb01287badaf088)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(information7_f756e3f3720c4fe5aeb01287badaf088)
# CREATE (episodic)-[:HAS_EVENT]->(event8_da6334e744454264900296319e14b532)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(event8_da6334e744454264900296319e14b532)
# CREATE (episodic)-[:HAS_EVENT]->(action9_48e45419604e4d66b3e718ee1d6c095f)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(action9_48e45419604e4d66b3e718ee1d6c095f)
# CREATE (episodic)-[:HAS_EVENT]->(action10_f48acb1db4da4934afbe17363e9e63a4)
# CREATE (buffer)-[:CURRENTLY_HOLDING]->(action10_f48acb1db4da4934afbe17363e9e63a4)"""
#
# out = cypher_statement_correcting(query)
# print(out)
#
# out = await user_query_to_graph_db(session, user_id, "I walked in the forest yesterday and added to my list I need to buy some milk in the store and get a summary from a classical book i read yesterday")
# print(out)
# load_doc_to_graph = await add_documents_to_graph_db(session, user_id)
# print(load_doc_to_graph)
# user_id = "test_user"
# loader_settings = {
# "format": "PDF",
# "source": "DEVICE",
# "path": [".data"]
# }
# await load_documents_to_vectorstore(session, user_id, loader_settings=loader_settings)
# await create_public_memory(user_id=user_id, labels=['sr'], topic="PublicMemory")
# await add_documents_to_graph_db(session, user_id)
#
# neo4j_graph_db = Neo4jGraphDB(
# url=config.graph_database_url,
# username=config.graph_database_username,
# password=config.graph_database_password,
# )
#
# out = neo4j_graph_db.run_merge_query(
# user_id=user_id, memory_type="SemanticMemory", similarity_threshold=0.5
# )
# bb = neo4j_graph_db.query(out)
# print(bb)
# await attach_user_to_memory(user_id=user_id, labels=['sr'], topic="PublicMemory")
user_id = "test_user"
return_ = await user_context_enrichment(user_id=user_id, query="I need to understand what did I do yesterday?", session=session, memory_type="SemanticMemory", generative_response=True)
print(return_)
# aa = await relevance_feedback("I need to understand what did I do yesterday", "PublicMemory")
# print(aa)
# document_summary = {
# 'DocumentCategory': 'Science',
# 'Title': 'The Future of AI',
# 'Summary': 'An insightful article about the advancements in AI.',
# 'd_id': 'doc123'
# }
#
# # Example user ID
# user_id = 'user'
#
# # value = await neo4j_graph_db.create_memory_node(labels=['sr'])
# # print(value)
# # neo4j_graph_db.close()
#
# await add_documents_to_graph_db(session, user_id)
# neo4j_graph_db.link_public_memory_to_user(memory_id = 17,user_id=user_id)
#
# ids = neo4j_graph_db.retrieve_node_id_for_memory_type(topic="Document")
# print(ids)
#
# for id in ids:
# print(id.get('memoryId'))
#
# neo4j_graph_db.delete_memory_node(memory_id = id.get('memoryId'), topic="Document")
#
# neo4j_graph_db.delete_memory_node(memory_id=16, topic="PublicSerbianArchitecture")
# neo4j_graph_db.unlink_memory_from_user(memory_id = 17,user_id=user_id)
# cypher_query_public = neo4j_graph_db.create_document_node_cypher(document_summary, user_id, memory_type="PUBLIC")
# neo4j_graph_db.query(cypher_query_public)
# link_memory_to_user(user_id, session)
# neo4j_graph_db.create_memory_node(labels=['sr'])
# out = await get_vectordb_namespace(session, user_id)
# params = {
# "version": "1.0",
# "agreement_id": "AG123456",
# "privacy_policy": "https://example.com/privacy",
# "terms_of_service": "https://example.com/terms",
# "format": "json",
# "schema_version": "1.1",
# "checksum": "a1b2c3d4e5f6",
# "owner": "John Doe",
# "license": "MIT",
# "validity_start": "2023-08-01",
# "validity_end": "2024-07-31",
# }
# loader_settings = {
# "format": "PDF",
# "source": "DEVICE",
# "path": [".data"],
# "strategy": "SUMMARY",
# }
# await load_documents_to_vectorstore(session, user_id, loader_settings=loader_settings)
# await user_query_to_graph_db(session, user_id, "I walked in the forest yesterday and added to my list I need to buy some milk in the store and get a summary from a classical book i read yesterday")
# await add_documents_to_graph_db(session, user_id, loader_settings=loader_settings)
# await user_context_enrichment(session, user_id, query="Tell me about the book I read yesterday")
if __name__ == "__main__":
import asyncio
asyncio.run(main())