From f2832fa6cd01a8a2be3491d3afa47cb2de441706 Mon Sep 17 00:00:00 2001 From: Boris Arzentar Date: Mon, 19 Feb 2024 18:43:55 +0100 Subject: [PATCH] fix: add python module resolution root and remove loading of env variables --- api.py | 32 +++--- cognitive_architecture/config.py | 2 +- .../database/create_database.py | 72 -------------- .../database/data/cognee.db | Bin 0 -> 90112 bytes .../database/database_manager.py | 91 ++++++++++++------ .../database/graphdb/graph.py | 2 - .../database/relationaldb/database.py | 42 ++++---- .../database/relationaldb/models/memory.py | 3 - .../database/vectordb/basevectordb.py | 2 - .../database/vectordb/vectordb.py | 12 --- cognitive_architecture/fetch_secret.py | 2 +- cognitive_architecture/llm/queries.py | 3 - cognitive_architecture/setup_database.py | 27 ++++++ .../shared/language_processing.py | 3 - cognitive_architecture/utils.py | 4 - cognitive_architecture/vectorstore_manager.py | 17 ++-- docker-compose.yml | 28 +++--- entrypoint.sh | 6 +- main.py | 3 +- pyproject.toml | 1 + 20 files changed, 158 insertions(+), 194 deletions(-) delete mode 100644 cognitive_architecture/database/create_database.py create mode 100644 cognitive_architecture/database/data/cognee.db create mode 100644 cognitive_architecture/setup_database.py diff --git a/api.py b/api.py index 55f949c96..7017d24da 100644 --- a/api.py +++ b/api.py @@ -1,23 +1,9 @@ -import json -import logging import os -from enum import Enum -from typing import Dict, Any - +import json import uvicorn -from fastapi import FastAPI, BackgroundTasks, HTTPException -from fastapi.responses import JSONResponse -from pydantic import BaseModel - -from cognitive_architecture.database.relationaldb.database import AsyncSessionLocal -from cognitive_architecture.database.relationaldb.database_crud import session_scope -from cognitive_architecture.vectorstore_manager import Memory -from dotenv import load_dotenv -from main import add_documents_to_graph_db, user_context_enrichment -from cognitive_architecture.config import Config from fastapi import Depends -OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") +import logging # Set up logging logging.basicConfig( @@ -27,11 +13,23 @@ logging.basicConfig( logger = logging.getLogger(__name__) -load_dotenv() +from cognitive_architecture.config import Config config = Config() config.load() +from typing import Dict, Any +from fastapi import FastAPI, BackgroundTasks, HTTPException +from fastapi.responses import JSONResponse +from pydantic import BaseModel + +from cognitive_architecture.database.relationaldb.database import AsyncSessionLocal +from cognitive_architecture.database.relationaldb.database_crud import session_scope +from cognitive_architecture.vectorstore_manager import Memory +from main import add_documents_to_graph_db, user_context_enrichment + +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") + app = FastAPI(debug=True) # # from auth.cognito.JWTBearer import JWTBearer diff --git a/cognitive_architecture/config.py b/cognitive_architecture/config.py index d92fe5803..d5cc58184 100644 --- a/cognitive_architecture/config.py +++ b/cognitive_architecture/config.py @@ -29,7 +29,7 @@ class Config: db_path = Path(__file__).resolve().parent / "database/data" vectordb: str = os.getenv("VECTORDB", "weaviate") - db_type: str = os.getenv("DB_TYPE", "postgres") + db_type: str = os.getenv("DB_TYPE", "sqlite") db_name: str = os.getenv("DB_NAME", "cognee.db") db_host: str = os.getenv("DB_HOST", "localhost") db_port: str = os.getenv("DB_PORT", "5432") diff --git a/cognitive_architecture/database/create_database.py b/cognitive_architecture/database/create_database.py deleted file mode 100644 index e7617c2c3..000000000 --- a/cognitive_architecture/database/create_database.py +++ /dev/null @@ -1,72 +0,0 @@ -"""This module provides functionalities for creating and managing databases.""" - -import asyncio -import os -import logging -from contextlib import asynccontextmanager -from sqlalchemy.ext.asyncio import create_async_engine -from relationaldb.models import memory, metadatas, operation, sessions, user, docs -from sqlalchemy import create_engine, text -from dotenv import load_dotenv -from relationaldb.database import ( - Base,get_sqlalchemy_database_url) -from cognitive_architecture.config import Config -config = Config() -config.load() - -load_dotenv() -logger = logging.getLogger(__name__) - - - - -class DatabaseManager: - """Manages database creation, deletion, and table initialization.""" - def __init__(self): - """Initialize the Database Url with a given configuration.""" - self.engine = create_async_engine(get_sqlalchemy_database_url(), echo=True) - self.db_type = config.db_type - - @asynccontextmanager - async def get_connection(self): - """Initialize the DatabaseManager with a given configuration.""" - if self.db_type in ["sqlite", "duckdb"]: - # For SQLite and DuckDB, the engine itself manages connections - yield self.engine - else: - async with self.engine.connect() as connection: - yield connection - - async def database_exists(self, db_name): - """Check if a database exists.""" - if self.db_type in ["sqlite", "duckdb"]: - # For SQLite and DuckDB, check if the database file exists - return os.path.exists(db_name) - else: - query = text(f"SELECT 1 FROM pg_database WHERE datname='{db_name}'") - async with self.get_connection() as connection: - result = await connection.execute(query) - return await result.fetchone() is not None - - async def create_database(self, db_name): - """Create a new database.""" - if self.db_type not in ["sqlite", "duckdb"]: - # For databases like PostgreSQL, create the database explicitly - async with self.get_connection() as connection: - await connection.execute(text(f"CREATE DATABASE {db_name}")) - - async def drop_database(self, db_name): - """Drop an existing database.""" - if self.db_type in ["sqlite", "duckdb"]: - # For SQLite and DuckDB, simply remove the database file - os.remove(db_name) - else: - async with self.get_connection() as connection: - await connection.execute(text(f"DROP DATABASE IF EXISTS {db_name}")) - - async def create_tables(self): - """Create tables based on the SQLAlchemy Base metadata.""" - async with self.engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - - diff --git a/cognitive_architecture/database/data/cognee.db b/cognitive_architecture/database/data/cognee.db new file mode 100644 index 0000000000000000000000000000000000000000..32d599adb4f4ad55095a346df05d9bbbcdb91105 GIT binary patch literal 90112 zcmeI)Pfy!s9Kdm#GzkGhC@rl2D=*!V9QMK|=OnA@p04zyauDY~*yD&*G->T+?h@R53Yu)11Q zofXt6XCpEFazL4h%3q$`(SEbqaPMZK_ikFDT{~%&#*RK74=T~9{LF9Y4ZYB)2O zT3cSvR)f@`dD(Z*%(!!m5NiD9ftHtul%Dd`N$lhUVnOXFCwb88AcTWx9tnV5P zebaClPmOX=?4{;rHyc&ms8%%b`m`vJ*O%`WtP1RjY@95U-6OZU{kXduk z*wNKUP??yJznZYpZAO^4v5q7kuh%ob?hU%LAy(6N^_{A_>`_DC+t!-^6WCz83YfDgI23Js#~qX4zM3X9j(GHeYIBcX8XC_oZX|b zS!Va{8=;p0+ee@!?YW_?U{IO6CLhXnzjKo57`ICXyOrCE+|KRNw)D}EKd4;0CO_3I z>2^qaF3_RfE`K#|XFbh0*y|Xj+e?~;ZtOSRedWK+QaTV&mag^c22MT2NZg(e&R+;9 z?K=FazUMHZ$CB^i}?b|Eu|NhRaGrzJzR(loE@69LVe;N>p}CM>#(@Vk)I{;$Aka^ z2q1s}0tg_000IagfB*uX5}1}N)jj2>3m5C=(@nlX-&fQGsZl+SxnQSVZ z$_$C)k|Tdf;*SXd1Q0*~0R#|0009ILKmY**&X2%|FB)>*88|-we<4L)oSy^aNDx2( z0R#|0009ILKmY**5I~?U;PXi#uK%|^m_q;o1Q0*~0R#|0009ILK;Rq+bX@;8zyCiF z8I~g7h&v_(5I_I{1Q0*~0R#|0009Kfzrdk?AXtv(-p=O}m5QECCFbIlOfDJEWM`}K z+-zbio=$CLk`L$7S$%WM`2>A=tx(d$&-RrzwDnbOsg%EWM_ZgaFU-UuLo00IagfB*srAb#s0(}3!e=8ypKmY**5I_I{ z1Q0*~0R#~6ssPvjy_z}djQ|1&Abq@q|Mzc2BmxK^fB*srAb None: - load_dotenv() - self.OPENAI_TEMPERATURE = config.openai_temperature - self.OPENAI_API_KEY = config.openai_key + self.OPENAI_TEMPERATURE = globalConfig.openai_temperature + self.OPENAI_API_KEY = globalConfig.openai_key @classmethod async def create_memory( @@ -194,7 +191,7 @@ class Memory: user_id, str(memory_id), index_name=memory_label, - db_type=config.vectordb, + db_type=globalConfig.vectordb, **kwargs, ) diff --git a/docker-compose.yml b/docker-compose.yml index 80e6826cd..927198116 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ services: environment: - HOST=0.0.0.0 - ENVIRONMENT=local + - PYTHONPATH=. profiles: ["exclude-from-up"] ports: - 8000:8000 @@ -32,8 +33,9 @@ services: - 80:80 - 50051:50051 - 5678:5678 + - 5432:5432 depends_on: - - postgres + # - postgres - neo4j deploy: resources: @@ -41,18 +43,18 @@ services: cpus: "4.0" memory: 8GB - postgres: - image: postgres - container_name: postgres - environment: - - POSTGRES_HOST_AUTH_METHOD=trust - - POSTGRES_USER=bla - - POSTGRES_PASSWORD=bla - - POSTGRES_DB=bubu - networks: - - cognee_backend - ports: - - "5432:5432" + # postgres: + # image: postgres + # container_name: postgres + # environment: + # - POSTGRES_HOST_AUTH_METHOD=trust + # - POSTGRES_USER=bla + # - POSTGRES_PASSWORD=bla + # - POSTGRES_DB=bubu + # networks: + # - cognee_backend + # ports: + # - "5432:5432" networks: cognee_backend: diff --git a/entrypoint.sh b/entrypoint.sh index e3fb504d9..fa55a3d54 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -17,11 +17,11 @@ else echo '"local" environment is active, skipping fetch_secret.py' fi -echo "Running create_database.py" +echo "Creating database..." -python cognitive_architecture/database/create_database.py +python cognitive_architecture/setup_database.py if [ $? -ne 0 ]; then - echo "Error: create_database.py failed" + echo "Error: setup_database.py failed" exit 1 fi diff --git a/main.py b/main.py index 06da08033..2a7b3b6e3 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,6 @@ from pydantic import BaseModel, Field from cognitive_architecture.database.graphdb.graph import Neo4jGraphDB from cognitive_architecture.database.relationaldb.models.memory import MemoryModel import os -from dotenv import load_dotenv from cognitive_architecture.database.relationaldb.database_crud import ( session_scope, update_entity_graph_summary, @@ -38,12 +37,12 @@ aclient = instructor.patch(OpenAI()) DEFAULT_PRESET = "promethai_chat" preset_options = [DEFAULT_PRESET] PROMETHAI_DIR = os.path.join(os.path.expanduser("~"), ".") -load_dotenv() OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") from cognitive_architecture.config import Config config = Config() config.load() + from cognitive_architecture.utils import get_document_names from sqlalchemy.orm import selectinload, joinedload, contains_eager import logging diff --git a/pyproject.toml b/pyproject.toml index 2fa803d95..b486bf14c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ debugpy = "^1.8.0" lancedb = "^0.5.5" pyarrow = "^15.0.0" pylint = "^3.0.3" +aiosqlite = "^0.19.0" [build-system] requires = ["poetry-core"]