Fixes to database manager
This commit is contained in:
parent
86555460d2
commit
bf9c80653e
5 changed files with 34 additions and 26 deletions
|
|
@ -41,6 +41,8 @@ RUN apt-get update -q && \
|
||||||
/var/tmp/*
|
/var/tmp/*
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
# Set the PYTHONPATH environment variable to include the /app directory
|
||||||
|
ENV PYTHONPATH=/app
|
||||||
|
|
||||||
COPY cognitive_architecture/ /app/cognitive_architecture
|
COPY cognitive_architecture/ /app/cognitive_architecture
|
||||||
COPY main.py /app
|
COPY main.py /app
|
||||||
|
|
|
||||||
|
|
@ -36,6 +36,7 @@ class Config:
|
||||||
db_user: str = os.getenv("DB_USER", "cognee")
|
db_user: str = os.getenv("DB_USER", "cognee")
|
||||||
db_password: str = os.getenv("DB_PASSWORD", "cognee")
|
db_password: str = os.getenv("DB_PASSWORD", "cognee")
|
||||||
sqlalchemy_logging: bool = os.getenv("SQLALCHEMY_LOGGING", True)
|
sqlalchemy_logging: bool = os.getenv("SQLALCHEMY_LOGGING", True)
|
||||||
|
graph_name = os.getenv("GRAPH_NAME", "cognee_graph.pkl")
|
||||||
|
|
||||||
# Model parameters
|
# Model parameters
|
||||||
model: str = "gpt-4-1106-preview"
|
model: str = "gpt-4-1106-preview"
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,16 @@
|
||||||
import pickle
|
import pickle
|
||||||
|
from pathlib import Path
|
||||||
|
from cognitive_architecture.config import Config
|
||||||
import networkx as nx
|
import networkx as nx
|
||||||
|
config = Config()
|
||||||
|
config = config.load()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class NetworkXGraphDB:
|
class NetworkXGraphDB:
|
||||||
def __init__(self, filename="cognee_graph.pkl"):
|
"""A class to manage a graph database using NetworkX"""
|
||||||
|
graph_path = (Path(config.base_path) / config.graph_name).absolute()
|
||||||
|
def __init__(self, filename=graph_path):
|
||||||
self.filename = filename
|
self.filename = filename
|
||||||
try:
|
try:
|
||||||
self.graph = self.load_graph() # Attempt to load an existing graph
|
self.graph = self.load_graph() # Attempt to load an existing graph
|
||||||
|
|
|
||||||
|
|
@ -12,12 +12,11 @@ parent_dir = os.path.dirname(current_dir)
|
||||||
# Add the parent directory to sys.path
|
# Add the parent directory to sys.path
|
||||||
sys.path.insert(0, parent_dir)
|
sys.path.insert(0, parent_dir)
|
||||||
|
|
||||||
# API_ENABLED = os.environ.get("API_ENABLED", "False").lower() == "true"
|
|
||||||
|
|
||||||
environment = os.getenv("AWS_ENV", "dev")
|
environment = os.getenv("AWS_ENV", "dev")
|
||||||
|
|
||||||
|
|
||||||
def fetch_secret(secret_name, region_name, env_file_path):
|
def fetch_secret(secret_name:str, region_name:str, env_file_path:str):
|
||||||
|
"""Fetch the secret from AWS Secrets Manager and write it to the .env file."""
|
||||||
print("Initializing session")
|
print("Initializing session")
|
||||||
session = boto3.session.Session()
|
session = boto3.session.Session()
|
||||||
print("Session initialized")
|
print("Session initialized")
|
||||||
|
|
@ -28,20 +27,19 @@ def fetch_secret(secret_name, region_name, env_file_path):
|
||||||
response = client.get_secret_value(SecretId=secret_name)
|
response = client.get_secret_value(SecretId=secret_name)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error retrieving secret: {e}")
|
print(f"Error retrieving secret: {e}")
|
||||||
return None
|
return f"Error retrieving secret: {e}"
|
||||||
|
|
||||||
if "SecretString" in response:
|
if "SecretString" in response:
|
||||||
secret = response["SecretString"]
|
secret = response["SecretString"]
|
||||||
else:
|
else:
|
||||||
secret = response["SecretBinary"]
|
secret = response["SecretBinary"]
|
||||||
|
|
||||||
|
with open(env_file_path, "w") as env_file:
|
||||||
|
env_file.write(secret)
|
||||||
|
print("Secrets are added to the .env file.")
|
||||||
|
|
||||||
if os.path.exists(env_file_path):
|
if os.path.exists(env_file_path):
|
||||||
print(f"The .env file is located at: {env_file_path}")
|
print(f"The .env file is located at: {env_file_path}")
|
||||||
|
|
||||||
with open(env_file_path, "w") as env_file:
|
|
||||||
env_file.write(secret)
|
|
||||||
print("Secrets are added to the .env file.")
|
|
||||||
|
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
print("The .env file is loaded.")
|
print("The .env file is loaded.")
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
""" This module contains utility functions for the cognitive architecture. """
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import string
|
import string
|
||||||
|
|
@ -13,7 +15,13 @@ from cognitive_architecture.database.relationaldb.models.user import User
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.future import select
|
from sqlalchemy.future import select
|
||||||
import logging
|
import logging
|
||||||
|
from cognitive_architecture.database.relationaldb.models.operation import Operation
|
||||||
|
from cognitive_architecture.database.relationaldb.database_crud import (
|
||||||
|
session_scope,
|
||||||
|
add_entity,
|
||||||
|
update_entity,
|
||||||
|
fetch_job_id,
|
||||||
|
)
|
||||||
|
|
||||||
class Node:
|
class Node:
|
||||||
def __init__(self, id, description, color):
|
def __init__(self, id, description, color):
|
||||||
|
|
@ -72,6 +80,7 @@ def get_document_names(doc_input):
|
||||||
|
|
||||||
|
|
||||||
def format_dict(d):
|
def format_dict(d):
|
||||||
|
""" Format a dictionary as a string."""
|
||||||
# Initialize an empty list to store formatted items
|
# Initialize an empty list to store formatted items
|
||||||
formatted_items = []
|
formatted_items = []
|
||||||
|
|
||||||
|
|
@ -93,6 +102,7 @@ def format_dict(d):
|
||||||
|
|
||||||
|
|
||||||
def append_uuid_to_variable_names(variable_mapping):
|
def append_uuid_to_variable_names(variable_mapping):
|
||||||
|
""" Append a UUID to the variable names to make them unique."""
|
||||||
unique_variable_mapping = {}
|
unique_variable_mapping = {}
|
||||||
for original_name in variable_mapping.values():
|
for original_name in variable_mapping.values():
|
||||||
unique_name = f"{original_name}_{uuid.uuid4().hex}"
|
unique_name = f"{original_name}_{uuid.uuid4().hex}"
|
||||||
|
|
@ -102,6 +112,7 @@ def append_uuid_to_variable_names(variable_mapping):
|
||||||
|
|
||||||
# Update the functions to use the unique variable names
|
# Update the functions to use the unique variable names
|
||||||
def create_node_variable_mapping(nodes):
|
def create_node_variable_mapping(nodes):
|
||||||
|
""" Create a mapping of node identifiers to unique variable names."""
|
||||||
mapping = {}
|
mapping = {}
|
||||||
for node in nodes:
|
for node in nodes:
|
||||||
variable_name = f"{node['category']}{node['id']}".lower()
|
variable_name = f"{node['category']}{node['id']}".lower()
|
||||||
|
|
@ -110,6 +121,7 @@ def create_node_variable_mapping(nodes):
|
||||||
|
|
||||||
|
|
||||||
def create_edge_variable_mapping(edges):
|
def create_edge_variable_mapping(edges):
|
||||||
|
""" Create a mapping of edge identifiers to unique variable names."""
|
||||||
mapping = {}
|
mapping = {}
|
||||||
for edge in edges:
|
for edge in edges:
|
||||||
# Construct a unique identifier for the edge
|
# Construct a unique identifier for the edge
|
||||||
|
|
@ -124,17 +136,10 @@ def generate_letter_uuid(length=8):
|
||||||
return "".join(random.choice(letters) for _ in range(length))
|
return "".join(random.choice(letters) for _ in range(length))
|
||||||
|
|
||||||
|
|
||||||
from cognitive_architecture.database.relationaldb.models.operation import Operation
|
|
||||||
from cognitive_architecture.database.relationaldb.database_crud import (
|
|
||||||
session_scope,
|
|
||||||
add_entity,
|
|
||||||
update_entity,
|
|
||||||
fetch_job_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async def get_vectordb_namespace(session: AsyncSession, user_id: str):
|
async def get_vectordb_namespace(session: AsyncSession, user_id: str):
|
||||||
|
""" Asynchronously retrieves the latest memory names for a given user."""
|
||||||
try:
|
try:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
select(MemoryModel.memory_name)
|
select(MemoryModel.memory_name)
|
||||||
|
|
@ -151,6 +156,7 @@ async def get_vectordb_namespace(session: AsyncSession, user_id: str):
|
||||||
|
|
||||||
|
|
||||||
async def get_vectordb_document_name(session: AsyncSession, user_id: str):
|
async def get_vectordb_document_name(session: AsyncSession, user_id: str):
|
||||||
|
""" Asynchronously retrieves the latest memory names for a given user."""
|
||||||
try:
|
try:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
select(DocsModel.doc_name)
|
select(DocsModel.doc_name)
|
||||||
|
|
@ -167,6 +173,7 @@ async def get_vectordb_document_name(session: AsyncSession, user_id: str):
|
||||||
|
|
||||||
|
|
||||||
async def get_model_id_name(session: AsyncSession, id: str):
|
async def get_model_id_name(session: AsyncSession, id: str):
|
||||||
|
""" Asynchronously retrieves the latest memory names for a given user."""
|
||||||
try:
|
try:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
select(MemoryModel.memory_name)
|
select(MemoryModel.memory_name)
|
||||||
|
|
@ -236,12 +243,6 @@ async def get_unsumarized_vector_db_namespace(session: AsyncSession, user_id: st
|
||||||
|
|
||||||
return memory_details, docs
|
return memory_details, docs
|
||||||
|
|
||||||
# except Exception as e:
|
|
||||||
# # Handle the exception as needed
|
|
||||||
# print(f"An error occurred: {e}")
|
|
||||||
# return None
|
|
||||||
|
|
||||||
|
|
||||||
async def get_memory_name_by_doc_id(session: AsyncSession, docs_id: str):
|
async def get_memory_name_by_doc_id(session: AsyncSession, docs_id: str):
|
||||||
"""
|
"""
|
||||||
Asynchronously retrieves memory names associated with a specific document ID.
|
Asynchronously retrieves memory names associated with a specific document ID.
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue