gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: remove obsolete code

Browse source
This commit is contained in:
Boris Arzentar 2024-03-13 10:19:03 +01:00
parent 9448e36201
commit d871a7b3d1
6 changed files with 203 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.dlt/config.toml
View file

@ -1,6 +1,6 @@
# put your configuration values here
[runtime]
log_level="WARNING" # the system log level of dlt
log_level = "WARNING" # the system log level of dlt
# use the dlthub_telemetry setting to enable/disable anonymous usage data reporting, see https://dlthub.com/docs/telemetry
dlthub_telemetry = false

5
.dlt/secrets.toml
View file

@ -1,5 +0,0 @@
# put your secret values and credentials here. do not share this file and do not push it to github
[destination.qdrant.credentials]
location = "https://cff4594b-c2de-4fcf-8365-4c1d3a1c1429.us-east4-0.gcp.cloud.qdrant.io:6333"
api_key = "K5BKjVGR8Qn4pVMk9nPFNTqITu3QVGR1O8qlDDH6kk52HUwB4lRjjw"

337
cognee.ipynb
View file

File diff suppressed because one or more lines are too long

2
cognitive_architecture/api/v1/add/add.py
View file

@ -41,7 +41,7 @@ async def add(file_paths: Union[str, List[str]], dataset_name: str = None):
if dataset_name is not None and not key.startswith(dataset_name):
continue
results.append(add_dlt(datasets[key], dataset_name = key))
results.append(add(datasets[key], dataset_name = key))
return await asyncio.gather(*results)

104
cognitive_architecture/infrastructure/llm/openai/adapter.py
View file

@ -15,102 +15,6 @@ class OpenAIAdapter(LLMInterface):
openai.api_key = api_key
self.aclient = instructor.apatch(AsyncOpenAI())
self.model = model
# OPENAI_API_KEY = config.openai_key
# @staticmethod
# def retry_with_exponential_backoff(
# func,
# initial_delay: float = 1,
# exponential_base: float = 2,
# jitter: bool = True,
# max_retries: int = 20,
# errors: tuple = (openai.RateLimitError,),
# ):
# """Retry a function with exponential backoff."""
# def wrapper(*args, **kwargs):
# """Wrapper for sync functions."""
# # Initialize variables
# num_retries = 0
# delay = initial_delay
# # Loop until a successful response or max_retries is hit or an exception is raised
# while True:
# try:
# return func(*args, **kwargs)
# # Retry on specified errors
# except errors:
# # Increment retries
# num_retries += 1
# # Check if max retries has been reached
# if num_retries > max_retries:
# raise Exception(
# f"Maximum number of retries ({max_retries}) exceeded."
# )
# # Increment the delay
# delay *= exponential_base * (1 + jitter * random.random())
# # Sleep for the delay
# time.sleep(delay)
# # Raise exceptions for any errors not specified
# except Exception as e:
# raise e
# return wrapper
# @staticmethod
# async def aretry_with_exponential_backoff(
# func,
# initial_delay: float = 1,
# exponential_base: float = 2,
# jitter: bool = True,
# max_retries: int = 20,
# errors: tuple = (openai.RateLimitError,),
# ):
# """Retry a function with exponential backoff."""
# async def wrapper(*args, **kwargs):
# """Wrapper for async functions.
# :param args: list
# :param kwargs: dict"""
# # Initialize variables
# num_retries = 0
# delay = initial_delay
# # Loop until a successful response or max_retries is hit or an exception is raised
# while True:
# try:
# return await func(*args, **kwargs)
# # Retry on specified errors
# except errors as e:
# print(f"acreate (backoff): caught error: {e}")
# # Increment retries
# num_retries += 1
# # Check if max retries has been reached
# if num_retries > max_retries:
# raise Exception(
# f"Maximum number of retries ({max_retries}) exceeded."
# )
# # Increment the delay
# delay *= exponential_base * (1 + jitter * random.random())
# # Sleep for the delay
# await asyncio.sleep(delay)
# # Raise exceptions for any errors not specified
# except Exception as e:
# raise e
# return wrapper
@retry(stop = stop_after_attempt(5))
def completions_with_backoff(self, **kwargs):
@ -127,11 +31,6 @@ class OpenAIAdapter(LLMInterface):
async def acreate_embedding_with_backoff(self, input: List[str], model: str = "text-embedding-ada-002"):
"""Wrapper around Embedding.acreate w/ backoff"""
# client = openai.AsyncOpenAI(
# # This is the default and can be omitted
# api_key=os.environ.get("OPENAI_API_KEY"),
# )
return await self.aclient.embeddings.create(input=input, model=model)
async def async_get_embedding_with_backoff(self, text, model="text-embedding-ada-002"):
@ -139,7 +38,6 @@ class OpenAIAdapter(LLMInterface):
It specifies defaults + handles rate-limiting + is async"""
text = text.replace("\n", " ")
response = await self.aclient.embeddings.create(input =text, model= model)
# response = await self.acreate_embedding_with_backoff(input=text, model=model)
embedding = response.data[0].embedding
return embedding
@ -162,7 +60,7 @@ class OpenAIAdapter(LLMInterface):
async def async_get_batch_embeddings_with_backoff(self, texts: List[str], models: List[str]):
"""To get multiple text embeddings in parallel, import/call this function
It specifies defaults + handles rate-limiting + is async"""
# Create a generator of coroutines
# Collect all coroutines
coroutines = (self.async_get_embedding_with_backoff(text, model)
for text, model in zip(texts, models))

3
cognitive_architecture/infrastructure/llm/openai/openai_tools.py
View file

@ -6,11 +6,11 @@ from tenacity import retry, stop_after_attempt
import openai
HOST = os.getenv("OPENAI_API_BASE")
HOST_TYPE = os.getenv("BACKEND_TYPE") # default None == ChatCompletion
if HOST is not None:
openai.api_base = HOST
@retry(stop = stop_after_attempt(5))
def completions_with_backoff(**kwargs):
"""Wrapper around ChatCompletion.create w/ backoff"""
@ -62,7 +62,6 @@ def get_embedding_with_backoff(text:str, model:str="text-embedding-ada-002"):
return embedding
async def async_get_batch_embeddings_with_backoff(texts: List[str], models: List[str]) :
"""To get multiple text embeddings in parallel, import/call this function
It specifies defaults + handles rate-limiting + is async"""