fix: remove obsolete code

.dlt/config.toml

@@ -1,6 +1,6 @@
 # put your configuration values here
 
 [runtime]
-log_level="WARNING"  # the system log level of dlt
+log_level = "WARNING"  # the system log level of dlt
 # use the dlthub_telemetry setting to enable/disable anonymous usage data reporting, see https://dlthub.com/docs/telemetry
 dlthub_telemetry = false

.dlt/secrets.toml

@@ -1,5 +0,0 @@
-# put your secret values and credentials here. do not share this file and do not push it to github
-
-[destination.qdrant.credentials]
-location = "https://cff4594b-c2de-4fcf-8365-4c1d3a1c1429.us-east4-0.gcp.cloud.qdrant.io:6333"
-api_key = "K5BKjVGR8Qn4pVMk9nPFNTqITu3QVGR1O8qlDDH6kk52HUwB4lRjjw"

cognitive_architecture/api/v1/add/add.py

@@ -41,7 +41,7 @@ async def add(file_paths: Union[str, List[str]], dataset_name: str = None):
             if dataset_name is not None and not key.startswith(dataset_name):
                 continue
 
-            results.append(add_dlt(datasets[key], dataset_name = key))
+            results.append(add(datasets[key], dataset_name = key))
 
         return await asyncio.gather(*results)
 

cognitive_architecture/infrastructure/llm/openai/adapter.py

@@ -15,102 +15,6 @@ class OpenAIAdapter(LLMInterface):
         openai.api_key = api_key
         self.aclient = instructor.apatch(AsyncOpenAI())
         self.model = model
-        # OPENAI_API_KEY = config.openai_key
-
-    # @staticmethod
-    # def retry_with_exponential_backoff(
-    #         func,
-    #         initial_delay: float = 1,
-    #         exponential_base: float = 2,
-    #         jitter: bool = True,
-    #         max_retries: int = 20,
-    #         errors: tuple = (openai.RateLimitError,),
-    # ):
-    #     """Retry a function with exponential backoff."""
-
-    #     def wrapper(*args, **kwargs):
-    #         """Wrapper for sync functions."""
-    #         # Initialize variables
-    #         num_retries = 0
-    #         delay = initial_delay
-
-    #         # Loop until a successful response or max_retries is hit or an exception is raised
-    #         while True:
-    #             try:
-    #                 return func(*args, **kwargs)
-
-    #             # Retry on specified errors
-    #             except errors:
-    #                 # Increment retries
-    #                 num_retries += 1
-
-    #                 # Check if max retries has been reached
-    #                 if num_retries > max_retries:
-    #                     raise Exception(
-    #                         f"Maximum number of retries ({max_retries}) exceeded."
-    #                     )
-
-    #                 # Increment the delay
-    #                 delay *= exponential_base * (1 + jitter * random.random())
-
-    #                 # Sleep for the delay
-    #                 time.sleep(delay)
-
-    #             # Raise exceptions for any errors not specified
-    #             except Exception as e:
-    #                 raise e
-
-    #     return wrapper
-
-
-    # @staticmethod
-    # async def aretry_with_exponential_backoff(
-    #         func,
-    #         initial_delay: float = 1,
-    #         exponential_base: float = 2,
-    #         jitter: bool = True,
-    #         max_retries: int = 20,
-    #         errors: tuple = (openai.RateLimitError,),
-    # ):
-    #     """Retry a function with exponential backoff."""
-
-    #     async def wrapper(*args, **kwargs):
-    #         """Wrapper for async functions.
-    #         :param args: list
-    #         :param kwargs: dict"""
-    #         # Initialize variables
-    #         num_retries = 0
-    #         delay = initial_delay
-
-    #         # Loop until a successful response or max_retries is hit or an exception is raised
-    #         while True:
-    #             try:
-    #                 return await func(*args, **kwargs)
-
-    #             # Retry on specified errors
-    #             except errors as e:
-    #                 print(f"acreate (backoff): caught error: {e}")
-    #                 # Increment retries
-    #                 num_retries += 1
-
-    #                 # Check if max retries has been reached
-    #                 if num_retries > max_retries:
-    #                     raise Exception(
-    #                         f"Maximum number of retries ({max_retries}) exceeded."
-    #                     )
-
-    #                 # Increment the delay
-    #                 delay *= exponential_base * (1 + jitter * random.random())
-
-    #                 # Sleep for the delay
-    #                 await asyncio.sleep(delay)
-
-    #             # Raise exceptions for any errors not specified
-    #             except Exception as e:
-    #                 raise e
-
-    #     return wrapper
-
 
     @retry(stop = stop_after_attempt(5))
     def completions_with_backoff(self, **kwargs):
@@ -127,11 +31,6 @@ class OpenAIAdapter(LLMInterface):
     async def acreate_embedding_with_backoff(self, input: List[str], model: str = "text-embedding-ada-002"):
         """Wrapper around Embedding.acreate w/ backoff"""
 
-        # client = openai.AsyncOpenAI(
-        #     # This is the default and can be omitted
-        #     api_key=os.environ.get("OPENAI_API_KEY"),
-        # )
-
         return await self.aclient.embeddings.create(input=input, model=model)
 
     async def async_get_embedding_with_backoff(self, text, model="text-embedding-ada-002"):
@@ -139,7 +38,6 @@ class OpenAIAdapter(LLMInterface):
         It specifies defaults + handles rate-limiting + is async"""
         text = text.replace("\n", " ")
         response = await self.aclient.embeddings.create(input =text, model= model)
-        # response = await self.acreate_embedding_with_backoff(input=text, model=model)
         embedding = response.data[0].embedding
         return embedding
 
@@ -162,7 +60,7 @@ class OpenAIAdapter(LLMInterface):
     async def async_get_batch_embeddings_with_backoff(self, texts: List[str], models: List[str]):
         """To get multiple text embeddings in parallel, import/call this function
         It specifies defaults + handles rate-limiting + is async"""
-        # Create a generator of coroutines
+        # Collect all coroutines
         coroutines = (self.async_get_embedding_with_backoff(text, model)
             for text, model in zip(texts, models))
 

cognitive_architecture/infrastructure/llm/openai/openai_tools.py

@@ -6,11 +6,11 @@ from tenacity import retry, stop_after_attempt
 import openai
 
 HOST = os.getenv("OPENAI_API_BASE")
-HOST_TYPE = os.getenv("BACKEND_TYPE")  # default None == ChatCompletion
 
 if HOST is not None:
     openai.api_base = HOST
 
+
 @retry(stop = stop_after_attempt(5))
 def completions_with_backoff(**kwargs):
     """Wrapper around ChatCompletion.create w/ backoff"""
@@ -62,7 +62,6 @@ def get_embedding_with_backoff(text:str, model:str="text-embedding-ada-002"):
     return embedding
 
 
-
 async def async_get_batch_embeddings_with_backoff(texts: List[str], models: List[str]) :
     """To get multiple text embeddings in parallel, import/call this function
     It specifies defaults + handles rate-limiting + is async"""