Merge branch 'dev' into loader_separation

.github/workflows/test_s3_file_storage.yml

@@ -23,8 +23,11 @@ jobs:
         env:
           STORAGE_BACKEND: s3
           ENABLE_BACKEND_ACCESS_CONTROL: True
+          AWS_REGION: eu-west-1
+          AWS_ENDPOINT_URL: https://s3-eu-west-1.amazonaws.com
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_DEV_USER_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_DEV_USER_SECRET_KEY }}
+          STORAGE_BUCKET_NAME: github-runner-cognee-tests
           LLM_MODEL: ${{ secrets.LLM_MODEL }}
           LLM_ENDPOINT: ${{ secrets.LLM_ENDPOINT }}
           LLM_API_KEY: ${{ secrets.LLM_API_KEY }}

cognee/infrastructure/files/storage/S3FileStorage.py

@@ -26,7 +26,8 @@ class S3FileStorage(Storage):
                 key=s3_config.aws_access_key_id,
                 secret=s3_config.aws_secret_access_key,
                 anon=False,
-                endpoint_url="https://s3-eu-west-1.amazonaws.com",
+                endpoint_url=s3_config.aws_endpoint_url,
+                client_kwargs={"region_name": s3_config.aws_region},
             )
         else:
             raise ValueError("S3 credentials are not set in the configuration.")

cognee/infrastructure/files/storage/s3_config.py

@@ -4,6 +4,8 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 class S3Config(BaseSettings):
+    aws_region: Optional[str] = None
+    aws_endpoint_url: Optional[str] = None
     aws_access_key_id: Optional[str] = None
     aws_secret_access_key: Optional[str] = None
     model_config = SettingsConfigDict(env_file=".env", extra="allow")

cognee/tests/test_s3_file_storage.py

@@ -13,10 +13,11 @@ logger = get_logger()
 
 
 async def main():
+    bucket_name = os.getenv("STORAGE_BUCKET_NAME")
     test_run_id = uuid4()
-    data_directory_path = f"s3://cognee-storage-dev/{test_run_id}/data"
+    data_directory_path = f"s3://{bucket_name}/{test_run_id}/data"
     cognee.config.data_root_directory(data_directory_path)
-    cognee_directory_path = f"s3://cognee-storage-dev/{test_run_id}/system"
+    cognee_directory_path = f"s3://{bucket_name}/{test_run_id}/system"
     cognee.config.system_root_directory(cognee_directory_path)
 
     await cognee.prune.prune_data()

poetry.lock

@@ -12230,4 +12230,4 @@ weaviate = ["weaviate-client"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<=3.13"
-content-hash = "4310fa7c1541538b5c8cf587443cb68ece4e7b0dd1ffee7b6681f53360f0a5da"
+content-hash = "5e2bfdd518c962648983340664f3915e013f6fd73da055f107011c9c293fab5b"

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognee"
-version = "0.2.1-dev5"
+version = "0.2.1-dev6"
 description = "Cognee - is a library for enriching LLM context with a semantic layer for better understanding and reasoning."
 authors = [
     { name = "Vasilije Markovic" },
@@ -25,7 +25,7 @@ dependencies = [
     "pydantic-settings>=2.2.1,<3",
     "typing_extensions>=4.12.2,<5.0.0",
     "nltk>=3.9.1,<4.0.0",
-    "numpy>=1.26.4, <=2.3.1",
+    "numpy>=1.26.4, <=4.0.0",
     "pandas>=2.2.2,<3.0.0",
     # Note: New s3fs and boto3 versions don't work well together
     # Always use comaptible fixed versions of these two dependencies

uv.lock

@@ -878,7 +878,7 @@ wheels = [
 
 [[package]]
 name = "cognee"
-version = "0.2.1.dev5"
+version = "0.2.1.dev6"
 source = { editable = "." }
 dependencies = [
     { name = "aiofiles" },
@@ -1086,7 +1086,7 @@ requires-dist = [
     { name = "nltk", specifier = ">=3.9.1,<4.0.0" },
     { name = "notebook", marker = "extra == 'dev'", specifier = ">=7.1.0,<8" },
     { name = "notebook", marker = "extra == 'notebook'", specifier = ">=7.1.0,<8" },
-    { name = "numpy", specifier = ">=1.26.4,<=2.3.1" },
+    { name = "numpy", specifier = ">=1.26.4,<=4.0.0" },
     { name = "onnxruntime", specifier = ">=1.0.0,<2.0.0" },
     { name = "openai", specifier = ">=1.80.1,<2" },
     { name = "pandas", specifier = ">=2.2.2,<3.0.0" },