feat: add s3-compatible storage boxes

common/data_source/blob_connector.py

@@ -87,6 +87,13 @@ class BlobStorageConnector(LoadConnector, PollConnector):
             ):
                 raise ConnectorMissingCredentialError("Oracle Cloud Infrastructure")
 
+        elif self.bucket_type == BlobType.S3_COMPATIBLE:
+            if not all(
+                credentials.get(key)
+                for key in ["endpoint_url", "aws_access_key_id", "aws_secret_access_key"]
+            ):
+                raise ConnectorMissingCredentialError("S3 Compatible Storage")
+
         else:
             raise ValueError(f"Unsupported bucket type: {self.bucket_type}")
 

common/data_source/config.py

@@ -32,6 +32,7 @@ class BlobType(str, Enum):
     R2 = "r2"
     GOOGLE_CLOUD_STORAGE = "google_cloud_storage"
     OCI_STORAGE = "oci_storage"
+    S3_COMPATIBLE = "s3_compatible"
 
 
 class DocumentSource(str, Enum):
@@ -47,6 +48,7 @@ class DocumentSource(str, Enum):
     GOOGLE_DRIVE = "google_drive"
     GMAIL = "gmail"
     DISCORD = "discord"
+    S3_COMPATIBLE = "s3_compatible"
 
 
 class FileOrigin(str, Enum):

common/data_source/utils.py

@@ -311,6 +311,13 @@ def create_s3_client(bucket_type: BlobType, credentials: dict[str, Any], europea
             aws_secret_access_key=credentials["secret_access_key"],
             region_name=credentials["region"],
         )
+    elif bucket_type == BlobType.S3_COMPATIBLE:
+        return boto3.client(
+            "s3",
+            endpoint_url=credentials["endpoint_url"],
+            aws_access_key_id=credentials["aws_access_key_id"],
+            aws_secret_access_key=credentials["aws_secret_access_key"],
+        )    
 
     else:
         raise ValueError(f"Unsupported bucket type: {bucket_type}")

web/src/locales/en.ts

@@ -705,6 +705,8 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
         'The base URL of your Confluence instance (e.g., https://your-domain.atlassian.net/wiki)',
       s3PrefixTip: `Specify the folder path within your S3 bucket to fetch files from. 
 Example: general/v2/`,
+      S3CompatibleEndpointUrlTip: `Required for S3 compatible Storage Box. Specify the S3-compatible endpoint URL.
+Example: https://fsn1.your-objectstorage.com`,
       addDataSourceModalTital: 'Create your {{name}} connector',
       deleteSourceModalTitle: 'Delete data source',
       deleteSourceModalContent: `

web/src/pages/user-setting/data-source/contant.tsx

@@ -105,9 +105,18 @@ export const DataSourceFormFields = {
         { label: 'R2', value: 'r2' },
         { label: 'Google Cloud Storage', value: 'google_cloud_storage' },
         { label: 'OCI Storage', value: 'oci_storage' },
+        { label: 'S3 Compatible', value: 's3_compatible' },
       ],
       required: true,
     },
+    {
+      label: 'Endpoint URL',
+      name: 'config.credentials.endpoint_url',
+      type: FormFieldType.Text,
+      required: false,
+      placeholder: 'https://fsn1.your-objectstorage.com',
+      tooltip: t('setting.S3CompatibleEndpointUrlTip'),
+    },
     {
       label: 'Prefix',
       name: 'config.prefix',
@@ -483,6 +492,7 @@ export const DataSourceFormDefaultValues = {
       credentials: {
         aws_access_key_id: '',
         aws_secret_access_key: '',
+        endpoint_url: '',
       },
     },
   },