feat: S3 objects are now typed in Python and TS SDK (#2878)

* feat: S3 objects are now typed in Python and TS SDK

* fix ts and python SDK after testing

* cleanup

python-client/tests/wmill_client_test.py

@@ -15,17 +15,6 @@ class TestStringMethods(unittest.TestCase):
         os.environ["BASE_INTERNAL_URL"] = self._host
 
     def test_duckdb_connection_settings(self):
-        s3_resource = {
-            "port": 9000,
-            "bucket": "windmill",
-            "region": "fr-paris",
-            "useSSL": False,
-            "endPoint": "localhost:9000",
-            "accessKey": "ACCESS_KEY",
-            "pathStyle": True,
-            "secretKey": "SECRET_KEY",
-        }
-
         settings = wmill.duckdb_connection_settings(self._resource_path)
         self.assertIsNotNone(settings)
 
@@ -38,31 +27,36 @@ SET s3_use_ssl=0;
 SET s3_access_key_id='IeuKPSYLKTO2h9CWfCVR';
 SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
 """
-        self.assertEqual(settings, {"connection_settings_str": expected_settings_str})
+
+        self.assertEqual(settings["connection_settings_str"], expected_settings_str)
+        self.assertEqual(settings.connection_settings_str, expected_settings_str)
 
         settings = wmill.polars_connection_settings(self._resource_path)
         print(settings)
 
     def test_polars_connection_settings(self):
         settings = wmill.polars_connection_settings(self._resource_path)
-        expected_settings = {
-            "s3fs_args": {
-                "endpoint_url": "http://localhost:9000",
-                "key": "IeuKPSYLKTO2h9CWfCVR",
-                "secret": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
-                "use_ssl": False,
-                "cache_regions": False,
-                "client_kwargs": {"region_name": "fr-paris"},
-            },
-            "polars_cloud_options": {
-                "aws_endpoint_url": "http://localhost:9000",
-                "aws_access_key_id": "IeuKPSYLKTO2h9CWfCVR",
-                "aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
-                "aws_region": "fr-paris",
-                "aws_allow_http": True,
-            },
+        s3fs_args_expected = {
+            "endpoint_url": "http://localhost:9000",
+            "key": "IeuKPSYLKTO2h9CWfCVR",
+            "secret": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
+            "use_ssl": False,
+            "cache_regions": False,
+            "client_kwargs": {"region_name": "fr-paris"},
         }
-        self.assertEqual(settings, expected_settings)
+        polars_cloud_options_expected = {
+            "aws_endpoint_url": "http://localhost:9000",
+            "aws_access_key_id": "IeuKPSYLKTO2h9CWfCVR",
+            "aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
+            "aws_region": "fr-paris",
+            "aws_allow_http": True,
+        }
+        self.assertEqual(settings["s3fs_args"], s3fs_args_expected)
+        self.assertEqual(settings.s3fs_args, s3fs_args_expected)
+        self.assertEqual(
+            settings["polars_cloud_options"], polars_cloud_options_expected
+        )
+        self.assertEqual(settings.polars_cloud_options, polars_cloud_options_expected)
 
     def test_boto3_connection_settings(self):
         settings = wmill.boto3_connection_settings(self._resource_path)
@@ -74,6 +68,8 @@ SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
             "aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
         }
         self.assertEqual(settings, expected_settings)
+        self.assertEqual(settings["endpoint_url"], "http://localhost:9000")
+        self.assertEqual(settings.endpoint_url, "http://localhost:9000")
 
 
 if __name__ == "__main__":

python-client/wmill/wmill/__init__.py

@@ -1 +1,2 @@
 from .client import *
+from .s3_types import *

python-client/wmill/wmill/client.py

@@ -13,6 +13,8 @@ from typing import Dict, Any, Union, Literal
 
 import httpx
 
+from .s3_types import Boto3ConnectionSettings, DuckDbConnectionSettings, PolarsConnectionSettings
+
 _client: "Windmill | None" = None
 
 logger = logging.getLogger("windmill_client")
@@ -318,16 +320,17 @@ class Windmill:
         self,
         s3_resource_path: str = "",
         none_if_undefined: bool = False,
-    ) -> Union[str, None]:
+    ) -> DuckDbConnectionSettings | None:
         """
         Convenient helpers that takes an S3 resource as input and returns the settings necessary to
         initiate an S3 connection from DuckDB
         """
         try:
-            return self.post(
+            raw_obj = self.post(
                 f"/w/{self.workspace}/job_helpers/v2/duckdb_connection_settings",
                 json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
             ).json()
+            return DuckDbConnectionSettings(raw_obj)
         except JSONDecodeError as e:
             if none_if_undefined:
                 return None
@@ -337,16 +340,17 @@ class Windmill:
         self,
         s3_resource_path: str = "",
         none_if_undefined: bool = False,
-    ) -> Any:
+    ) -> PolarsConnectionSettings | None:
         """
         Convenient helpers that takes an S3 resource as input and returns the settings necessary to
         initiate an S3 connection from Polars
         """
         try:
-            return self.post(
+            raw_obj = self.post(
                 f"/w/{self.workspace}/job_helpers/v2/polars_connection_settings",
                 json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
             ).json()
+            return PolarsConnectionSettings(raw_obj)
         except JSONDecodeError as e:
             if none_if_undefined:
                 return None
@@ -356,16 +360,28 @@ class Windmill:
         self,
         s3_resource_path: str = "",
         none_if_undefined: bool = False,
-    ) -> Any:
+    ) -> Boto3ConnectionSettings | None:
         """
         Convenient helpers that takes an S3 resource as input and returns the settings necessary to
         initiate an S3 connection using boto3
         """
         try:
-            return self.post(
-                f"/w/{self.workspace}/job_helpers/v2/boto3_connection_settings",
+            s3_resource = self.post(
+                f"/w/{self.workspace}/job_helpers/v2/s3_resource_info",
                 json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
             ).json()
+            endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://"
+            boto3_settings = Boto3ConnectionSettings(
+                {
+                    "endpoint_url": "{}{}".format(endpoint_url_prefix, s3_resource["endPoint"]),
+                    "region_name": s3_resource["region"],
+                    "use_ssl": s3_resource["useSSL"],
+                    "aws_access_key_id": s3_resource["accessKey"],
+                    "aws_secret_access_key": s3_resource["secretKey"],
+                    # no need for path_style here as boto3 is clever enough to determine which one to use
+                }
+            )
+            return boto3_settings
         except JSONDecodeError as e:
             if none_if_undefined:
                 return None
@@ -574,7 +590,9 @@ def get_result(job_id: str, assert_result_is_not_none=True) -> Dict[str, Any]:
 
 
 @init_global_client
-def duckdb_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Union[str, None]:
+def duckdb_connection_settings(
+    s3_resource_path: str = "", none_if_undefined: bool = False
+) -> DuckDbConnectionSettings | None:
     """
     Convenient helpers that takes an S3 resource as input and returns the settings necessary to
     initiate an S3 connection from DuckDB
@@ -585,7 +603,9 @@ def duckdb_connection_settings(s3_resource_path: str = "", none_if_undefined: bo
 
 
 @init_global_client
-def polars_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Any:
+def polars_connection_settings(
+    s3_resource_path: str = "", none_if_undefined: bool = False
+) -> PolarsConnectionSettings | None:
     """
     Convenient helpers that takes an S3 resource as input and returns the settings necessary to
     initiate an S3 connection from Polars
@@ -594,7 +614,9 @@ def polars_connection_settings(s3_resource_path: str = "", none_if_undefined: bo
 
 
 @init_global_client
-def boto3_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Any:
+def boto3_connection_settings(
+    s3_resource_path: str = "", none_if_undefined: bool = False
+) -> Boto3ConnectionSettings | None:
     """
     Convenient helpers that takes an S3 resource as input and returns the settings necessary to
     initiate an S3 connection using boto3

python-client/wmill/wmill/s3_types.py

@@ -0,0 +1,61 @@
+class S3Object(dict):
+    s3: str
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class S3FsClientKwargs(dict):
+    region_name: str
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class S3FsArgs(dict):
+    endpoint_url: str
+    key: str
+    secret: str
+    use_ssl: bool
+    cache_regions: bool
+    client_kwargs: S3FsClientKwargs
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class PolarsCloudOptions(dict):
+    aws_endpoint_url: str
+    aws_access_key_id: str
+    aws_secret_access_key: str
+    aws_region: bool
+    aws_allow_http: bool
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class PolarsConnectionSettings(dict):
+    s3fs_args: S3FsArgs
+    polars_cloud_options: PolarsCloudOptions
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class Boto3ConnectionSettings(dict):
+    endpoint_url: str
+    region_name: str
+    use_ssl: bool
+    aws_access_key_id: str
+    aws_secret_access_key: str
+
+    def __getattr__(self, attr):
+        return self[attr]
+
+
+class DuckDbConnectionSettings(dict):
+    connection_settings_str: str
+
+    def __getattr__(self, attr):
+        return self[attr]