mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-18 16:02:10 +00:00
feat: S3 objects are now typed in Python and TS SDK (#2878)
* feat: S3 objects are now typed in Python and TS SDK * fix ts and python SDK after testing * cleanup
This commit is contained in:
committed by
GitHub
parent
ba036e0576
commit
fd55c3d8e3
File diff suppressed because it is too large
Load Diff
@@ -2231,8 +2231,7 @@ paths:
|
||||
|
||||
/w/{workspace}/resources/get_value_interpolated/{path}:
|
||||
get:
|
||||
summary:
|
||||
get resource interpolated (variables and resources are fully unrolled)
|
||||
summary: get resource interpolated (variables and resources are fully unrolled)
|
||||
operationId: getResourceValueInterpolated
|
||||
tags:
|
||||
- resource
|
||||
@@ -2920,8 +2919,7 @@ paths:
|
||||
schema:
|
||||
type: string
|
||||
- name: first_parent_hash
|
||||
description:
|
||||
mask to filter scripts whom first direct parent has exact hash
|
||||
description: mask to filter scripts whom first direct parent has exact hash
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
@@ -3116,8 +3114,7 @@ paths:
|
||||
|
||||
/workers/custom_tags:
|
||||
get:
|
||||
summary:
|
||||
get all instance custom tags (tags are used to dispatch jobs to
|
||||
summary: get all instance custom tags (tags are used to dispatch jobs to
|
||||
different worker groups)
|
||||
operationId: getCustomTags
|
||||
tags:
|
||||
@@ -3168,8 +3165,7 @@ paths:
|
||||
|
||||
/w/{workspace}/scripts/delete/h/{hash}:
|
||||
post:
|
||||
summary:
|
||||
delete script by hash (erase content but keep hash, require admin)
|
||||
summary: delete script by hash (erase content but keep hash, require admin)
|
||||
operationId: deleteScriptByHash
|
||||
tags:
|
||||
- script
|
||||
@@ -3408,8 +3404,7 @@ paths:
|
||||
type: string
|
||||
format: date-time
|
||||
- name: scheduled_in_secs
|
||||
description:
|
||||
schedule the script to execute in the number of seconds starting now
|
||||
description: schedule the script to execute in the number of seconds starting now
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
@@ -3417,8 +3412,7 @@ paths:
|
||||
- $ref: "#/components/parameters/WorkerTag"
|
||||
- $ref: "#/components/parameters/NewJobId"
|
||||
- name: invisible_to_owner
|
||||
description:
|
||||
make the run invisible to the the script owner (default false)
|
||||
description: make the run invisible to the the script owner (default false)
|
||||
in: query
|
||||
schema:
|
||||
type: boolean
|
||||
@@ -4451,8 +4445,7 @@ paths:
|
||||
type: string
|
||||
format: date-time
|
||||
- name: scheduled_in_secs
|
||||
description:
|
||||
schedule the script to execute in the number of seconds starting now
|
||||
description: schedule the script to execute in the number of seconds starting now
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
@@ -4461,8 +4454,7 @@ paths:
|
||||
- $ref: "#/components/parameters/NewJobId"
|
||||
- $ref: "#/components/parameters/IncludeHeader"
|
||||
- name: invisible_to_owner
|
||||
description:
|
||||
make the run invisible to the the flow owner (default false)
|
||||
description: make the run invisible to the the flow owner (default false)
|
||||
in: query
|
||||
schema:
|
||||
type: boolean
|
||||
@@ -4514,8 +4506,7 @@ paths:
|
||||
type: string
|
||||
format: date-time
|
||||
- name: scheduled_in_secs
|
||||
description:
|
||||
schedule the script to execute in the number of seconds starting now
|
||||
description: schedule the script to execute in the number of seconds starting now
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
@@ -4524,8 +4515,7 @@ paths:
|
||||
- $ref: "#/components/parameters/NewJobId"
|
||||
- $ref: "#/components/parameters/IncludeHeader"
|
||||
- name: invisible_to_owner
|
||||
description:
|
||||
make the run invisible to the the flow owner (default false)
|
||||
description: make the run invisible to the the flow owner (default false)
|
||||
in: query
|
||||
schema:
|
||||
type: boolean
|
||||
@@ -4563,8 +4553,7 @@ paths:
|
||||
type: string
|
||||
format: date-time
|
||||
- name: scheduled_in_secs
|
||||
description:
|
||||
schedule the script to execute in the number of seconds starting now
|
||||
description: schedule the script to execute in the number of seconds starting now
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
@@ -4573,8 +4562,7 @@ paths:
|
||||
- $ref: "#/components/parameters/NewJobId"
|
||||
- $ref: "#/components/parameters/IncludeHeader"
|
||||
- name: invisible_to_owner
|
||||
description:
|
||||
make the run invisible to the the script owner (default false)
|
||||
description: make the run invisible to the the script owner (default false)
|
||||
in: query
|
||||
schema:
|
||||
type: boolean
|
||||
@@ -4605,8 +4593,7 @@ paths:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/IncludeHeader"
|
||||
- name: invisible_to_owner
|
||||
description:
|
||||
make the run invisible to the the script owner (default false)
|
||||
description: make the run invisible to the the script owner (default false)
|
||||
in: query
|
||||
schema:
|
||||
type: boolean
|
||||
@@ -4639,8 +4626,7 @@ paths:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/IncludeHeader"
|
||||
- name: invisible_to_owner
|
||||
description:
|
||||
make the run invisible to the the script owner (default false)
|
||||
description: make the run invisible to the the script owner (default false)
|
||||
in: query
|
||||
schema:
|
||||
type: boolean
|
||||
@@ -4857,8 +4843,7 @@ paths:
|
||||
- job
|
||||
responses:
|
||||
"200":
|
||||
description:
|
||||
the timestamp of the db that can be used to compute the drift
|
||||
description: the timestamp of the db that can be used to compute the drift
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
@@ -5122,8 +5107,7 @@ paths:
|
||||
|
||||
/w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
|
||||
get:
|
||||
summary:
|
||||
get resume urls given a job_id, resume_id and a nonce to resume a flow
|
||||
summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
|
||||
operationId: getResumeUrls
|
||||
tags:
|
||||
- job
|
||||
@@ -5674,8 +5658,7 @@ paths:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- name: only_member_of
|
||||
in: query
|
||||
description:
|
||||
only list the groups the user is member of (default false)
|
||||
description: only list the groups the user is member of (default false)
|
||||
schema:
|
||||
type: boolean
|
||||
responses:
|
||||
@@ -5863,8 +5846,7 @@ paths:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- name: only_member_of
|
||||
in: query
|
||||
description:
|
||||
only list the folders the user is member of (default false)
|
||||
description: only list the folders the user is member of (default false)
|
||||
schema:
|
||||
type: boolean
|
||||
responses:
|
||||
@@ -6690,20 +6672,16 @@ paths:
|
||||
required:
|
||||
- s3fs_args
|
||||
- polars_cloud_options
|
||||
/w/{workspace}/job_helpers/v2/boto3_connection_settings:
|
||||
/w/{workspace}/job_helpers/v2/s3_resource_info:
|
||||
post:
|
||||
summary:
|
||||
Converts an S3 resource to the set of arguments necessary to connect to
|
||||
an S3 bucket using boto3 in Python
|
||||
operationId: boto3ConnectionSettingsV2
|
||||
summary: Returns the s3 resource associated to the provided path, or the workspace default S3 resource
|
||||
operationId: s3ResourceInfo
|
||||
tags:
|
||||
- helpers
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
requestBody:
|
||||
description:
|
||||
S3 resource path to use to generate the connection settings. If empty,
|
||||
the S3 resource defined in the workspace settings will be used
|
||||
description: S3 resource path to use. If empty, the S3 resource defined in the workspace settings will be used
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
@@ -6718,22 +6696,7 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
endpoint_url:
|
||||
type: string
|
||||
region_name:
|
||||
type: string
|
||||
use_ssl:
|
||||
type: boolean
|
||||
aws_access_key_id:
|
||||
type: string
|
||||
aws_secret_access_key:
|
||||
type: string
|
||||
required:
|
||||
- endpoint_url
|
||||
- region_name
|
||||
- use_ssl
|
||||
$ref: "#/components/schemas/S3Resource"
|
||||
|
||||
/w/{workspace}/job_helpers/test_connection:
|
||||
get:
|
||||
@@ -6948,8 +6911,7 @@ components:
|
||||
type: integer
|
||||
PerPage:
|
||||
name: per_page
|
||||
description:
|
||||
number of items to return for a given page (default 30, max 100)
|
||||
description: number of items to return for a given page (default 30, max 100)
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
@@ -7077,8 +7039,7 @@ components:
|
||||
type: boolean
|
||||
ScheduledForBeforeNow:
|
||||
name: scheduled_for_before_now
|
||||
description:
|
||||
filter on jobs scheduled_for before now (hence waitinf for a worker)
|
||||
description: filter on jobs scheduled_for before now (hence waitinf for a worker)
|
||||
in: query
|
||||
schema:
|
||||
type: boolean
|
||||
@@ -7096,8 +7057,7 @@ components:
|
||||
type: boolean
|
||||
ArgsFilter:
|
||||
name: args
|
||||
description:
|
||||
filter on jobs containing those args as a json subset (@> in postgres)
|
||||
description: filter on jobs containing those args as a json subset (@> in postgres)
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
@@ -7109,8 +7069,7 @@ components:
|
||||
type: string
|
||||
ResultFilter:
|
||||
name: result
|
||||
description:
|
||||
filter on jobs containing those result as a json subset (@> in postgres)
|
||||
description: filter on jobs containing those result as a json subset (@> in postgres)
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
|
||||
@@ -56,8 +56,8 @@ pub fn workspaced_service() -> Router {
|
||||
post(polars_connection_settings_v2).layer(cors.clone()),
|
||||
)
|
||||
.route(
|
||||
"/v2/boto3_connection_settings",
|
||||
post(boto3_connection_settings_v2).layer(cors.clone()),
|
||||
"/v2/s3_resource_info",
|
||||
post(s3_resource_info).layer(cors.clone()),
|
||||
)
|
||||
.route("/test_connection", get(test_connection).layer(cors.clone()))
|
||||
.route(
|
||||
@@ -74,7 +74,7 @@ pub fn workspaced_service() -> Router {
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
struct S3Resource {
|
||||
#[serde(rename = "bucket")]
|
||||
bucket: String,
|
||||
@@ -272,27 +272,18 @@ async fn polars_connection_settings_v2(
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Boto3ConnectionSettingsQueryV2 {
|
||||
struct S3ResourceInfoQuery {
|
||||
s3_resource_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Boto3ConnectionSettingsResponse {
|
||||
region_name: String,
|
||||
use_ssl: bool,
|
||||
endpoint_url: String,
|
||||
aws_access_key_id: Option<String>,
|
||||
aws_secret_access_key: Option<String>,
|
||||
}
|
||||
|
||||
async fn boto3_connection_settings_v2(
|
||||
async fn s3_resource_info(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(db): Extension<DB>,
|
||||
Tokened { token }: Tokened,
|
||||
Path(w_id): Path<String>,
|
||||
Json(query): Json<Boto3ConnectionSettingsQueryV2>,
|
||||
) -> error::JsonResult<Boto3ConnectionSettingsResponse> {
|
||||
Json(query): Json<S3ResourceInfoQuery>,
|
||||
) -> error::JsonResult<S3Resource> {
|
||||
let s3_resource_opt = match query.s3_resource_path {
|
||||
Some(s3_resource_path) => {
|
||||
get_s3_resource(
|
||||
@@ -310,14 +301,7 @@ async fn boto3_connection_settings_v2(
|
||||
let s3_resource = s3_resource_opt.ok_or(error::Error::NotFound(
|
||||
"No datasets storage resource defined at the workspace level".to_string(),
|
||||
))?;
|
||||
let response = Boto3ConnectionSettingsResponse {
|
||||
endpoint_url: render_endpoint(&s3_resource),
|
||||
region_name: s3_resource.region,
|
||||
use_ssl: s3_resource.use_ssl,
|
||||
aws_access_key_id: s3_resource.access_key,
|
||||
aws_secret_access_key: s3_resource.secret_key,
|
||||
};
|
||||
return Ok(Json(response));
|
||||
return Ok(Json(s3_resource));
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
|
||||
@@ -15,17 +15,6 @@ class TestStringMethods(unittest.TestCase):
|
||||
os.environ["BASE_INTERNAL_URL"] = self._host
|
||||
|
||||
def test_duckdb_connection_settings(self):
|
||||
s3_resource = {
|
||||
"port": 9000,
|
||||
"bucket": "windmill",
|
||||
"region": "fr-paris",
|
||||
"useSSL": False,
|
||||
"endPoint": "localhost:9000",
|
||||
"accessKey": "ACCESS_KEY",
|
||||
"pathStyle": True,
|
||||
"secretKey": "SECRET_KEY",
|
||||
}
|
||||
|
||||
settings = wmill.duckdb_connection_settings(self._resource_path)
|
||||
self.assertIsNotNone(settings)
|
||||
|
||||
@@ -38,31 +27,36 @@ SET s3_use_ssl=0;
|
||||
SET s3_access_key_id='IeuKPSYLKTO2h9CWfCVR';
|
||||
SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
|
||||
"""
|
||||
self.assertEqual(settings, {"connection_settings_str": expected_settings_str})
|
||||
|
||||
self.assertEqual(settings["connection_settings_str"], expected_settings_str)
|
||||
self.assertEqual(settings.connection_settings_str, expected_settings_str)
|
||||
|
||||
settings = wmill.polars_connection_settings(self._resource_path)
|
||||
print(settings)
|
||||
|
||||
def test_polars_connection_settings(self):
|
||||
settings = wmill.polars_connection_settings(self._resource_path)
|
||||
expected_settings = {
|
||||
"s3fs_args": {
|
||||
"endpoint_url": "http://localhost:9000",
|
||||
"key": "IeuKPSYLKTO2h9CWfCVR",
|
||||
"secret": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
|
||||
"use_ssl": False,
|
||||
"cache_regions": False,
|
||||
"client_kwargs": {"region_name": "fr-paris"},
|
||||
},
|
||||
"polars_cloud_options": {
|
||||
"aws_endpoint_url": "http://localhost:9000",
|
||||
"aws_access_key_id": "IeuKPSYLKTO2h9CWfCVR",
|
||||
"aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
|
||||
"aws_region": "fr-paris",
|
||||
"aws_allow_http": True,
|
||||
},
|
||||
s3fs_args_expected = {
|
||||
"endpoint_url": "http://localhost:9000",
|
||||
"key": "IeuKPSYLKTO2h9CWfCVR",
|
||||
"secret": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
|
||||
"use_ssl": False,
|
||||
"cache_regions": False,
|
||||
"client_kwargs": {"region_name": "fr-paris"},
|
||||
}
|
||||
self.assertEqual(settings, expected_settings)
|
||||
polars_cloud_options_expected = {
|
||||
"aws_endpoint_url": "http://localhost:9000",
|
||||
"aws_access_key_id": "IeuKPSYLKTO2h9CWfCVR",
|
||||
"aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
|
||||
"aws_region": "fr-paris",
|
||||
"aws_allow_http": True,
|
||||
}
|
||||
self.assertEqual(settings["s3fs_args"], s3fs_args_expected)
|
||||
self.assertEqual(settings.s3fs_args, s3fs_args_expected)
|
||||
self.assertEqual(
|
||||
settings["polars_cloud_options"], polars_cloud_options_expected
|
||||
)
|
||||
self.assertEqual(settings.polars_cloud_options, polars_cloud_options_expected)
|
||||
|
||||
def test_boto3_connection_settings(self):
|
||||
settings = wmill.boto3_connection_settings(self._resource_path)
|
||||
@@ -74,6 +68,8 @@ SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
|
||||
"aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
|
||||
}
|
||||
self.assertEqual(settings, expected_settings)
|
||||
self.assertEqual(settings["endpoint_url"], "http://localhost:9000")
|
||||
self.assertEqual(settings.endpoint_url, "http://localhost:9000")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
from .client import *
|
||||
from .s3_types import *
|
||||
|
||||
@@ -13,6 +13,8 @@ from typing import Dict, Any, Union, Literal
|
||||
|
||||
import httpx
|
||||
|
||||
from .s3_types import Boto3ConnectionSettings, DuckDbConnectionSettings, PolarsConnectionSettings
|
||||
|
||||
_client: "Windmill | None" = None
|
||||
|
||||
logger = logging.getLogger("windmill_client")
|
||||
@@ -318,16 +320,17 @@ class Windmill:
|
||||
self,
|
||||
s3_resource_path: str = "",
|
||||
none_if_undefined: bool = False,
|
||||
) -> Union[str, None]:
|
||||
) -> DuckDbConnectionSettings | None:
|
||||
"""
|
||||
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
|
||||
initiate an S3 connection from DuckDB
|
||||
"""
|
||||
try:
|
||||
return self.post(
|
||||
raw_obj = self.post(
|
||||
f"/w/{self.workspace}/job_helpers/v2/duckdb_connection_settings",
|
||||
json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
|
||||
).json()
|
||||
return DuckDbConnectionSettings(raw_obj)
|
||||
except JSONDecodeError as e:
|
||||
if none_if_undefined:
|
||||
return None
|
||||
@@ -337,16 +340,17 @@ class Windmill:
|
||||
self,
|
||||
s3_resource_path: str = "",
|
||||
none_if_undefined: bool = False,
|
||||
) -> Any:
|
||||
) -> PolarsConnectionSettings | None:
|
||||
"""
|
||||
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
|
||||
initiate an S3 connection from Polars
|
||||
"""
|
||||
try:
|
||||
return self.post(
|
||||
raw_obj = self.post(
|
||||
f"/w/{self.workspace}/job_helpers/v2/polars_connection_settings",
|
||||
json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
|
||||
).json()
|
||||
return PolarsConnectionSettings(raw_obj)
|
||||
except JSONDecodeError as e:
|
||||
if none_if_undefined:
|
||||
return None
|
||||
@@ -356,16 +360,28 @@ class Windmill:
|
||||
self,
|
||||
s3_resource_path: str = "",
|
||||
none_if_undefined: bool = False,
|
||||
) -> Any:
|
||||
) -> Boto3ConnectionSettings | None:
|
||||
"""
|
||||
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
|
||||
initiate an S3 connection using boto3
|
||||
"""
|
||||
try:
|
||||
return self.post(
|
||||
f"/w/{self.workspace}/job_helpers/v2/boto3_connection_settings",
|
||||
s3_resource = self.post(
|
||||
f"/w/{self.workspace}/job_helpers/v2/s3_resource_info",
|
||||
json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
|
||||
).json()
|
||||
endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://"
|
||||
boto3_settings = Boto3ConnectionSettings(
|
||||
{
|
||||
"endpoint_url": "{}{}".format(endpoint_url_prefix, s3_resource["endPoint"]),
|
||||
"region_name": s3_resource["region"],
|
||||
"use_ssl": s3_resource["useSSL"],
|
||||
"aws_access_key_id": s3_resource["accessKey"],
|
||||
"aws_secret_access_key": s3_resource["secretKey"],
|
||||
# no need for path_style here as boto3 is clever enough to determine which one to use
|
||||
}
|
||||
)
|
||||
return boto3_settings
|
||||
except JSONDecodeError as e:
|
||||
if none_if_undefined:
|
||||
return None
|
||||
@@ -574,7 +590,9 @@ def get_result(job_id: str, assert_result_is_not_none=True) -> Dict[str, Any]:
|
||||
|
||||
|
||||
@init_global_client
|
||||
def duckdb_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Union[str, None]:
|
||||
def duckdb_connection_settings(
|
||||
s3_resource_path: str = "", none_if_undefined: bool = False
|
||||
) -> DuckDbConnectionSettings | None:
|
||||
"""
|
||||
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
|
||||
initiate an S3 connection from DuckDB
|
||||
@@ -585,7 +603,9 @@ def duckdb_connection_settings(s3_resource_path: str = "", none_if_undefined: bo
|
||||
|
||||
|
||||
@init_global_client
|
||||
def polars_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Any:
|
||||
def polars_connection_settings(
|
||||
s3_resource_path: str = "", none_if_undefined: bool = False
|
||||
) -> PolarsConnectionSettings | None:
|
||||
"""
|
||||
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
|
||||
initiate an S3 connection from Polars
|
||||
@@ -594,7 +614,9 @@ def polars_connection_settings(s3_resource_path: str = "", none_if_undefined: bo
|
||||
|
||||
|
||||
@init_global_client
|
||||
def boto3_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Any:
|
||||
def boto3_connection_settings(
|
||||
s3_resource_path: str = "", none_if_undefined: bool = False
|
||||
) -> Boto3ConnectionSettings | None:
|
||||
"""
|
||||
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
|
||||
initiate an S3 connection using boto3
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
class S3Object(dict):
|
||||
s3: str
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return self[attr]
|
||||
|
||||
|
||||
class S3FsClientKwargs(dict):
|
||||
region_name: str
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return self[attr]
|
||||
|
||||
|
||||
class S3FsArgs(dict):
|
||||
endpoint_url: str
|
||||
key: str
|
||||
secret: str
|
||||
use_ssl: bool
|
||||
cache_regions: bool
|
||||
client_kwargs: S3FsClientKwargs
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return self[attr]
|
||||
|
||||
|
||||
class PolarsCloudOptions(dict):
|
||||
aws_endpoint_url: str
|
||||
aws_access_key_id: str
|
||||
aws_secret_access_key: str
|
||||
aws_region: bool
|
||||
aws_allow_http: bool
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return self[attr]
|
||||
|
||||
|
||||
class PolarsConnectionSettings(dict):
|
||||
s3fs_args: S3FsArgs
|
||||
polars_cloud_options: PolarsCloudOptions
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return self[attr]
|
||||
|
||||
|
||||
class Boto3ConnectionSettings(dict):
|
||||
endpoint_url: str
|
||||
region_name: str
|
||||
use_ssl: bool
|
||||
aws_access_key_id: str
|
||||
aws_secret_access_key: str
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return self[attr]
|
||||
|
||||
|
||||
class DuckDbConnectionSettings(dict):
|
||||
connection_settings_str: str
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return self[attr]
|
||||
@@ -1,9 +1,11 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
set -eou pipefail
|
||||
|
||||
npx --yes openapi-typescript-codegen --input ../backend/windmill-api/openapi.yaml \
|
||||
--output ./src --useOptions \
|
||||
&& sed -i '213 i \\ request.referrerPolicy = \"no-referrer\"\n' src/core/request.ts
|
||||
|
||||
cp client.ts src/
|
||||
echo 'export { setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState } from "./client";' >> src/index.ts
|
||||
cp s3Types.ts src/
|
||||
echo 'export { type S3Object } from "./s3Types";' >> src/index.ts
|
||||
echo 'export { setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, denoS3LightClientSettings } from "./client";' >> src/index.ts
|
||||
|
||||
+31
-15
@@ -1,5 +1,6 @@
|
||||
import { ResourceService, VariableService, JobService, HelpersService } from "./index";
|
||||
import { OpenAPI } from "./index";
|
||||
import { DenoS3LightClientSettings } from "./s3Types";
|
||||
|
||||
export {
|
||||
AdminService,
|
||||
@@ -247,30 +248,45 @@ export async function databaseUrlFromResource(path: string): Promise<string> {
|
||||
return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}`;
|
||||
}
|
||||
|
||||
export async function polarsConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
|
||||
const workspace = getWorkspace();
|
||||
return await HelpersService.polarsConnectionSettingsV2({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
s3_resource_path: s3_resource_path
|
||||
}
|
||||
});
|
||||
}
|
||||
// TODO(gb): need to investigate more how Polars and DuckDB work in TS
|
||||
// export async function polarsConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
|
||||
// const workspace = getWorkspace();
|
||||
// return await HelpersService.polarsConnectionSettingsV2({
|
||||
// workspace: workspace,
|
||||
// requestBody: {
|
||||
// s3_resource_path: s3_resource_path
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
|
||||
export async function duckdbConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
|
||||
// export async function duckdbConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
|
||||
// const workspace = getWorkspace();
|
||||
// return await HelpersService.duckdbConnectionSettingsV2({
|
||||
// workspace: workspace,
|
||||
// requestBody: {
|
||||
// s3_resource_path: s3_resource_path
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
|
||||
export async function denoS3LightClientSettings(s3_resource_path: string | undefined): Promise<DenoS3LightClientSettings> {
|
||||
const workspace = getWorkspace();
|
||||
return await HelpersService.duckdbConnectionSettingsV2({
|
||||
const s3Resource = await HelpersService.s3ResourceInfo({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
s3_resource_path: s3_resource_path
|
||||
}
|
||||
requestBody: {
|
||||
s3_resource_path: s3_resource_path
|
||||
}
|
||||
});
|
||||
let settings: DenoS3LightClientSettings = {
|
||||
...s3Resource,
|
||||
}
|
||||
return settings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get URLs needed for resuming a flow after this step
|
||||
* @param approver approver name
|
||||
* @returns approval page UI URL, resume and cancel API URLs for resumeing the flow
|
||||
* @returns approval page UI URL, resume and cancel API URLs for resuming the flow
|
||||
*/
|
||||
export async function getResumeUrls(approver?: string): Promise<{
|
||||
approvalPage: string;
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -eou pipefail
|
||||
|
||||
./build.sh
|
||||
rm client.ts
|
||||
tsc
|
||||
cp src/client.ts .
|
||||
npm publish
|
||||
npm publish
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
export type S3Object = {
|
||||
s3: string
|
||||
}
|
||||
|
||||
export type DenoS3LightClientSettings = {
|
||||
endPoint: string,
|
||||
region: string ,
|
||||
bucket?: string,
|
||||
useSSL?: boolean,
|
||||
accessKey?: string,
|
||||
secretKey?: string,
|
||||
pathStyle?: boolean,
|
||||
}
|
||||
Reference in New Issue
Block a user