feat: S3 objects are now typed in Python and TS SDK (#2878)

* feat: S3 objects are now typed in Python and TS SDK

* fix ts and python SDK after testing

* cleanup
This commit is contained in:
Guillaume Bouvignies
2023-12-20 09:16:33 +01:00
committed by GitHub
parent ba036e0576
commit fd55c3d8e3
11 changed files with 714 additions and 626 deletions
File diff suppressed because it is too large Load Diff
+27 -68
View File
@@ -2231,8 +2231,7 @@ paths:
/w/{workspace}/resources/get_value_interpolated/{path}:
get:
summary:
get resource interpolated (variables and resources are fully unrolled)
summary: get resource interpolated (variables and resources are fully unrolled)
operationId: getResourceValueInterpolated
tags:
- resource
@@ -2920,8 +2919,7 @@ paths:
schema:
type: string
- name: first_parent_hash
description:
mask to filter scripts whom first direct parent has exact hash
description: mask to filter scripts whom first direct parent has exact hash
in: query
schema:
type: string
@@ -3116,8 +3114,7 @@ paths:
/workers/custom_tags:
get:
summary:
get all instance custom tags (tags are used to dispatch jobs to
summary: get all instance custom tags (tags are used to dispatch jobs to
different worker groups)
operationId: getCustomTags
tags:
@@ -3168,8 +3165,7 @@ paths:
/w/{workspace}/scripts/delete/h/{hash}:
post:
summary:
delete script by hash (erase content but keep hash, require admin)
summary: delete script by hash (erase content but keep hash, require admin)
operationId: deleteScriptByHash
tags:
- script
@@ -3408,8 +3404,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -3417,8 +3412,7 @@ paths:
- $ref: "#/components/parameters/WorkerTag"
- $ref: "#/components/parameters/NewJobId"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4451,8 +4445,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4461,8 +4454,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the flow owner (default false)
description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4514,8 +4506,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4524,8 +4515,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the flow owner (default false)
description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4563,8 +4553,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4573,8 +4562,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4605,8 +4593,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4639,8 +4626,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4857,8 +4843,7 @@ paths:
- job
responses:
"200":
description:
the timestamp of the db that can be used to compute the drift
description: the timestamp of the db that can be used to compute the drift
content:
application/json:
schema:
@@ -5122,8 +5107,7 @@ paths:
/w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
get:
summary:
get resume urls given a job_id, resume_id and a nonce to resume a flow
summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
operationId: getResumeUrls
tags:
- job
@@ -5674,8 +5658,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
description:
only list the groups the user is member of (default false)
description: only list the groups the user is member of (default false)
schema:
type: boolean
responses:
@@ -5863,8 +5846,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
description:
only list the folders the user is member of (default false)
description: only list the folders the user is member of (default false)
schema:
type: boolean
responses:
@@ -6690,20 +6672,16 @@ paths:
required:
- s3fs_args
- polars_cloud_options
/w/{workspace}/job_helpers/v2/boto3_connection_settings:
/w/{workspace}/job_helpers/v2/s3_resource_info:
post:
summary:
Converts an S3 resource to the set of arguments necessary to connect to
an S3 bucket using boto3 in Python
operationId: boto3ConnectionSettingsV2
summary: Returns the s3 resource associated to the provided path, or the workspace default S3 resource
operationId: s3ResourceInfo
tags:
- helpers
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description:
S3 resource path to use to generate the connection settings. If empty,
the S3 resource defined in the workspace settings will be used
description: S3 resource path to use. If empty, the S3 resource defined in the workspace settings will be used
required: true
content:
application/json:
@@ -6718,22 +6696,7 @@ paths:
content:
application/json:
schema:
type: object
properties:
endpoint_url:
type: string
region_name:
type: string
use_ssl:
type: boolean
aws_access_key_id:
type: string
aws_secret_access_key:
type: string
required:
- endpoint_url
- region_name
- use_ssl
$ref: "#/components/schemas/S3Resource"
/w/{workspace}/job_helpers/test_connection:
get:
@@ -6948,8 +6911,7 @@ components:
type: integer
PerPage:
name: per_page
description:
number of items to return for a given page (default 30, max 100)
description: number of items to return for a given page (default 30, max 100)
in: query
schema:
type: integer
@@ -7077,8 +7039,7 @@ components:
type: boolean
ScheduledForBeforeNow:
name: scheduled_for_before_now
description:
filter on jobs scheduled_for before now (hence waitinf for a worker)
description: filter on jobs scheduled_for before now (hence waitinf for a worker)
in: query
schema:
type: boolean
@@ -7096,8 +7057,7 @@ components:
type: boolean
ArgsFilter:
name: args
description:
filter on jobs containing those args as a json subset (@> in postgres)
description: filter on jobs containing those args as a json subset (@> in postgres)
in: query
schema:
type: string
@@ -7109,8 +7069,7 @@ components:
type: string
ResultFilter:
name: result
description:
filter on jobs containing those result as a json subset (@> in postgres)
description: filter on jobs containing those result as a json subset (@> in postgres)
in: query
schema:
type: string
+8 -24
View File
@@ -56,8 +56,8 @@ pub fn workspaced_service() -> Router {
post(polars_connection_settings_v2).layer(cors.clone()),
)
.route(
"/v2/boto3_connection_settings",
post(boto3_connection_settings_v2).layer(cors.clone()),
"/v2/s3_resource_info",
post(s3_resource_info).layer(cors.clone()),
)
.route("/test_connection", get(test_connection).layer(cors.clone()))
.route(
@@ -74,7 +74,7 @@ pub fn workspaced_service() -> Router {
)
}
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Deserialize, Serialize, Clone)]
struct S3Resource {
#[serde(rename = "bucket")]
bucket: String,
@@ -272,27 +272,18 @@ async fn polars_connection_settings_v2(
}
#[derive(Deserialize)]
struct Boto3ConnectionSettingsQueryV2 {
struct S3ResourceInfoQuery {
s3_resource_path: Option<String>,
}
#[derive(Serialize)]
struct Boto3ConnectionSettingsResponse {
region_name: String,
use_ssl: bool,
endpoint_url: String,
aws_access_key_id: Option<String>,
aws_secret_access_key: Option<String>,
}
async fn boto3_connection_settings_v2(
async fn s3_resource_info(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Tokened { token }: Tokened,
Path(w_id): Path<String>,
Json(query): Json<Boto3ConnectionSettingsQueryV2>,
) -> error::JsonResult<Boto3ConnectionSettingsResponse> {
Json(query): Json<S3ResourceInfoQuery>,
) -> error::JsonResult<S3Resource> {
let s3_resource_opt = match query.s3_resource_path {
Some(s3_resource_path) => {
get_s3_resource(
@@ -310,14 +301,7 @@ async fn boto3_connection_settings_v2(
let s3_resource = s3_resource_opt.ok_or(error::Error::NotFound(
"No datasets storage resource defined at the workspace level".to_string(),
))?;
let response = Boto3ConnectionSettingsResponse {
endpoint_url: render_endpoint(&s3_resource),
region_name: s3_resource.region,
use_ssl: s3_resource.use_ssl,
aws_access_key_id: s3_resource.access_key,
aws_secret_access_key: s3_resource.secret_key,
};
return Ok(Json(response));
return Ok(Json(s3_resource));
}
#[derive(Serialize, Deserialize, Clone)]
+25 -29
View File
@@ -15,17 +15,6 @@ class TestStringMethods(unittest.TestCase):
os.environ["BASE_INTERNAL_URL"] = self._host
def test_duckdb_connection_settings(self):
s3_resource = {
"port": 9000,
"bucket": "windmill",
"region": "fr-paris",
"useSSL": False,
"endPoint": "localhost:9000",
"accessKey": "ACCESS_KEY",
"pathStyle": True,
"secretKey": "SECRET_KEY",
}
settings = wmill.duckdb_connection_settings(self._resource_path)
self.assertIsNotNone(settings)
@@ -38,31 +27,36 @@ SET s3_use_ssl=0;
SET s3_access_key_id='IeuKPSYLKTO2h9CWfCVR';
SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
"""
self.assertEqual(settings, {"connection_settings_str": expected_settings_str})
self.assertEqual(settings["connection_settings_str"], expected_settings_str)
self.assertEqual(settings.connection_settings_str, expected_settings_str)
settings = wmill.polars_connection_settings(self._resource_path)
print(settings)
def test_polars_connection_settings(self):
settings = wmill.polars_connection_settings(self._resource_path)
expected_settings = {
"s3fs_args": {
"endpoint_url": "http://localhost:9000",
"key": "IeuKPSYLKTO2h9CWfCVR",
"secret": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
"use_ssl": False,
"cache_regions": False,
"client_kwargs": {"region_name": "fr-paris"},
},
"polars_cloud_options": {
"aws_endpoint_url": "http://localhost:9000",
"aws_access_key_id": "IeuKPSYLKTO2h9CWfCVR",
"aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
"aws_region": "fr-paris",
"aws_allow_http": True,
},
s3fs_args_expected = {
"endpoint_url": "http://localhost:9000",
"key": "IeuKPSYLKTO2h9CWfCVR",
"secret": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
"use_ssl": False,
"cache_regions": False,
"client_kwargs": {"region_name": "fr-paris"},
}
self.assertEqual(settings, expected_settings)
polars_cloud_options_expected = {
"aws_endpoint_url": "http://localhost:9000",
"aws_access_key_id": "IeuKPSYLKTO2h9CWfCVR",
"aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
"aws_region": "fr-paris",
"aws_allow_http": True,
}
self.assertEqual(settings["s3fs_args"], s3fs_args_expected)
self.assertEqual(settings.s3fs_args, s3fs_args_expected)
self.assertEqual(
settings["polars_cloud_options"], polars_cloud_options_expected
)
self.assertEqual(settings.polars_cloud_options, polars_cloud_options_expected)
def test_boto3_connection_settings(self):
settings = wmill.boto3_connection_settings(self._resource_path)
@@ -74,6 +68,8 @@ SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
"aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
}
self.assertEqual(settings, expected_settings)
self.assertEqual(settings["endpoint_url"], "http://localhost:9000")
self.assertEqual(settings.endpoint_url, "http://localhost:9000")
if __name__ == "__main__":
+1
View File
@@ -1 +1,2 @@
from .client import *
from .s3_types import *
+32 -10
View File
@@ -13,6 +13,8 @@ from typing import Dict, Any, Union, Literal
import httpx
from .s3_types import Boto3ConnectionSettings, DuckDbConnectionSettings, PolarsConnectionSettings
_client: "Windmill | None" = None
logger = logging.getLogger("windmill_client")
@@ -318,16 +320,17 @@ class Windmill:
self,
s3_resource_path: str = "",
none_if_undefined: bool = False,
) -> Union[str, None]:
) -> DuckDbConnectionSettings | None:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection from DuckDB
"""
try:
return self.post(
raw_obj = self.post(
f"/w/{self.workspace}/job_helpers/v2/duckdb_connection_settings",
json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
).json()
return DuckDbConnectionSettings(raw_obj)
except JSONDecodeError as e:
if none_if_undefined:
return None
@@ -337,16 +340,17 @@ class Windmill:
self,
s3_resource_path: str = "",
none_if_undefined: bool = False,
) -> Any:
) -> PolarsConnectionSettings | None:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection from Polars
"""
try:
return self.post(
raw_obj = self.post(
f"/w/{self.workspace}/job_helpers/v2/polars_connection_settings",
json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
).json()
return PolarsConnectionSettings(raw_obj)
except JSONDecodeError as e:
if none_if_undefined:
return None
@@ -356,16 +360,28 @@ class Windmill:
self,
s3_resource_path: str = "",
none_if_undefined: bool = False,
) -> Any:
) -> Boto3ConnectionSettings | None:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection using boto3
"""
try:
return self.post(
f"/w/{self.workspace}/job_helpers/v2/boto3_connection_settings",
s3_resource = self.post(
f"/w/{self.workspace}/job_helpers/v2/s3_resource_info",
json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
).json()
endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://"
boto3_settings = Boto3ConnectionSettings(
{
"endpoint_url": "{}{}".format(endpoint_url_prefix, s3_resource["endPoint"]),
"region_name": s3_resource["region"],
"use_ssl": s3_resource["useSSL"],
"aws_access_key_id": s3_resource["accessKey"],
"aws_secret_access_key": s3_resource["secretKey"],
# no need for path_style here as boto3 is clever enough to determine which one to use
}
)
return boto3_settings
except JSONDecodeError as e:
if none_if_undefined:
return None
@@ -574,7 +590,9 @@ def get_result(job_id: str, assert_result_is_not_none=True) -> Dict[str, Any]:
@init_global_client
def duckdb_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Union[str, None]:
def duckdb_connection_settings(
s3_resource_path: str = "", none_if_undefined: bool = False
) -> DuckDbConnectionSettings | None:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection from DuckDB
@@ -585,7 +603,9 @@ def duckdb_connection_settings(s3_resource_path: str = "", none_if_undefined: bo
@init_global_client
def polars_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Any:
def polars_connection_settings(
s3_resource_path: str = "", none_if_undefined: bool = False
) -> PolarsConnectionSettings | None:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection from Polars
@@ -594,7 +614,9 @@ def polars_connection_settings(s3_resource_path: str = "", none_if_undefined: bo
@init_global_client
def boto3_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Any:
def boto3_connection_settings(
s3_resource_path: str = "", none_if_undefined: bool = False
) -> Boto3ConnectionSettings | None:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection using boto3
+61
View File
@@ -0,0 +1,61 @@
class S3Object(dict):
s3: str
def __getattr__(self, attr):
return self[attr]
class S3FsClientKwargs(dict):
region_name: str
def __getattr__(self, attr):
return self[attr]
class S3FsArgs(dict):
endpoint_url: str
key: str
secret: str
use_ssl: bool
cache_regions: bool
client_kwargs: S3FsClientKwargs
def __getattr__(self, attr):
return self[attr]
class PolarsCloudOptions(dict):
aws_endpoint_url: str
aws_access_key_id: str
aws_secret_access_key: str
aws_region: bool
aws_allow_http: bool
def __getattr__(self, attr):
return self[attr]
class PolarsConnectionSettings(dict):
s3fs_args: S3FsArgs
polars_cloud_options: PolarsCloudOptions
def __getattr__(self, attr):
return self[attr]
class Boto3ConnectionSettings(dict):
endpoint_url: str
region_name: str
use_ssl: bool
aws_access_key_id: str
aws_secret_access_key: str
def __getattr__(self, attr):
return self[attr]
class DuckDbConnectionSettings(dict):
connection_settings_str: str
def __getattr__(self, attr):
return self[attr]
+4 -2
View File
@@ -1,9 +1,11 @@
#!/bin/bash
set -e
set -eou pipefail
npx --yes openapi-typescript-codegen --input ../backend/windmill-api/openapi.yaml \
--output ./src --useOptions \
&& sed -i '213 i \\ request.referrerPolicy = \"no-referrer\"\n' src/core/request.ts
cp client.ts src/
echo 'export { setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState } from "./client";' >> src/index.ts
cp s3Types.ts src/
echo 'export { type S3Object } from "./s3Types";' >> src/index.ts
echo 'export { setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, denoS3LightClientSettings } from "./client";' >> src/index.ts
+31 -15
View File
@@ -1,5 +1,6 @@
import { ResourceService, VariableService, JobService, HelpersService } from "./index";
import { OpenAPI } from "./index";
import { DenoS3LightClientSettings } from "./s3Types";
export {
AdminService,
@@ -247,30 +248,45 @@ export async function databaseUrlFromResource(path: string): Promise<string> {
return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}`;
}
export async function polarsConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
const workspace = getWorkspace();
return await HelpersService.polarsConnectionSettingsV2({
workspace: workspace,
requestBody: {
s3_resource_path: s3_resource_path
}
});
}
// TODO(gb): need to investigate more how Polars and DuckDB work in TS
// export async function polarsConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
// const workspace = getWorkspace();
// return await HelpersService.polarsConnectionSettingsV2({
// workspace: workspace,
// requestBody: {
// s3_resource_path: s3_resource_path
// }
// });
// }
export async function duckdbConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
// export async function duckdbConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
// const workspace = getWorkspace();
// return await HelpersService.duckdbConnectionSettingsV2({
// workspace: workspace,
// requestBody: {
// s3_resource_path: s3_resource_path
// }
// });
// }
export async function denoS3LightClientSettings(s3_resource_path: string | undefined): Promise<DenoS3LightClientSettings> {
const workspace = getWorkspace();
return await HelpersService.duckdbConnectionSettingsV2({
const s3Resource = await HelpersService.s3ResourceInfo({
workspace: workspace,
requestBody: {
s3_resource_path: s3_resource_path
}
requestBody: {
s3_resource_path: s3_resource_path
}
});
let settings: DenoS3LightClientSettings = {
...s3Resource,
}
return settings;
}
/**
* Get URLs needed for resuming a flow after this step
* @param approver approver name
* @returns approval page UI URL, resume and cancel API URLs for resumeing the flow
* @returns approval page UI URL, resume and cancel API URLs for resuming the flow
*/
export async function getResumeUrls(approver?: string): Promise<{
approvalPage: string;
+4 -1
View File
@@ -1,5 +1,8 @@
#!/bin/bash
set -eou pipefail
./build.sh
rm client.ts
tsc
cp src/client.ts .
npm publish
npm publish
+13
View File
@@ -0,0 +1,13 @@
export type S3Object = {
s3: string
}
export type DenoS3LightClientSettings = {
endPoint: string,
region: string ,
bucket?: string,
useSSL?: boolean,
accessKey?: string,
secretKey?: string,
pathStyle?: boolean,
}