From 539312bb7e70dfc7804e76e1f58fc558e6db9510 Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Fri, 22 Dec 2023 08:39:35 +0100 Subject: [PATCH] fix: Update S3 TS code snippets and Python SDK for Polars 0.20.X (#2911) * fix: update S3 TS code snippets * use S3Object type from SDK * update python snippets as well * Update python SDK for Polars 0.20.X --- backend/windmill-api/openapi-deref.yaml | 6 ++--- backend/windmill-api/openapi.yaml | 6 ++--- backend/windmill-api/src/job_helpers.rs | 10 ++++---- .../flows/content/s3Scripts/deno.ts | 23 ++++++++++--------- .../flows/content/s3Scripts/python3.ts | 14 +++++++---- python-client/wmill/wmill/s3_types.py | 8 +++---- typescript-client/client.ts | 1 + 7 files changed, 37 insertions(+), 31 deletions(-) diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 2b7bad3ed6..da63ad4cea 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -9967,7 +9967,7 @@ paths: - use_ssl - cache_regions - client_kwargs - polars_cloud_options: + storage_options: type: object properties: aws_endpoint_url: @@ -9979,14 +9979,14 @@ paths: aws_region: type: string aws_allow_http: - type: boolean + type: string required: - aws_endpoint_url - aws_region - aws_allow_http required: - s3fs_args - - polars_cloud_options + - storage_options /w/{workspace}/job_helpers/v2/s3_resource_info: post: summary: >- diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 397d52208c..5d2110f061 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6652,7 +6652,7 @@ paths: - use_ssl - cache_regions - client_kwargs - polars_cloud_options: + storage_options: type: object properties: aws_endpoint_url: @@ -6664,14 +6664,14 @@ paths: aws_region: type: string aws_allow_http: - type: boolean + type: string required: - aws_endpoint_url - aws_region - aws_allow_http required: - s3fs_args - - polars_cloud_options + - storage_options /w/{workspace}/job_helpers/v2/s3_resource_info: post: summary: Returns the s3 resource associated to the provided path, or the workspace default S3 resource diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs index c597564211..e914299e53 100644 --- a/backend/windmill-api/src/job_helpers.rs +++ b/backend/windmill-api/src/job_helpers.rs @@ -213,7 +213,7 @@ struct PolarsConnectionSettingsQueryV2 { #[derive(Serialize)] struct PolarsConnectionSettingsResponse { s3fs_args: S3fsArgs, - polars_cloud_options: PolarsCloudOptions, + storage_options: PolarsStorageOptions, } #[derive(Serialize)] @@ -229,14 +229,14 @@ struct S3fsArgs { } #[derive(Serialize)] -struct PolarsCloudOptions { +struct PolarsStorageOptions { aws_endpoint_url: String, #[serde(skip_serializing_if = "Option::is_none")] aws_access_key_id: Option, #[serde(skip_serializing_if = "Option::is_none")] aws_secret_access_key: Option, aws_region: String, - aws_allow_http: bool, + aws_allow_http: String, } async fn polars_connection_settings_v2( @@ -272,12 +272,12 @@ async fn polars_connection_settings_v2( .0; let response = PolarsConnectionSettingsResponse { s3fs_args: s3fs, - polars_cloud_options: PolarsCloudOptions { + storage_options: PolarsStorageOptions { aws_endpoint_url: render_endpoint(&s3_resource), aws_access_key_id: s3_resource.access_key, aws_secret_access_key: s3_resource.secret_key, aws_region: s3_resource.region, - aws_allow_http: !s3_resource.use_ssl, + aws_allow_http: (!s3_resource.use_ssl).to_string(), }, }; return Ok(Json(response)); diff --git a/frontend/src/lib/components/flows/content/s3Scripts/deno.ts b/frontend/src/lib/components/flows/content/s3Scripts/deno.ts index f4e9324572..ba21426ebb 100644 --- a/frontend/src/lib/components/flows/content/s3Scripts/deno.ts +++ b/frontend/src/lib/components/flows/content/s3Scripts/deno.ts @@ -1,19 +1,19 @@ const deno = { - s3_client: `import * as wmill from "npm:windmill-client@${__pkg__.version}"; + s3_client: `import type { S3Object } from "npm:windmill-client@${__pkg__.version}"; +import * as wmill from "npm:windmill-client@${__pkg__.version}"; import { S3Client } from "https://deno.land/x/s3_lite_client@0.2.0/mod.ts"; -type s3object = object; - -export async function main(inputFile: s3object) { - const s3Resource = await wmill.getResource( - "", - ); - const s3Client = new S3Client(s3Resource); +export async function main(inputFile: S3Object) { + // this will default to the workspace s3 resource + let args = await wmill.denoS3LightClientSettings(); + // this will use the designated resource + // let args = await wmill.denoS3LightClientSettings(""); + const s3Client = new S3Client(args); const outputFile = "output/hello.txt" // read object from S3 - const getObjectResponse = await s3Client.getObject(inputFile["s3"]); + const getObjectResponse = await s3Client.getObject(inputFile.s3); const inputObjContent = await getObjectResponse.text(); console.log(inputObjContent); @@ -25,9 +25,10 @@ export async function main(inputFile: s3object) { console.log(obj.key); } - return { - "s3": outputFile, + const result: S3Object = { + s3: outputFile, }; + return result; } ` } diff --git a/frontend/src/lib/components/flows/content/s3Scripts/python3.ts b/frontend/src/lib/components/flows/content/s3Scripts/python3.ts index 43c75965f8..9d3f13e695 100644 --- a/frontend/src/lib/components/flows/content/s3Scripts/python3.ts +++ b/frontend/src/lib/components/flows/content/s3Scripts/python3.ts @@ -1,7 +1,9 @@ const python3 = { s3_client: `#requirements: -#wmill>=1.228.0 -import wmillimport wmill +#boto3==1.34.4 +#wmill>=1.229.0 + +import wmill from wmill import S3Object import boto3 @@ -47,8 +49,9 @@ def main(input_file: S3Object): `, polars: `#requirements: -#polars==0.19.1 -#wmill>=1.228.0 +#polars==0.19.19 +#s3fs==2023.12.0 +#wmill>=1.229.0 import wmill from wmill import S3Object @@ -91,8 +94,9 @@ def main(input_file: S3Object): `, duckdb: `#requirements: +#wmill>=1.229.0 #duckdb==0.9.1 -#wmill>=1.228.0 + import wmill from wmill import S3Object import duckdb diff --git a/python-client/wmill/wmill/s3_types.py b/python-client/wmill/wmill/s3_types.py index 93cef6e01e..b3bbe4db3f 100644 --- a/python-client/wmill/wmill/s3_types.py +++ b/python-client/wmill/wmill/s3_types.py @@ -24,12 +24,12 @@ class S3FsArgs(dict): return self[attr] -class PolarsCloudOptions(dict): +class StorageOptions(dict): aws_endpoint_url: str aws_access_key_id: str aws_secret_access_key: str - aws_region: bool - aws_allow_http: bool + aws_region: str + aws_allow_http: str def __getattr__(self, attr): return self[attr] @@ -37,7 +37,7 @@ class PolarsCloudOptions(dict): class PolarsConnectionSettings(dict): s3fs_args: S3FsArgs - polars_cloud_options: PolarsCloudOptions + storage_options: StorageOptions def __getattr__(self, attr): return self[attr] diff --git a/typescript-client/client.ts b/typescript-client/client.ts index ad2413b22f..75ebdff7c8 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -270,6 +270,7 @@ export async function databaseUrlFromResource(path: string): Promise { // } export async function denoS3LightClientSettings(s3_resource_path: string | undefined): Promise { + !clientSet && setClient(); const workspace = getWorkspace(); const s3Resource = await HelpersService.s3ResourceInfo({ workspace: workspace,