starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-24 08:01:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Update S3 TS code snippets and Python SDK for Polars 0.20.X (#2911)

Browse Source 
* fix: update S3 TS code snippets

* use S3Object type from SDK

* update python snippets as well

* Update python SDK for Polars 0.20.X
This commit is contained in:
Guillaume Bouvignies
2023-12-22 08:39:35 +01:00
committed by GitHub
parent d238fc366a
commit 539312bb7e
7 changed files with 37 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-api/openapi-deref.yaml
+3 -3
View File
@@ -9967,7 +9967,7 @@ paths:
- use_ssl
- cache_regions
- client_kwargs
polars_cloud_options:
storage_options:
type: object
properties:
aws_endpoint_url:
@@ -9979,14 +9979,14 @@ paths:
aws_region:
type: string
aws_allow_http:
type: boolean
type: string
required:
- aws_endpoint_url
- aws_region
- aws_allow_http
required:
- s3fs_args
- polars_cloud_options
- storage_options
/w/{workspace}/job_helpers/v2/s3_resource_info:
post:
summary: >-
backend/windmill-api/openapi.yaml
+3 -3
View File
@@ -6652,7 +6652,7 @@ paths:
- use_ssl
- cache_regions
- client_kwargs
polars_cloud_options:
storage_options:
type: object
properties:
aws_endpoint_url:
@@ -6664,14 +6664,14 @@ paths:
aws_region:
type: string
aws_allow_http:
type: boolean
type: string
required:
- aws_endpoint_url
- aws_region
- aws_allow_http
required:
- s3fs_args
- polars_cloud_options
- storage_options
/w/{workspace}/job_helpers/v2/s3_resource_info:
post:
summary: Returns the s3 resource associated to the provided path, or the workspace default S3 resource
backend/windmill-api/src/job_helpers.rs
+5 -5
View File
@@ -213,7 +213,7 @@ struct PolarsConnectionSettingsQueryV2 {
#[derive(Serialize)]
struct PolarsConnectionSettingsResponse {
s3fs_args: S3fsArgs,
polars_cloud_options: PolarsCloudOptions,
storage_options: PolarsStorageOptions,
}
#[derive(Serialize)]
@@ -229,14 +229,14 @@ struct S3fsArgs {
}
#[derive(Serialize)]
struct PolarsCloudOptions {
struct PolarsStorageOptions {
aws_endpoint_url: String,
#[serde(skip_serializing_if = "Option::is_none")]
aws_access_key_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
aws_secret_access_key: Option<String>,
aws_region: String,
aws_allow_http: bool,
aws_allow_http: String,
}
async fn polars_connection_settings_v2(
@@ -272,12 +272,12 @@ async fn polars_connection_settings_v2(
.0;
let response = PolarsConnectionSettingsResponse {
s3fs_args: s3fs,
polars_cloud_options: PolarsCloudOptions {
storage_options: PolarsStorageOptions {
aws_endpoint_url: render_endpoint(&s3_resource),
aws_access_key_id: s3_resource.access_key,
aws_secret_access_key: s3_resource.secret_key,
aws_region: s3_resource.region,
aws_allow_http: !s3_resource.use_ssl,
aws_allow_http: (!s3_resource.use_ssl).to_string(),
},
};
return Ok(Json(response));
frontend/src/lib/components/flows/content/s3Scripts/deno.ts
+12 -11
View File
@@ -1,19 +1,19 @@
const deno = {
s3_client: `import * as wmill from "npm:windmill-client@${__pkg__.version}";
s3_client: `import type { S3Object } from "npm:windmill-client@${__pkg__.version}";
import * as wmill from "npm:windmill-client@${__pkg__.version}";
import { S3Client } from "https://deno.land/x/s3_lite_client@0.2.0/mod.ts";
type s3object = object;
export async function main(inputFile: s3object) {
const s3Resource = await wmill.getResource(
"<PATH_TO_S3_RESOURCE>",
);
const s3Client = new S3Client(s3Resource);
export async function main(inputFile: S3Object) {
// this will default to the workspace s3 resource
let args = await wmill.denoS3LightClientSettings();
// this will use the designated resource
// let args = await wmill.denoS3LightClientSettings("<PATH_TO_S3_RESOURCE>");
const s3Client = new S3Client(args);
const outputFile = "output/hello.txt"
// read object from S3
const getObjectResponse = await s3Client.getObject(inputFile["s3"]);
const getObjectResponse = await s3Client.getObject(inputFile.s3);
const inputObjContent = await getObjectResponse.text();
console.log(inputObjContent);
@@ -25,9 +25,10 @@ export async function main(inputFile: s3object) {
console.log(obj.key);
}
return {
"s3": outputFile,
const result: S3Object = {
s3: outputFile,
};
return result;
}
`
}
frontend/src/lib/components/flows/content/s3Scripts/python3.ts
+9 -5
View File
@@ -1,7 +1,9 @@
const python3 = {
s3_client: `#requirements:
#wmill>=1.228.0
import wmillimport wmill
#boto3==1.34.4
#wmill>=1.229.0
import wmill
from wmill import S3Object
import boto3
@@ -47,8 +49,9 @@ def main(input_file: S3Object):
`,
polars: `#requirements:
#polars==0.19.1
#wmill>=1.228.0
#polars==0.19.19
#s3fs==2023.12.0
#wmill>=1.229.0
import wmill
from wmill import S3Object
@@ -91,8 +94,9 @@ def main(input_file: S3Object):
`,
duckdb: `#requirements:
#wmill>=1.229.0
#duckdb==0.9.1
#wmill>=1.228.0
import wmill
from wmill import S3Object
import duckdb
python-client/wmill/wmill/s3_types.py
+4 -4
View File
@@ -24,12 +24,12 @@ class S3FsArgs(dict):
return self[attr]
class PolarsCloudOptions(dict):
class StorageOptions(dict):
aws_endpoint_url: str
aws_access_key_id: str
aws_secret_access_key: str
aws_region: bool
aws_allow_http: bool
aws_region: str
aws_allow_http: str
def __getattr__(self, attr):
return self[attr]
@@ -37,7 +37,7 @@ class PolarsCloudOptions(dict):
class PolarsConnectionSettings(dict):
s3fs_args: S3FsArgs
polars_cloud_options: PolarsCloudOptions
storage_options: StorageOptions
def __getattr__(self, attr):
return self[attr]
typescript-client/client.ts
+1
View File
@@ -270,6 +270,7 @@ export async function databaseUrlFromResource(path: string): Promise<string> {
// }
export async function denoS3LightClientSettings(s3_resource_path: string | undefined): Promise<DenoS3LightClientSettings> {
!clientSet && setClient();
const workspace = getWorkspace();
const s3Resource = await HelpersService.s3ResourceInfo({
workspace: workspace,