From d316af993db7070e7165ccc6f2a561fa77ac971e Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Wed, 27 Dec 2023 15:02:06 +0100 Subject: [PATCH] chore: update snippets for Polars 0.20 (#2925) --- backend/windmill-worker/src/bun_executor.rs | 8 +--- .../flows/content/s3Scripts/python3.ts | 39 +++++++++---------- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index ac16ad4453..ff9646c0e2 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -86,7 +86,7 @@ pub async fn gen_lockfile( // if custom NPM registry is being used, write bunfig.toml at the root of the job dir if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() { - let (raw_url, token_opt) = if s.contains(":_authToken=") { + let (url, token_opt) = if s.contains(":_authToken=") { let split_url = s.split(":_authToken=").collect::>(); let url = split_url .get(0) @@ -100,12 +100,6 @@ pub async fn gen_lockfile( } else { (s.to_owned(), None) }; - // somehow bun fails to resolve deps if the url does not end with a slash ... - let url = if !raw_url.ends_with("/") { - format!("{raw_url}/") - } else { - raw_url.to_string() - }; let registry_toml_string = if let Some(token) = token_opt { format!("{{ url = \"{url}\", token = \"{token}\" }}") } else { diff --git a/frontend/src/lib/components/flows/content/s3Scripts/python3.ts b/frontend/src/lib/components/flows/content/s3Scripts/python3.ts index 9d3f13e695..5f3420e186 100644 --- a/frontend/src/lib/components/flows/content/s3Scripts/python3.ts +++ b/frontend/src/lib/components/flows/content/s3Scripts/python3.ts @@ -49,7 +49,7 @@ def main(input_file: S3Object): `, polars: `#requirements: -#polars==0.19.19 +#polars==0.20.2 #s3fs==2023.12.0 #wmill>=1.229.0 @@ -60,33 +60,30 @@ import s3fs def main(input_file: S3Object): - bucket = wmill.get_resource("u/admin/windmill-cloud-demo")["bucket"] + bucket = wmill.get_resource("")["bucket"] # this will default to the workspace s3 resource - args = wmill.polars_connection_settings().s3fs_args + storage_options = wmill.polars_connection_settings().storage_options # this will use the designated resource - # args = wmill.polars_connection_settings("").s3fs_args - s3 = s3fs.S3FileSystem(**args) + # storage_options = wmill.polars_connection_settings("").storage_options + # input is a parquet file, we use read_parquet in lazy mode. + # Polars can read various file types, see + # https://pola-rs.github.io/polars/py-polars/html/reference/io.html input_uri = "s3://{}/{}".format(bucket, input_file["s3"]) + input_df = pl.read_parquet(input_uri, storage_options=storage_options).lazy() + + # process the Polars dataframe. See Polars docs: + # for dataframe: https://pola-rs.github.io/polars/py-polars/html/reference/dataframe/index.html + # for lazy dataframe: https://pola-rs.github.io/polars/py-polars/html/reference/lazyframe/index.html + output_df = input_df.collect() + print(output_df) + + # To write back the result to S3, Polars needs an s3fs connection + s3 = s3fs.S3FileSystem(**wmill.polars_connection_settings().s3fs_args) output_file = "output/result.parquet" output_uri = "s3://{}/{}".format(bucket, output_file) - - with ( - s3.open(input_uri, mode="rb") as input_s3, - s3.open(output_uri, mode="wb") as output_s3, - ): - # input is a parquet file, we use read_parquet in lazy mode. - # Polars can read various file types, see - # https://pola-rs.github.io/polars/py-polars/html/reference/io.html - input_df = pl.read_parquet(input_s3).lazy() - - # process the Polars dataframe. See Polars docs: - # for dataframe: https://pola-rs.github.io/polars/py-polars/html/reference/dataframe/index.html - # for lazy dataframe: https://pola-rs.github.io/polars/py-polars/html/reference/lazyframe/index.html - output_df = input_df.collect() - print(output_df) - + with s3.open(output_uri, mode="wb") as output_s3: # persist the output dataframe back to S3 and return it output_df.write_parquet(output_s3)