starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-25 00:01:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: update snippets for Polars 0.20 (#2925)

Browse Source
This commit is contained in:
Guillaume Bouvignies
2023-12-27 15:02:06 +01:00
committed by GitHub
parent 263f41cda3
commit d316af993d
2 changed files with 19 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-worker/src/bun_executor.rs
+1 -7
View File
@@ -86,7 +86,7 @@ pub async fn gen_lockfile(
// if custom NPM registry is being used, write bunfig.toml at the root of the job dir
if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() {
let (raw_url, token_opt) = if s.contains(":_authToken=") {
let (url, token_opt) = if s.contains(":_authToken=") {
let split_url = s.split(":_authToken=").collect::<Vec<&str>>();
let url = split_url
.get(0)
@@ -100,12 +100,6 @@ pub async fn gen_lockfile(
} else {
(s.to_owned(), None)
};
// somehow bun fails to resolve deps if the url does not end with a slash ...
let url = if !raw_url.ends_with("/") {
format!("{raw_url}/")
} else {
raw_url.to_string()
};
let registry_toml_string = if let Some(token) = token_opt {
format!("{{ url = \"{url}\", token = \"{token}\" }}")
} else {
frontend/src/lib/components/flows/content/s3Scripts/python3.ts
+18 -21
View File
@@ -49,7 +49,7 @@ def main(input_file: S3Object):
`,
polars: `#requirements:
#polars==0.19.19
#polars==0.20.2
#s3fs==2023.12.0
#wmill>=1.229.0
@@ -60,33 +60,30 @@ import s3fs
def main(input_file: S3Object):
bucket = wmill.get_resource("u/admin/windmill-cloud-demo")["bucket"]
bucket = wmill.get_resource("<PATH_TO_S3_RESOURCE>")["bucket"]
# this will default to the workspace s3 resource
args = wmill.polars_connection_settings().s3fs_args
storage_options = wmill.polars_connection_settings().storage_options
# this will use the designated resource
# args = wmill.polars_connection_settings("<PATH_TO_S3_RESOURCE>").s3fs_args
s3 = s3fs.S3FileSystem(**args)
# storage_options = wmill.polars_connection_settings("<PATH_TO_S3_RESOURCE>").storage_options
# input is a parquet file, we use read_parquet in lazy mode.
# Polars can read various file types, see
# https://pola-rs.github.io/polars/py-polars/html/reference/io.html
input_uri = "s3://{}/{}".format(bucket, input_file["s3"])
input_df = pl.read_parquet(input_uri, storage_options=storage_options).lazy()
# process the Polars dataframe. See Polars docs:
# for dataframe: https://pola-rs.github.io/polars/py-polars/html/reference/dataframe/index.html
# for lazy dataframe: https://pola-rs.github.io/polars/py-polars/html/reference/lazyframe/index.html
output_df = input_df.collect()
print(output_df)
# To write back the result to S3, Polars needs an s3fs connection
s3 = s3fs.S3FileSystem(**wmill.polars_connection_settings().s3fs_args)
output_file = "output/result.parquet"
output_uri = "s3://{}/{}".format(bucket, output_file)
with (
s3.open(input_uri, mode="rb") as input_s3,
s3.open(output_uri, mode="wb") as output_s3,
):
# input is a parquet file, we use read_parquet in lazy mode.
# Polars can read various file types, see
# https://pola-rs.github.io/polars/py-polars/html/reference/io.html
input_df = pl.read_parquet(input_s3).lazy()
# process the Polars dataframe. See Polars docs:
# for dataframe: https://pola-rs.github.io/polars/py-polars/html/reference/dataframe/index.html
# for lazy dataframe: https://pola-rs.github.io/polars/py-polars/html/reference/lazyframe/index.html
output_df = input_df.collect()
print(output_df)
with s3.open(output_uri, mode="wb") as output_s3:
# persist the output dataframe back to S3 and return it
output_df.write_parquet(output_s3)