starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-10 16:05:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: S3 file picker as a drawer (#2640)

Browse Source 
* feat: Pick a S3 file drawer

* Add endpoint to generate simple file preview

* FE prototype for file preview

* More info on the preview

* Small fix

* Fix BE compile

* fix BE compile

* remove weird vite.config.js file

* Fix fronte NodeJS.Timer -> NodeJS.Timeout and rename dataset to file for S3

* Add EE restrictions and polish FE

* Fix merge conflicst

* replace dataset with s3 object

* Update wording

* BE throws an error when bucket too large in CE

* regenerate Cargo.lock

* Fix typo in error message
This commit is contained in:
Guillaume Bouvignies
2023-11-20 10:26:36 +01:00
committed by GitHub
parent 895fedc8cb
commit 624b4d0e98
10 changed files with 1652 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+676 -1
View File
@@ -186,6 +186,15 @@ dependencies = [
"zip",
]
[[package]]
name = "argminmax"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "202108b46429b765ef483f8a24d5c46f48c14acfdacc086dd4ab6dddf6bcdbd2"
dependencies = [
"num-traits",
]
[[package]]
name = "argon2"
version = "0.5.2"
@@ -204,12 +213,28 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
[[package]]
name = "array-init-cursor"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76"
[[package]]
name = "arrayvec"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]]
name = "arrow-format"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07884ea216994cdc32a2d5f8274a8bee979cfe90274b83f86f440866ee3132c7"
dependencies = [
"planus",
"serde",
]
[[package]]
name = "ast_node"
version = "0.9.5"
@@ -403,6 +428,28 @@ dependencies = [
"wasm-bindgen-futures",
]
[[package]]
name = "async-stream"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
dependencies = [
"async-stream-impl",
"futures-core",
"pin-project-lite",
]
[[package]]
name = "async-stream-impl"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.39",
]
[[package]]
name = "async-stripe"
version = "0.25.2"
@@ -491,6 +538,12 @@ dependencies = [
"num-traits",
]
[[package]]
name = "atoi_simd"
version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc41b65e01b6851bdcd2d741824e6b310d571396bf3915e31e4792034ee65126"
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -1558,6 +1611,18 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "comfy-table"
version = "7.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686"
dependencies = [
"crossterm",
"strum",
"strum_macros",
"unicode-width",
]
[[package]]
name = "concurrent-queue"
version = "2.3.0"
@@ -1800,6 +1865,28 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossterm"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df"
dependencies = [
"bitflags 2.4.1",
"crossterm_winapi",
"libc",
"parking_lot 0.12.1",
"winapi",
]
[[package]]
name = "crossterm_winapi"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
dependencies = [
"winapi",
]
[[package]]
name = "crunchy"
version = "0.2.2"
@@ -2312,6 +2399,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "doc-comment"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "dotenv"
version = "0.15.0"
@@ -2494,6 +2587,18 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "enum_dispatch"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f33313078bb8d4d05a2733a94ac4c2d8a0df9a2b84424ebf4f33bfc224a890e"
dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.39",
]
[[package]]
name = "enumflags2"
version = "0.7.8"
@@ -2550,6 +2655,12 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "ethnum"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b90ca2580b73ab6a1f724b76ca11ab632df820fd6040c336200d2c1df7b3c82c"
[[package]]
name = "event-listener"
version = "2.5.3"
@@ -2583,6 +2694,18 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fast-float"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c"
[[package]]
name = "fastrand"
version = "1.9.0"
@@ -2670,6 +2793,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "foreign_vec"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673"
[[package]]
name = "form_urlencoded"
version = "1.2.0"
@@ -3233,6 +3362,7 @@ checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156"
dependencies = [
"ahash 0.8.6",
"allocator-api2",
"rayon",
]
[[package]]
@@ -3394,6 +3524,12 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.27"
@@ -3874,6 +4010,26 @@ dependencies = [
"hashbrown 0.14.2",
]
[[package]]
name = "lz4"
version = "1.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1"
dependencies = [
"libc",
"lz4-sys",
]
[[package]]
name = "lz4-sys"
version = "1.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "lzma-sys"
version = "0.1.20"
@@ -4130,6 +4286,28 @@ dependencies = [
"syn 2.0.39",
]
[[package]]
name = "multiversion"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2c7b9d7fe61760ce5ea19532ead98541f6b4c495d87247aff9826445cf6872a"
dependencies = [
"multiversion-macros",
"target-features",
]
[[package]]
name = "multiversion-macros"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26a83d8500ed06d68877e9de1dde76c1dbb83885dcdbda4ef44ccbc3fbda2ac8"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
"target-features",
]
[[package]]
name = "mysql-common-derive"
version = "0.30.2"
@@ -4266,6 +4444,24 @@ dependencies = [
"minimal-lexical",
]
[[package]]
name = "now"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d89e9874397a1f0a52fc1f197a8effd9735223cb2390e9dcc83ac6cd02923d0"
dependencies = [
"chrono",
]
[[package]]
name = "ntapi"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
dependencies = [
"winapi",
]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
@@ -4372,6 +4568,35 @@ dependencies = [
"memchr",
]
[[package]]
name = "object_store"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050"
dependencies = [
"async-trait",
"base64 0.21.5",
"bytes",
"chrono",
"futures",
"humantime",
"hyper",
"itertools 0.11.0",
"parking_lot 0.12.1",
"percent-encoding",
"quick-xml 0.31.0",
"rand 0.8.5",
"reqwest",
"ring 0.17.5",
"serde",
"serde_json",
"snafu",
"tokio",
"tracing",
"url",
"walkdir",
]
[[package]]
name = "once_cell"
version = "1.18.0"
@@ -4544,6 +4769,16 @@ dependencies = [
"windows-targets 0.48.5",
]
[[package]]
name = "parquet-format-safe"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f"
dependencies = [
"async-trait",
"futures",
]
[[package]]
name = "parse-zoneinfo"
version = "0.3.0"
@@ -4855,6 +5090,15 @@ version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "planus"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc1691dd09e82f428ce8d6310bd6d5da2557c82ff17694d2a32cad7242aea89f"
dependencies = [
"array-init-cursor",
]
[[package]]
name = "pmutil"
version = "0.5.3"
@@ -4877,6 +5121,328 @@ dependencies = [
"syn 2.0.39",
]
[[package]]
name = "polars"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df8e52f9236eb722da0990a70bbb1216dcc7a77bcb00c63439d2d982823e90d5"
dependencies = [
"getrandom 0.2.11",
"polars-core",
"polars-io",
"polars-lazy",
"polars-ops",
"polars-sql",
"polars-time",
"version_check",
]
[[package]]
name = "polars-arrow"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd503430a6d9779b07915d858865fe998317ef3cfef8973881f578ac5d4baae7"
dependencies = [
"ahash 0.8.6",
"arrow-format",
"atoi_simd",
"bytemuck",
"chrono",
"dyn-clone",
"either",
"ethnum",
"fast-float",
"foreign_vec",
"futures",
"getrandom 0.2.11",
"hashbrown 0.14.2",
"itoa",
"lz4",
"multiversion",
"num-traits",
"polars-error",
"polars-utils",
"rustc_version 0.4.0",
"ryu",
"simdutf8",
"streaming-iterator",
"strength_reduce",
"zstd 0.13.0",
]
[[package]]
name = "polars-core"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae73d5b8e55decde670caba1cc82b61f14bfb9a72503198f0997d657a98dcfd6"
dependencies = [
"ahash 0.8.6",
"bitflags 2.4.1",
"bytemuck",
"chrono",
"comfy-table",
"either",
"hashbrown 0.14.2",
"indexmap 2.1.0",
"num-traits",
"once_cell",
"polars-arrow",
"polars-error",
"polars-row",
"polars-utils",
"rand 0.8.5",
"rand_distr",
"rayon",
"regex",
"smartstring",
"thiserror",
"version_check",
"xxhash-rust",
]
[[package]]
name = "polars-error"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb0520d68eaa9993ae0c741409d1526beff5b8f48e1d73e4381616f8152cf488"
dependencies = [
"arrow-format",
"object_store",
"regex",
"simdutf8",
"thiserror",
]
[[package]]
name = "polars-io"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96e10a0745acd6009db64bef0ceb9e23a70b1c27b26a0a6517c91f3e6363bc06"
dependencies = [
"ahash 0.8.6",
"async-trait",
"atoi_simd",
"bytes",
"chrono",
"fast-float",
"flate2",
"futures",
"home",
"itoa",
"memchr",
"memmap2",
"num-traits",
"object_store",
"once_cell",
"percent-encoding",
"polars-arrow",
"polars-core",
"polars-error",
"polars-parquet",
"polars-time",
"polars-utils",
"rayon",
"regex",
"reqwest",
"ryu",
"simdutf8",
"smartstring",
"tokio",
"tokio-util",
"url",
"zstd 0.13.0",
]
[[package]]
name = "polars-lazy"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3555f759705be6dd0d3762d16a0b8787b2dc4da73b57465f3b2bf1a070ba8f20"
dependencies = [
"ahash 0.8.6",
"bitflags 2.4.1",
"futures",
"glob",
"once_cell",
"polars-arrow",
"polars-core",
"polars-io",
"polars-ops",
"polars-pipe",
"polars-plan",
"polars-time",
"polars-utils",
"rayon",
"smartstring",
"tokio",
"version_check",
]
[[package]]
name = "polars-ops"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a7eb218296aaa7f79945f08288ca32ca3cf25fa505649eeee689ec21eebf636"
dependencies = [
"ahash 0.8.6",
"argminmax",
"bytemuck",
"either",
"hashbrown 0.14.2",
"indexmap 2.1.0",
"memchr",
"num-traits",
"polars-arrow",
"polars-core",
"polars-error",
"polars-utils",
"rayon",
"regex",
"smartstring",
"version_check",
]
[[package]]
name = "polars-parquet"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "146010e4b7dd4d2d0e58ddc762f6361f77d7a0385c54471199370c17164f67dd"
dependencies = [
"ahash 0.8.6",
"async-stream",
"base64 0.21.5",
"brotli",
"ethnum",
"flate2",
"futures",
"lz4",
"num-traits",
"parquet-format-safe",
"polars-arrow",
"polars-error",
"polars-utils",
"seq-macro",
"simdutf8",
"snap",
"streaming-decompression",
"zstd 0.13.0",
]
[[package]]
name = "polars-pipe"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66094e7df64c932a9a7bdfe7df0c65efdcb192096e11a6a765a9778f78b4bdec"
dependencies = [
"crossbeam-channel",
"crossbeam-queue",
"enum_dispatch",
"futures",
"hashbrown 0.14.2",
"num-traits",
"polars-arrow",
"polars-core",
"polars-io",
"polars-ops",
"polars-plan",
"polars-row",
"polars-utils",
"rayon",
"smartstring",
"tokio",
"version_check",
]
[[package]]
name = "polars-plan"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10e32a0958ef854b132bad7f8369cb3237254635d5e864c99505bc0bc1035fbc"
dependencies = [
"ahash 0.8.6",
"bytemuck",
"once_cell",
"percent-encoding",
"polars-arrow",
"polars-core",
"polars-io",
"polars-ops",
"polars-parquet",
"polars-time",
"polars-utils",
"rayon",
"regex",
"smartstring",
"strum_macros",
"version_check",
]
[[package]]
name = "polars-row"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d135ab81cac2906ba74ea8984c7e6025d081ae5867615bcefb4d84dfdb456dac"
dependencies = [
"polars-arrow",
"polars-error",
"polars-utils",
]
[[package]]
name = "polars-sql"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8dbd7786849a5e3ad1fde188bf38141632f626e3a57319b0bbf7a5f1d75519e"
dependencies = [
"polars-arrow",
"polars-core",
"polars-error",
"polars-lazy",
"polars-plan",
"rand 0.8.5",
"serde",
"serde_json",
"sqlparser",
]
[[package]]
name = "polars-time"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae56f79e9cedd617773c1c8f5ca84a31a8b1d593714959d5f799e7bdd98fe51"
dependencies = [
"atoi 2.0.0",
"chrono",
"now",
"once_cell",
"polars-arrow",
"polars-core",
"polars-error",
"polars-ops",
"polars-utils",
"regex",
"smartstring",
]
[[package]]
name = "polars-utils"
version = "0.35.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da6ce68169fe61d46958c8eab7447360f30f2f23f6e24a0ce703a14b0a3cfbfc"
dependencies = [
"ahash 0.8.6",
"bytemuck",
"hashbrown 0.14.2",
"indexmap 2.1.0",
"num-traits",
"once_cell",
"polars-error",
"rayon",
"smartstring",
"sysinfo",
"version_check",
]
[[package]]
name = "polling"
version = "2.8.0"
@@ -5237,6 +5803,16 @@ dependencies = [
"serde",
]
[[package]]
name = "quick-xml"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "quick_cache"
version = "0.4.0"
@@ -5977,7 +6553,7 @@ dependencies = [
"openssl-probe",
"openssl-sys",
"pkg-config",
"quick-xml",
"quick-xml 0.28.2",
"rand 0.8.5",
"serde",
"thiserror",
@@ -6479,6 +7055,28 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4b756ac662e92a0e5b360349bea5f0b0784d4be4541eff2972049dfdfd7f862"
[[package]]
name = "snafu"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6"
dependencies = [
"doc-comment",
"snafu-derive",
]
[[package]]
name = "snafu-derive"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "snap"
version = "1.1.0"
@@ -6589,6 +7187,15 @@ dependencies = [
"unicode_categories",
]
[[package]]
name = "sqlparser"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7"
dependencies = [
"log",
]
[[package]]
name = "sqlx"
version = "0.6.3"
@@ -6917,6 +7524,27 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "streaming-decompression"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3"
dependencies = [
"fallible-streaming-iterator",
]
[[package]]
name = "streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
[[package]]
name = "strength_reduce"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
[[package]]
name = "string_cache"
version = "0.8.7"
@@ -7401,6 +8029,20 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "sysinfo"
version = "0.29.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5"
dependencies = [
"cfg-if",
"core-foundation-sys",
"libc",
"ntapi",
"once_cell",
"winapi",
]
[[package]]
name = "system-configuration"
version = "0.5.1"
@@ -7439,6 +8081,12 @@ dependencies = [
"xattr",
]
[[package]]
name = "target-features"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfb5fa503293557c5158bd215fdc225695e567a77e453f5d4452a50a193969bd"
[[package]]
name = "tempfile"
version = "3.8.1"
@@ -8749,6 +9397,9 @@ dependencies = [
"magic-crypt",
"mail-send",
"mime_guess",
"object_store",
"polars",
"polars-io",
"prometheus",
"quick_cache",
"rand 0.8.5",
@@ -9240,6 +9891,12 @@ version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
[[package]]
name = "xxhash-rust"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b"
[[package]]
name = "xz2"
version = "0.1.7"
@@ -9358,6 +10015,15 @@ dependencies = [
"zstd-safe 6.0.6",
]
[[package]]
name = "zstd"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110"
dependencies = [
"zstd-safe 7.0.0",
]
[[package]]
name = "zstd-safe"
version = "5.0.2+zstd.1.5.2"
@@ -9378,6 +10044,15 @@ dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-safe"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.9+zstd.1.5.5"
backend/windmill-api/Cargo.toml
+3
View File
@@ -84,3 +84,6 @@ candle-transformers.workspace = true
candle-nn.workspace = true
aws-sdk-s3 = "0.36.0"
aws-config = "0.57.2"
polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv"] }
polars-io = { version = "0.35.2", features = ["csv"] }
object_store = { version = "0.8.0", features = ["aws"] }
backend/windmill-api/openapi-deref.yaml
+157 -58
View File
@@ -1,6 +1,6 @@
openapi: 3.0.3
info:
version: 1.206.0
version: 1.210.0
title: Windmill API
contact:
name: Windmill Team
@@ -1534,6 +1534,24 @@ paths:
properties:
deploy_to:
type: string
/w/{workspace}/workspaces/is_premium:
get:
summary: get if workspace is premium
operationId: getIsPremium
tags:
- workspace
parameters:
- name: workspace
in: path
required: true
schema: *ref_0
responses:
'200':
description: status
content:
application/json:
schema:
type: boolean
/w/{workspace}/workspaces/premium_info:
get:
summary: get premium info
@@ -1557,6 +1575,8 @@ paths:
type: boolean
usage:
type: number
seats:
type: number
required:
- premium
/w/{workspace}/workspaces/edit_slack_command:
@@ -1670,6 +1690,8 @@ paths:
properties:
operator:
type: boolean
invite_all:
type: boolean
responses:
'200':
description: status
@@ -3266,32 +3288,32 @@ paths:
id:
type: string
value:
oneOf: &ref_177
oneOf: &ref_179
- type: object
properties: &ref_161
properties: &ref_163
input_transforms:
type: object
additionalProperties:
oneOf: &ref_24
- type: object
properties: &ref_157
properties: &ref_159
value: {}
type:
type: string
enum:
- javascript
required: &ref_158
required: &ref_160
- expr
- type
- type: object
properties: &ref_159
properties: &ref_161
expr:
type: string
type:
type: string
enum:
- javascript
required: &ref_160
required: &ref_162
- expr
- type
discriminator: &ref_25
@@ -3331,13 +3353,13 @@ paths:
type: number
concurrency_time_window_s:
type: number
required: &ref_162
required: &ref_164
- type
- content
- language
- input_transforms
- type: object
properties: &ref_163
properties: &ref_165
input_transforms:
type: object
additionalProperties:
@@ -3351,12 +3373,12 @@ paths:
type: string
enum:
- script
required: &ref_164
required: &ref_166
- type
- path
- input_transforms
- type: object
properties: &ref_165
properties: &ref_167
input_transforms:
type: object
additionalProperties:
@@ -3368,12 +3390,12 @@ paths:
type: string
enum:
- flow
required: &ref_166
required: &ref_168
- type
- path
- input_transforms
- type: object
properties: &ref_167
properties: &ref_169
modules:
type: array
items:
@@ -3395,13 +3417,13 @@ paths:
type: boolean
parallelism:
type: integer
required: &ref_168
required: &ref_170
- modules
- iterator
- skip_failures
- type
- type: object
properties: &ref_169
properties: &ref_171
branches:
type: array
items:
@@ -3432,12 +3454,12 @@ paths:
type: string
enum:
- branchone
required: &ref_170
required: &ref_172
- branches
- default
- type
- type: object
properties: &ref_171
properties: &ref_173
branches:
type: array
items:
@@ -3462,28 +3484,28 @@ paths:
- branchall
parallel:
type: boolean
required: &ref_172
- branches
- type
- type: object
properties: &ref_173
type:
type: string
enum:
- identity
flow:
type: boolean
required: &ref_174
- branches
- type
- type: object
properties: &ref_175
type:
type: string
enum:
- graphql
- identity
flow:
type: boolean
required: &ref_176
- type
discriminator: &ref_178
- type: object
properties: &ref_177
type:
type: string
enum:
- graphql
required: &ref_178
- type
discriminator: &ref_180
propertyName: type
mapping:
rawscript: '#/components/schemas/RawScript'
@@ -3539,7 +3561,7 @@ paths:
type: number
retry:
type: object
properties: &ref_179
properties: &ref_181
constant:
type: object
properties:
@@ -5132,6 +5154,8 @@ paths:
type: boolean
priority:
type: integer
dedicated_worker:
type: boolean
required: &ref_144
- path
- edited_by
@@ -5279,6 +5303,8 @@ paths:
type: boolean
priority:
type: integer
dedicated_worker:
type: boolean
required:
- path
- type: object
@@ -9465,10 +9491,10 @@ paths:
content:
application/json:
schema: {}
/w/{workspace}/job_helpers/list_stored_datasets:
post:
summary: List the dataset keys available in the worspace datasets storage
operationId: polarsConnectionSettings
/w/{workspace}/job_helpers/list_stored_files:
get:
summary: List the file keys available in the worspace files storage (S3)
operationId: listStoredFiles
tags:
- helpers
parameters:
@@ -9478,23 +9504,92 @@ paths:
schema: *ref_0
responses:
'200':
description: Connection settings
description: List of file keys
content:
application/json:
schema:
type: object
properties:
dataset_keys:
file_count:
type: integer
windmill_large_files:
type: array
items:
type: object
properties: &ref_155
s3:
type: string
s3_bucket:
type: string
required: &ref_156
- s3
required:
- file_count
- windmill_large_files
/w/{workspace}/job_helpers/load_file_preview:
get:
summary: Load a preview of the file
operationId: loadFilePreview
tags:
- helpers
parameters:
- name: workspace
in: path
required: true
schema: *ref_0
- name: file_key
in: query
required: true
schema:
type: string
- name: from
in: query
schema:
type: integer
- name: length
in: query
schema:
type: integer
- name: separator
in: query
schema:
type: string
responses:
'200':
description: FilePreview
content:
application/json:
schema:
type: object
properties: &ref_157
mime_type:
type: string
last_modified:
type: string
format: date-time
size_in_bytes:
type: integer
expires:
type: string
format: date-time
version_id:
type: string
content_preview:
type: object
properties:
msg:
type: string
content:
type: string
content_type:
type: string
enum:
- RawText
- Csv
- Parquet
- Unknown
required:
- content_type
required: &ref_158
- content_preview
components:
securitySchemes:
bearerAuth:
@@ -10100,6 +10195,10 @@ components:
type: object
properties: *ref_155
required: *ref_156
WindmillFilePreview:
type: object
properties: *ref_157
required: *ref_158
S3Resource:
type: object
properties:
@@ -10124,58 +10223,58 @@ components:
- useSSL
- pathStyle
StaticTransform:
type: object
properties: *ref_157
required: *ref_158
JavascriptTransform:
type: object
properties: *ref_159
required: *ref_160
JavascriptTransform:
type: object
properties: *ref_161
required: *ref_162
InputTransform:
oneOf: *ref_24
discriminator: *ref_25
RawScript:
type: object
properties: *ref_161
required: *ref_162
PathScript:
type: object
properties: *ref_163
required: *ref_164
PathFlow:
PathScript:
type: object
properties: *ref_165
required: *ref_166
PathFlow:
type: object
properties: *ref_167
required: *ref_168
FlowModule:
type: object
properties: *ref_26
required: *ref_27
ForloopFlow:
type: object
properties: *ref_167
required: *ref_168
BranchOne:
type: object
properties: *ref_169
required: *ref_170
BranchAll:
BranchOne:
type: object
properties: *ref_171
required: *ref_172
Identity:
BranchAll:
type: object
properties: *ref_173
required: *ref_174
Graphql:
Identity:
type: object
properties: *ref_175
required: *ref_176
Graphql:
type: object
properties: *ref_177
required: *ref_178
FlowModuleValue:
oneOf: *ref_177
discriminator: *ref_178
oneOf: *ref_179
discriminator: *ref_180
Retry:
type: object
properties: *ref_179
properties: *ref_181
FlowValue:
type: object
properties: *ref_48
backend/windmill-api/openapi.yaml
+95 -50
View File
@@ -2192,8 +2192,7 @@ paths:
/w/{workspace}/resources/get_value_interpolated/{path}:
get:
summary:
get resource interpolated (variables and resources are fully unrolled)
summary: get resource interpolated (variables and resources are fully unrolled)
operationId: getResourceValueInterpolated
tags:
- resource
@@ -2881,8 +2880,7 @@ paths:
schema:
type: string
- name: first_parent_hash
description:
mask to filter scripts whom first direct parent has exact hash
description: mask to filter scripts whom first direct parent has exact hash
in: query
schema:
type: string
@@ -3077,8 +3075,7 @@ paths:
/workers/custom_tags:
get:
summary:
get all instance custom tags (tags are used to dispatch jobs to
summary: get all instance custom tags (tags are used to dispatch jobs to
different worker groups)
operationId: getCustomTags
tags:
@@ -3129,8 +3126,7 @@ paths:
/w/{workspace}/scripts/delete/h/{hash}:
post:
summary:
delete script by hash (erase content but keep hash, require admin)
summary: delete script by hash (erase content but keep hash, require admin)
operationId: deleteScriptByHash
tags:
- script
@@ -3322,16 +3318,14 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
- $ref: "#/components/parameters/ParentJob"
- $ref: "#/components/parameters/NewJobId"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4304,8 +4298,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4313,8 +4306,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the flow owner (default false)
description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4366,8 +4358,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4375,8 +4366,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the flow owner (default false)
description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4414,8 +4404,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4423,8 +4412,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4455,8 +4443,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4489,8 +4476,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4705,8 +4691,7 @@ paths:
- job
responses:
"200":
description:
the timestamp of the db that can be used to compute the drift
description: the timestamp of the db that can be used to compute the drift
content:
application/json:
schema:
@@ -4942,8 +4927,7 @@ paths:
/w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
get:
summary:
get resume urls given a job_id, resume_id and a nonce to resume a flow
summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
operationId: getResumeUrls
tags:
- job
@@ -5494,8 +5478,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
description:
only list the groups the user is member of (default false)
description: only list the groups the user is member of (default false)
schema:
type: boolean
responses:
@@ -5683,8 +5666,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
description:
only list the folders the user is member of (default false)
description: only list the folders the user is member of (default false)
schema:
type: boolean
responses:
@@ -6354,26 +6336,64 @@ paths:
application/json:
schema: {}
/w/{workspace}/job_helpers/list_stored_datasets:
post:
summary: List the dataset keys available in the worspace datasets storage
operationId: polarsConnectionSettings
/w/{workspace}/job_helpers/list_stored_files:
get:
summary: List the file keys available in the worspace files storage (S3)
operationId: listStoredFiles
tags:
- helpers
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: Connection settings
description: List of file keys
content:
application/json:
schema:
type: object
properties:
dataset_keys:
file_count:
type: integer
windmill_large_files:
type: array
items:
$ref: "#/components/schemas/WindmillLargeFile"
required:
- file_count
- windmill_large_files
/w/{workspace}/job_helpers/load_file_preview:
get:
summary: Load a preview of the file
operationId: loadFilePreview
tags:
- helpers
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: file_key
in: query
required: true
schema:
type: string
- name: from
in: query
schema:
type: integer
- name: length
in: query
schema:
type: integer
- name: separator
in: query
schema:
type: string
responses:
"200":
description: FilePreview
content:
application/json:
schema:
$ref: "#/components/schemas/WindmillFilePreview"
components:
securitySchemes:
@@ -6467,8 +6487,7 @@ components:
type: integer
PerPage:
name: per_page
description:
number of items to return for a given page (default 30, max 100)
description: number of items to return for a given page (default 30, max 100)
in: query
schema:
type: integer
@@ -6602,8 +6621,7 @@ components:
type: boolean
ArgsFilter:
name: args
description:
filter on jobs containing those args as a json subset (@> in postgres)
description: filter on jobs containing those args as a json subset (@> in postgres)
in: query
schema:
type: string
@@ -6615,8 +6633,7 @@ components:
type: string
ResultFilter:
name: result
description:
filter on jobs containing those result as a json subset (@> in postgres)
description: filter on jobs containing those result as a json subset (@> in postgres)
in: query
schema:
type: string
@@ -8372,11 +8389,39 @@ components:
properties:
s3:
type: string
s3_bucket:
type: string
required:
- s3
WindmillFilePreview:
type: object
properties:
mime_type:
type: string
last_modified:
type: string
format: date-time
size_in_bytes:
type: integer
expires:
type: string
format: date-time
version_id:
type: string
content_preview:
type: object
properties:
msg:
type: string
content:
type: string
content_type:
type: string
enum: ["RawText", "Csv", "Parquet", "Unknown"]
required:
- content_type
required:
- content_preview
S3Resource:
type: object
properties:
backend/windmill-api/src/job_helpers.rs
+350 -22
View File
@@ -1,13 +1,27 @@
use std::cmp;
use crate::{
db::DB, resources::transform_json_value, users::Tokened, workspaces::LargeFileStorage,
};
use aws_sdk_s3::config::{Credentials, Region};
use axum::{
extract::Path,
extract::{Path, Query},
routing::{get, post},
Extension, Json, Router,
};
use hyper::http;
use object_store::ClientConfigKey;
use polars::{
io::{
cloud::{AmazonS3ConfigKey, CloudOptions},
SerReader,
},
lazy::{
dsl::col,
frame::{LazyFrame, ScanArgsParquet},
},
prelude::CsvReader,
};
use serde::{Deserialize, Serialize};
use tower_http::cors::{Any, CorsLayer};
use windmill_common::{db::UserDB, error};
@@ -31,8 +45,12 @@ pub fn workspaced_service() -> Router {
)
.route("/test_connection", get(test_connection).layer(cors.clone()))
.route(
"/list_stored_datasets",
get(list_stored_datasets).layer(cors.clone()),
"/list_stored_files",
get(list_stored_files).layer(cors.clone()),
)
.route(
"/load_file_preview",
get(load_file_preview).layer(cors.clone()),
)
}
@@ -133,13 +151,13 @@ async fn polars_connection_settings(
#[derive(Serialize)]
struct ListStoredDatasetsResponse {
file_count: usize,
windmill_large_files: Vec<WindmillLargeFile>,
}
#[derive(Serialize, Clone)]
struct WindmillLargeFile {
s3: String,
s3_bucket: Option<String>,
}
async fn test_connection(
@@ -156,7 +174,9 @@ async fn test_connection(
));
}
let s3_resource = s3_resource_opt.unwrap();
let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
"No files storage resource defined at the workspace level".to_string(),
))?;
let s3_client = build_s3_client(&s3_resource);
s3_client
.list_objects()
@@ -168,7 +188,7 @@ async fn test_connection(
return Ok(Json(()));
}
async fn list_stored_datasets(
async fn list_stored_files(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
@@ -177,7 +197,9 @@ async fn list_stored_datasets(
) -> error::JsonResult<ListStoredDatasetsResponse> {
let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?;
let s3_resource = s3_resource_opt.unwrap();
let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
"No files storage resource defined at the workspace level".to_string(),
))?;
let s3_client = build_s3_client(&s3_resource);
let mut stored_datasets = Vec::<WindmillLargeFile>::new();
@@ -206,10 +228,7 @@ async fn list_stored_datasets(
.map(|object| object.key())
.map(Option::unwrap)
.map(&str::to_string)
.map(|object_key| WindmillLargeFile {
s3: object_key.clone(),
s3_bucket: Some(s3_bucket.clone()),
})
.map(|object_key| WindmillLargeFile { s3: object_key.clone() })
.collect::<Vec<WindmillLargeFile>>();
stored_datasets.extend(page_datasets.clone());
@@ -219,11 +238,159 @@ async fn list_stored_datasets(
}
}
#[cfg(not(feature = "enterprise"))]
if stored_datasets.len() > 10 {
return Err(error::Error::ExecutionErr(
"The workspace s3 bucket contains more than 10 files. Consider upgrading to Windmill Enterprise Edition to continue to use this feature, "
.to_string(),
));
}
return Ok(Json(ListStoredDatasetsResponse {
file_count: stored_datasets.len(), // TODO: for now, no pagination. Add in the future to support large buckets
windmill_large_files: stored_datasets,
}));
}
#[derive(Deserialize)]
struct LoadFilePreviewQuery {
pub file_key: String,
pub from: Option<usize>,
pub length: Option<usize>,
pub separator: Option<String>,
}
#[derive(Serialize)]
struct LoadFilePreviewResponse {
pub mime_type: Option<String>,
pub size_in_bytes: Option<i64>,
pub last_modified: Option<chrono::DateTime<chrono::Utc>>,
pub expires: Option<chrono::DateTime<chrono::Utc>>,
pub version_id: Option<String>,
pub content_preview: FileContentPreview,
}
#[derive(Serialize)]
struct FileContentPreview {
pub content: Option<String>,
pub content_type: WindmillContentType,
pub msg: Option<String>,
}
#[derive(Serialize)]
enum WindmillContentType {
RawText,
Csv,
Parquet,
Unknown,
}
async fn load_file_preview(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Tokened { token }: Tokened,
Path(w_id): Path<String>,
Query(query): Query<LoadFilePreviewQuery>,
) -> error::JsonResult<LoadFilePreviewResponse> {
let file_key = query.file_key.clone();
let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?;
let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr(
"No files storage resource defined at the workspace level".to_string(),
))?;
let s3_client = build_s3_client(&s3_resource);
let s3_bucket = s3_resource.bucket.clone();
let s3_object_metadata = s3_client
.head_object()
.bucket(&s3_bucket)
.key(&file_key)
.send()
.await
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
let file_chunk_length = cmp::min(
query.length.unwrap_or(8 * 1024 * 1024) as i64, // no more than 8MB per chunk by default
s3_object_metadata.content_length() - query.from.unwrap_or(0) as i64,
);
let object_content_type = s3_object_metadata.content_type();
let content_type: WindmillContentType;
let content_preview = match object_content_type {
Some("application/json") | Some("application/x-yaml") => {
content_type = WindmillContentType::RawText;
read_s3_text_object_head(
&s3_client,
&s3_bucket,
&file_key,
query.from.unwrap_or(0),
file_chunk_length, // 1KB by default
)
.await
}
Some("text/csv") => {
content_type = WindmillContentType::Csv;
read_s3_csv_object_head(&s3_client, &s3_bucket, &file_key, query.separator).await
}
Some("application/octet-stream") => {
if file_key.to_lowercase().ends_with(".parquet") {
content_type = WindmillContentType::Parquet;
read_s3_parquet_object_head(&s3_resource, &file_key).await
} else {
content_type = WindmillContentType::Unknown;
Err(error::Error::ExecutionErr(
"Preview is not available for content of type application/octet-stream"
.to_string(),
))
}
}
Some(mt) if mt.starts_with("text/") => {
content_type = WindmillContentType::RawText;
read_s3_text_object_head(
&s3_client,
&s3_bucket,
&file_key,
query.from.unwrap_or(0),
file_chunk_length,
)
.await
}
_ => {
content_type = WindmillContentType::Unknown;
Err(error::Error::ExecutionErr(
"Preview is not available for content of type application/octet-stream".to_string(),
))
}
};
let response = LoadFilePreviewResponse {
mime_type: s3_object_metadata.content_type().map(&str::to_string),
size_in_bytes: Some(s3_object_metadata.content_length()),
last_modified: s3_object_metadata
.last_modified()
.map(|dt| chrono::DateTime::from_timestamp(dt.secs(), dt.subsec_nanos()))
.flatten(),
expires: s3_object_metadata
.expires()
.map(|dt| chrono::DateTime::from_timestamp(dt.secs(), dt.subsec_nanos()))
.flatten(),
version_id: s3_object_metadata.version_id().map(&str::to_string),
content_preview: match content_preview {
Ok(content) => {
FileContentPreview { content_type: content_type, content: Some(content), msg: None }
}
Err(err) => FileContentPreview {
content_type: content_type,
content: None,
msg: Some(err.to_string()),
},
},
};
return Ok(Json(response));
}
async fn get_workspace_s3_resource<'c>(
authed: &ApiAuthed,
user_db: &UserDB,
@@ -282,18 +449,9 @@ async fn get_workspace_s3_resource<'c>(
fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client {
let s3_resource = s3_resource_ref.clone();
let endpoint_with_prefix = if s3_resource.endpoint.starts_with("http://")
|| s3_resource.endpoint.starts_with("https://")
{
s3_resource.endpoint.clone()
} else if s3_resource.use_ssl {
format!("https://{}", s3_resource.endpoint)
} else {
format!("http://{}", s3_resource.endpoint)
};
let endpoint = render_endpoint(&s3_resource);
let mut s3_config_builder = aws_sdk_s3::Config::builder()
.endpoint_url(endpoint_with_prefix)
.endpoint_url(endpoint)
.region(Region::new(s3_resource.region));
if s3_resource.access_key.is_some() {
s3_config_builder = s3_config_builder.credentials_provider(Credentials::new(
@@ -310,3 +468,173 @@ fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client {
let s3_config = s3_config_builder.build();
return aws_sdk_s3::Client::from_conf(s3_config);
}
fn build_polars_s3_config(s3_resource_ref: &S3Resource) -> CloudOptions {
let s3_resource = s3_resource_ref.to_owned();
let mut s3_configs: Vec<(AmazonS3ConfigKey, String)> = vec![
(AmazonS3ConfigKey::Region, s3_resource.region),
(AmazonS3ConfigKey::Bucket, s3_resource.bucket),
(
AmazonS3ConfigKey::Endpoint,
render_endpoint(s3_resource_ref),
),
(
AmazonS3ConfigKey::Client(ClientConfigKey::AllowHttp),
(!s3_resource.use_ssl).to_string(),
),
(
AmazonS3ConfigKey::VirtualHostedStyleRequest,
(!s3_resource.path_style).to_string(),
),
];
if let Some(access_key) = s3_resource.access_key {
s3_configs.push((AmazonS3ConfigKey::AccessKeyId, access_key));
}
if let Some(secret_key) = s3_resource.secret_key {
s3_configs.push((AmazonS3ConfigKey::SecretAccessKey, secret_key));
}
return CloudOptions::default().with_aws(s3_configs);
}
fn render_endpoint(s3_resource: &S3Resource) -> String {
if s3_resource.endpoint.starts_with("http://") || s3_resource.endpoint.starts_with("https://") {
s3_resource.endpoint.clone()
} else if s3_resource.use_ssl {
format!("https://{}", s3_resource.endpoint)
} else {
format!("http://{}", s3_resource.endpoint)
}
}
async fn read_s3_text_object_head(
s3_client: &aws_sdk_s3::Client,
s3_bucket: &str,
file_key: &str,
from_char: usize,
length: i64,
) -> error::Result<String> {
let s3_object = s3_client
.get_object()
.range(format!("bytes={}-{}", from_char, length).to_string())
.bucket(s3_bucket)
.key(file_key)
.send()
.await
.map_err(|err| {
tracing::warn!("Error fetching text file from S3: {:?}", err);
error::Error::InternalErr(err.to_string())
})?;
let payload = s3_object
.body
.collect()
.await
.map_err(|err| {
tracing::warn!(
"Error reading raw text file {}. Error was: {:?}",
file_key,
err
);
error::Error::InternalErr("File encoding is not supported".to_string())
})?
.into_bytes()
.to_vec();
let file_header_str = String::from_utf8(payload).map_err(|err| {
tracing::warn!(
"Encoding of file {} unsupported. Error was: {:?}",
file_key,
err
);
error::Error::InternalErr("File encoding is not supported".to_string())
})?;
return Ok(file_header_str);
}
async fn read_s3_parquet_object_head(
s3_resource_ref: &S3Resource,
file_key: &str,
) -> error::Result<String> {
let s3_cloud_config = build_polars_s3_config(s3_resource_ref);
let args: ScanArgsParquet = ScanArgsParquet {
n_rows: Some(1),
cache: false,
parallel: polars::io::parquet::ParallelStrategy::Auto,
rechunk: false,
row_count: None,
low_memory: false,
use_statistics: false,
hive_partitioning: false,
cloud_options: Some(s3_cloud_config),
};
let file_key_clone = file_key.to_string();
let s3_bucket_clone = s3_resource_ref.bucket.to_string();
let polars_df_result = tokio::task::spawn_blocking(move || {
let s3_file_key = format!("s3://{}/{}", s3_bucket_clone, file_key_clone);
let lzdf_result = LazyFrame::scan_parquet(s3_file_key, args);
match lzdf_result {
Err(err) => {
tracing::warn!("Error fetching parquet file from S3: {:?}", err);
return Err(error::Error::InternalErr(err.to_string()));
}
Ok(lzdf) => {
let df = lzdf
.select(&[col("*")])
.limit(10) // for now read only first 10 lines
.collect()
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
return Ok(format!("{:?}", df).to_string());
}
}
})
.await
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
return polars_df_result;
}
async fn read_s3_csv_object_head(
s3_client: &aws_sdk_s3::Client,
s3_bucket: &str,
file_key: &str,
separator: Option<String>,
) -> error::Result<String> {
let separator_final = if let Some(separator_char) = separator {
if separator_char.len() != 1 {
return Err(error::Error::BadRequest(
"Separator must be a single character".to_string(),
));
}
separator_char.as_bytes()[0]
} else {
",".as_bytes()[0]
};
let s3_object = s3_client
.get_object()
.bucket(s3_bucket)
.range("bytes=0-33554432".to_string()) // safeguard - do not load CSV files larger than 32MB
.key(file_key)
.send()
.await
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
// TODO: polars does not seem to support lazy csv reader, unfortunately. We can implement it ourselves if needed
let file_content_bytes = s3_object
.body
.collect()
.await
.map_err(|err| error::Error::InternalErr(err.to_string()))?
.into_bytes();
let cursor = std::io::Cursor::new(file_content_bytes);
let csv_df = CsvReader::new(cursor)
.with_n_rows(Some(10)) // for now read only first 10 lines
.with_separator(separator_final)
.finish()
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
return Ok(format!("{:?}", csv_df).to_string());
}
backend/windmill-api/src/openai.rs
+2 -1
View File
@@ -126,8 +126,9 @@ async fn proxy(
"https://api.openai.com/v1"
};
let url = format!("{}/{}", base_url, openai_path);
let mut request = HTTP_CLIENT
.post(base_url.to_string() + "/" + &openai_path)
.post(url)
.header("content-type", "application/json")
.body(body);
frontend/src/lib/components/ArgInput.svelte
+38 -1
View File
@@ -1,7 +1,7 @@
<script lang="ts">
import type { SchemaProperty } from '$lib/common'
import { setInputCat as computeInputCat, emptyString } from '$lib/utils'
import { ChevronDown, DollarSign, Plus, X } from 'lucide-svelte'
import { ChevronDown, DollarSign, Pipette, Plus, X } from 'lucide-svelte'
import { createEventDispatcher } from 'svelte'
import autosize from 'svelte-autosize'
import Multiselect from 'svelte-multiselect'
@@ -25,6 +25,7 @@
import ArgEnum from './ArgEnum.svelte'
import ArrayTypeNarrowing from './ArrayTypeNarrowing.svelte'
import DateTimeInput from './DateTimeInput.svelte'
import S3FilePicker from './S3FilePicker.svelte'
export let label: string = ''
export let value: any
@@ -73,6 +74,8 @@
let error: string = ''
let s3FilePicker: S3FilePicker
let el: HTMLTextAreaElement | undefined = undefined
export let editor: SimpleEditor | undefined = undefined
@@ -191,6 +194,16 @@
$: validateInput(pattern, value, required)
</script>
<S3FilePicker
bind:this={s3FilePicker}
initialFileKey={value}
bind:selectedFileKey={value}
on:close={() => {
rawValue = JSON.stringify(value, null, 2)
editor?.setCode(rawValue)
}}
/>
<!-- svelte-ignore a11y-autofocus -->
<div class="flex flex-col w-full {minW ? 'min-w-[250px]' : ''}">
<div>
@@ -398,6 +411,30 @@
<span class="text-2xs text-tertiary">Loading resource types...</span>
{:else if inputCat == 'resource-object' && (resourceTypes == undefined || (format.split('-').length > 1 && resourceTypes.includes(format.substring('resource-'.length))))}
<ObjectResourceInput {disablePortal} {format} bind:value {showSchemaExplorer} />
{:else if inputCat == 'resource-object' && format.split('-').length > 1 && format.replace('resource-', '') == 's3object'}
<div class="flex flex-col w-full gap-1">
<JsonEditor
bind:editor
on:focus={(e) => {
dispatch('focus')
}}
code={JSON.stringify({ s3: '' }, null, 2)}
bind:value
/>
<Button
variant="border"
color="light"
size="xs"
btnClasses="mt-1"
on:click={() => {
s3FilePicker?.open?.()
}}
id="arg-input-file-picker"
startIcon={{ icon: Pipette }}
>
Choose an object from the catalog
</Button>
</div>
{:else if inputCat == 'object' || inputCat == 'resource-object'}
{#if properties && Object.keys(properties).length > 0}
<div class="p-4 pl-8 border rounded w-full">
frontend/src/lib/components/S3FilePicker.svelte
+314
View File
@@ -0,0 +1,314 @@
<script lang="ts">
import { File, FolderClosed, FolderOpen, RotateCw } from 'lucide-svelte'
import { workspaceStore } from '$lib/stores'
import { HelpersService } from '$lib/gen'
import { displayDate, displaySize, emptyString } from '$lib/utils'
import { Alert, Button, Drawer } from './common'
import DrawerContent from './common/drawer/DrawerContent.svelte'
import Section from './Section.svelte'
import { createEventDispatcher } from 'svelte'
import VirtualList from 'svelte-tiny-virtual-list'
import TableSimple from './TableSimple.svelte'
let workspaceSettingsInitialized = true
export let initialFileKey: { s3: string }
let initialFileKeyInternalCopy: { s3: string }
export let selectedFileKey: { s3: string } | undefined = undefined
let csvSeparatorDefaultChar: string = ','
let csvSeparatorChar: string = ','
let dispatch = createEventDispatcher()
let drawer: Drawer
let allFilesByKey: Record<
string,
{
type: 'folder' | 'leaf'
full_key: string
display_name: string
collapsed: boolean
parentPath: string | undefined
nestingLevel: number
}
> = {}
let displayedFileKeys: string[] = []
let listDivHeight: number = 0
let filePreview:
| {
file_key: string
mime_type: string | undefined
size: string | undefined
last_modified: string | undefined
content_preview: string | undefined
content_type: string | undefined
}
| undefined = undefined
async function loadFiles() {
let availableFiles = await HelpersService.listStoredFiles({ workspace: $workspaceStore! })
for (let i = 0; i < availableFiles.file_count; i++) {
let file_path = availableFiles.windmill_large_files[i]
let split_path = file_path.s3.split('/')
let parent_path: string | undefined = undefined
let current_path: string | undefined = undefined
let nestingLevel = 0
for (let i = 0; i < split_path.length; i++) {
parent_path = current_path
current_path =
current_path === undefined ? split_path[i] : current_path + '/' + split_path[i]
nestingLevel = i * 2
if (allFilesByKey[current_path] !== undefined) {
continue
}
allFilesByKey[current_path] = {
type: i === split_path.length - 1 ? 'leaf' : 'folder',
full_key: current_path,
display_name: split_path[i],
collapsed: false,
parentPath: parent_path,
nestingLevel: nestingLevel
}
displayedFileKeys.push(current_path)
}
}
displayedFileKeys = displayedFileKeys.sort()
}
async function loadFilePreview(file_key: string | undefined) {
if (file_key === undefined) {
return
}
let filePreviewRaw = await HelpersService.loadFilePreview({
workspace: $workspaceStore!,
fileKey: file_key,
from: undefined,
length: undefined,
separator: csvSeparatorChar
})
if (filePreviewRaw !== undefined) {
filePreview = {
file_key: file_key,
size: displaySize(filePreviewRaw.size_in_bytes),
mime_type: filePreviewRaw.mime_type,
last_modified: displayDate(filePreviewRaw.last_modified),
content_preview: filePreviewRaw.content_preview.content,
content_type: filePreviewRaw.content_preview.content_type
}
}
}
export async function open() {
reloadContent()
drawer.openDrawer?.()
}
async function reloadContent() {
initialFileKeyInternalCopy = { ...initialFileKey }
try {
await HelpersService.datasetStorageTestConnection({ workspace: $workspaceStore! })
} catch (e) {
console.error('Workspace not connected to S3 bucket: ', e)
workspaceSettingsInitialized = false
return
}
await loadFiles() // TODO: Potentially load only on the first open and add a refresh button
if (selectedFileKey !== undefined) {
if (allFilesByKey[selectedFileKey.s3] === undefined) {
selectedFileKey = { s3: '' }
} else {
loadFilePreview(selectedFileKey.s3)
}
}
}
async function selectAndClose() {
drawer.closeDrawer?.()
}
async function exit() {
if (initialFileKeyInternalCopy !== undefined) {
selectedFileKey = { ...initialFileKeyInternalCopy }
}
drawer.closeDrawer?.()
}
function selectItem(index: number, toggleCollapsed: boolean = true) {
csvSeparatorChar = csvSeparatorDefaultChar
let item_key = displayedFileKeys[index]
let item = allFilesByKey[item_key]
if (item.type === 'folder') {
if (toggleCollapsed) {
item.collapsed = !item.collapsed
}
if (item.collapsed) {
// Remove the element nested in that folder from displayed_file_keys
let elt_to_remove = 0
for (let i = index + 1; i < displayedFileKeys.length; i++) {
let file_key = displayedFileKeys[i]
if (file_key.startsWith(item_key + '/')) {
elt_to_remove += 1
} else {
break
}
}
if (elt_to_remove > 0) {
displayedFileKeys.splice(index + 1, elt_to_remove)
}
} else {
// Re-add the currently hidden element to displayed_file_keys
for (let file_key in allFilesByKey) {
let file_info = allFilesByKey[file_key]
if (file_info.parentPath === item_key) {
displayedFileKeys.push(file_key)
if (file_info.type === 'folder' && !file_info.collapsed) {
selectItem(displayedFileKeys.length - 1, false)
}
}
}
}
displayedFileKeys = displayedFileKeys.sort()
} else {
selectedFileKey = {
s3: item_key
}
loadFilePreview(selectedFileKey.s3)
}
}
</script>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<Drawer
bind:this={drawer}
on:close={() => {
dispatch('close')
}}
size="1200px"
>
<DrawerContent
title="Pick a file"
overflow_y={false}
on:close={exit}
tooltip="Files present in the Workspace S3 bucket. You can set the workspace S3 bucket in the settings."
documentationLink="https://www.windmill.dev/docs/integrations/s3"
>
{#if workspaceSettingsInitialized === false}
<Alert type="error" title="Workspace not connected to any S3 storage">
<div class="flex flex-row gap-x-1 w-full items-center">
<p class="text-clip grow min-w-0">
The workspace needs to be connected to an S3 storage to use this feature. You can <a
target="_blank"
href="/workspace_settings?tab=windmill_lfs">configure it here</a
>.
</p>
<Button
variant="border"
color="light"
on:click={reloadContent}
startIcon={{ icon: RotateCw }}
/>
</div>
</Alert>
{:else}
<div class="flex flex-row border rounded-md h-full" bind:clientHeight={listDivHeight}>
<div class="min-w-[30%] border-r">
{#if displayedFileKeys.length === 0}
<div class="p-4 text-tertiary text-xs text-center italic">
No files in the workspace S3 bucket
</div>
{:else}
<VirtualList
width="100%"
height={listDivHeight}
itemCount={displayedFileKeys.length}
itemSize={42}
>
<div slot="item" let:index let:style {style} class="hover:bg-surface-hover border">
{@const file_info = allFilesByKey[displayedFileKeys[index]]}
<div
on:click={() => selectItem(index)}
class={`flex flex-row h-full font-semibold text-xs items-center justify-start ${
selectedFileKey !== undefined && selectedFileKey.s3 === file_info.full_key
? 'bg-surface-hover'
: ''
} `}
>
<div
class={`flex flex-row w-full ml-${
2 + file_info.nestingLevel
} gap-2 h-full items-center`}
>
{#if file_info.type === 'folder'}
{#if file_info.collapsed}<FolderClosed />{:else}<FolderOpen />{/if}
{file_info.display_name}
{:else}
<File />
{file_info.display_name}
{/if}
</div>
</div>
</div></VirtualList
>
{/if}
</div>
<div class="flex flex-col h-full w-full overflow-auto">
{#if filePreview === undefined}
<div class="p-4">
<Section label="Select a file for preview" />
</div>
{:else}
<div class="p-4 gap-2">
<Section label={filePreview.file_key} />
<TableSimple
headers={['Last modified', 'Size', 'Type']}
data={[filePreview]}
keys={['last_modified', 'size', 'mime_type']}
/>
</div>
{/if}
<div class="flex flex-col h-full w-full overflow-auto text-xs p-4 bg-surface-secondary">
{#if filePreview !== undefined}
<div class="flex h-6 items-center text-tertiary mb-4">
{#if filePreview.content_type === 'Unknown'}
Type of file not supported for preview
{:else if filePreview.content_type === 'Csv'}
Previewing a {filePreview.content_type?.toLowerCase()} file. Change the separator:
<div class="inline-flex w-12 ml-2">
<select
class="h-8"
bind:value={csvSeparatorChar}
on:change={(e) => loadFilePreview(filePreview?.file_key)}
>
<option value=",">,</option>
<option value=";">;</option>
</select>
</div>
{:else}
Previewing a {filePreview.content_type?.toLowerCase()} file
{/if}
</div>
<pre class="grow whitespace-no-wrap break-words"
>{#if !emptyString(filePreview.content_preview)}{filePreview.content_preview}{:else if filePreview.content_type !== undefined}Preview impossible. If it's a CSV file, you can try changing the separator{/if}</pre
>
{/if}
</div>
</div>
</div>
{/if}
<div slot="actions" class="flex gap-1">
<Button
disable={selectedFileKey === undefined || emptyString(selectedFileKey.s3)}
on:click={selectAndClose}>Select</Button
>
</div>
</DrawerContent>
</Drawer>
frontend/src/lib/utils.ts
+14
View File
@@ -49,6 +49,20 @@ export function displayDate(dateString: string | Date | undefined, displaySecond
}
}
export function displaySize(sizeInBytes: number | undefined): string | undefined {
if (sizeInBytes === undefined) {
return undefined
}
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']
let size = sizeInBytes
let unit_idx = 0
while (unit_idx < units.length - 1 && size > 1024) {
size /= 1024
unit_idx += 1
}
return `${size.toFixed(1)}${units[unit_idx]}`
}
export function msToSec(ms: number | undefined, maximumFractionDigits?: number): string {
if (ms === undefined) return '?'
return (ms / 1000).toLocaleString(undefined, {
frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte
+3
View File
@@ -295,6 +295,9 @@
<Tab size="xs" value="openai">
<div class="flex gap-2 items-center my-1">Windmill AI</div>
</Tab>
<Tab size="xs" value="windmill_lfs">
<div class="flex gap-2 items-center my-1"> S3 Storage </div>
</Tab>
<Tab size="xs" value="export_delete">
<div class="flex gap-2 items-center my-1"> Delete Workspace </div>
</Tab>