From 624b4d0e9898dddcce3cb2ce989ce1f9e4736061 Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Mon, 20 Nov 2023 10:26:36 +0100 Subject: [PATCH] feat: S3 file picker as a drawer (#2640) * feat: Pick a S3 file drawer * Add endpoint to generate simple file preview * FE prototype for file preview * More info on the preview * Small fix * Fix BE compile * fix BE compile * remove weird vite.config.js file * Fix fronte NodeJS.Timer -> NodeJS.Timeout and rename dataset to file for S3 * Add EE restrictions and polish FE * Fix merge conflicst * replace dataset with s3 object * Update wording * BE throws an error when bucket too large in CE * regenerate Cargo.lock * Fix typo in error message --- backend/Cargo.lock | 677 +++++++++++++++++- backend/windmill-api/Cargo.toml | 3 + backend/windmill-api/openapi-deref.yaml | 215 ++++-- backend/windmill-api/openapi.yaml | 145 ++-- backend/windmill-api/src/job_helpers.rs | 372 +++++++++- backend/windmill-api/src/openai.rs | 3 +- frontend/src/lib/components/ArgInput.svelte | 39 +- .../src/lib/components/S3FilePicker.svelte | 314 ++++++++ frontend/src/lib/utils.ts | 14 + .../(logged)/workspace_settings/+page.svelte | 3 + 10 files changed, 1652 insertions(+), 133 deletions(-) create mode 100644 frontend/src/lib/components/S3FilePicker.svelte diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 9e948d0035..3fa05c2666 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -186,6 +186,15 @@ dependencies = [ "zip", ] +[[package]] +name = "argminmax" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202108b46429b765ef483f8a24d5c46f48c14acfdacc086dd4ab6dddf6bcdbd2" +dependencies = [ + "num-traits", +] + [[package]] name = "argon2" version = "0.5.2" @@ -204,12 +213,28 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" +[[package]] +name = "array-init-cursor" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" + [[package]] name = "arrayvec" version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "arrow-format" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07884ea216994cdc32a2d5f8274a8bee979cfe90274b83f86f440866ee3132c7" +dependencies = [ + "planus", + "serde", +] + [[package]] name = "ast_node" version = "0.9.5" @@ -403,6 +428,28 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "async-stripe" version = "0.25.2" @@ -491,6 +538,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi_simd" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc41b65e01b6851bdcd2d741824e6b310d571396bf3915e31e4792034ee65126" + [[package]] name = "atomic-waker" version = "1.1.2" @@ -1558,6 +1611,18 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "comfy-table" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" +dependencies = [ + "crossterm", + "strum", + "strum_macros", + "unicode-width", +] + [[package]] name = "concurrent-queue" version = "2.3.0" @@ -1800,6 +1865,28 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossterm" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +dependencies = [ + "bitflags 2.4.1", + "crossterm_winapi", + "libc", + "parking_lot 0.12.1", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -2312,6 +2399,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dotenv" version = "0.15.0" @@ -2494,6 +2587,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum_dispatch" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f33313078bb8d4d05a2733a94ac4c2d8a0df9a2b84424ebf4f33bfc224a890e" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "enumflags2" version = "0.7.8" @@ -2550,6 +2655,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ethnum" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b90ca2580b73ab6a1f724b76ca11ab632df820fd6040c336200d2c1df7b3c82c" + [[package]] name = "event-listener" version = "2.5.3" @@ -2583,6 +2694,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fast-float" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" + [[package]] name = "fastrand" version = "1.9.0" @@ -2670,6 +2793,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -3233,6 +3362,7 @@ checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ "ahash 0.8.6", "allocator-api2", + "rayon", ] [[package]] @@ -3394,6 +3524,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.27" @@ -3874,6 +4010,26 @@ dependencies = [ "hashbrown 0.14.2", ] +[[package]] +name = "lz4" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lzma-sys" version = "0.1.20" @@ -4130,6 +4286,28 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "multiversion" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2c7b9d7fe61760ce5ea19532ead98541f6b4c495d87247aff9826445cf6872a" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26a83d8500ed06d68877e9de1dde76c1dbb83885dcdbda4ef44ccbc3fbda2ac8" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "target-features", +] + [[package]] name = "mysql-common-derive" version = "0.30.2" @@ -4266,6 +4444,24 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "now" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89e9874397a1f0a52fc1f197a8effd9735223cb2390e9dcc83ac6cd02923d0" +dependencies = [ + "chrono", +] + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -4372,6 +4568,35 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050" +dependencies = [ + "async-trait", + "base64 0.21.5", + "bytes", + "chrono", + "futures", + "humantime", + "hyper", + "itertools 0.11.0", + "parking_lot 0.12.1", + "percent-encoding", + "quick-xml 0.31.0", + "rand 0.8.5", + "reqwest", + "ring 0.17.5", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -4544,6 +4769,16 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "parquet-format-safe" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" +dependencies = [ + "async-trait", + "futures", +] + [[package]] name = "parse-zoneinfo" version = "0.3.0" @@ -4855,6 +5090,15 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "planus" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1691dd09e82f428ce8d6310bd6d5da2557c82ff17694d2a32cad7242aea89f" +dependencies = [ + "array-init-cursor", +] + [[package]] name = "pmutil" version = "0.5.3" @@ -4877,6 +5121,328 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "polars" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8e52f9236eb722da0990a70bbb1216dcc7a77bcb00c63439d2d982823e90d5" +dependencies = [ + "getrandom 0.2.11", + "polars-core", + "polars-io", + "polars-lazy", + "polars-ops", + "polars-sql", + "polars-time", + "version_check", +] + +[[package]] +name = "polars-arrow" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd503430a6d9779b07915d858865fe998317ef3cfef8973881f578ac5d4baae7" +dependencies = [ + "ahash 0.8.6", + "arrow-format", + "atoi_simd", + "bytemuck", + "chrono", + "dyn-clone", + "either", + "ethnum", + "fast-float", + "foreign_vec", + "futures", + "getrandom 0.2.11", + "hashbrown 0.14.2", + "itoa", + "lz4", + "multiversion", + "num-traits", + "polars-error", + "polars-utils", + "rustc_version 0.4.0", + "ryu", + "simdutf8", + "streaming-iterator", + "strength_reduce", + "zstd 0.13.0", +] + +[[package]] +name = "polars-core" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae73d5b8e55decde670caba1cc82b61f14bfb9a72503198f0997d657a98dcfd6" +dependencies = [ + "ahash 0.8.6", + "bitflags 2.4.1", + "bytemuck", + "chrono", + "comfy-table", + "either", + "hashbrown 0.14.2", + "indexmap 2.1.0", + "num-traits", + "once_cell", + "polars-arrow", + "polars-error", + "polars-row", + "polars-utils", + "rand 0.8.5", + "rand_distr", + "rayon", + "regex", + "smartstring", + "thiserror", + "version_check", + "xxhash-rust", +] + +[[package]] +name = "polars-error" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb0520d68eaa9993ae0c741409d1526beff5b8f48e1d73e4381616f8152cf488" +dependencies = [ + "arrow-format", + "object_store", + "regex", + "simdutf8", + "thiserror", +] + +[[package]] +name = "polars-io" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96e10a0745acd6009db64bef0ceb9e23a70b1c27b26a0a6517c91f3e6363bc06" +dependencies = [ + "ahash 0.8.6", + "async-trait", + "atoi_simd", + "bytes", + "chrono", + "fast-float", + "flate2", + "futures", + "home", + "itoa", + "memchr", + "memmap2", + "num-traits", + "object_store", + "once_cell", + "percent-encoding", + "polars-arrow", + "polars-core", + "polars-error", + "polars-parquet", + "polars-time", + "polars-utils", + "rayon", + "regex", + "reqwest", + "ryu", + "simdutf8", + "smartstring", + "tokio", + "tokio-util", + "url", + "zstd 0.13.0", +] + +[[package]] +name = "polars-lazy" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3555f759705be6dd0d3762d16a0b8787b2dc4da73b57465f3b2bf1a070ba8f20" +dependencies = [ + "ahash 0.8.6", + "bitflags 2.4.1", + "futures", + "glob", + "once_cell", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-pipe", + "polars-plan", + "polars-time", + "polars-utils", + "rayon", + "smartstring", + "tokio", + "version_check", +] + +[[package]] +name = "polars-ops" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a7eb218296aaa7f79945f08288ca32ca3cf25fa505649eeee689ec21eebf636" +dependencies = [ + "ahash 0.8.6", + "argminmax", + "bytemuck", + "either", + "hashbrown 0.14.2", + "indexmap 2.1.0", + "memchr", + "num-traits", + "polars-arrow", + "polars-core", + "polars-error", + "polars-utils", + "rayon", + "regex", + "smartstring", + "version_check", +] + +[[package]] +name = "polars-parquet" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146010e4b7dd4d2d0e58ddc762f6361f77d7a0385c54471199370c17164f67dd" +dependencies = [ + "ahash 0.8.6", + "async-stream", + "base64 0.21.5", + "brotli", + "ethnum", + "flate2", + "futures", + "lz4", + "num-traits", + "parquet-format-safe", + "polars-arrow", + "polars-error", + "polars-utils", + "seq-macro", + "simdutf8", + "snap", + "streaming-decompression", + "zstd 0.13.0", +] + +[[package]] +name = "polars-pipe" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66094e7df64c932a9a7bdfe7df0c65efdcb192096e11a6a765a9778f78b4bdec" +dependencies = [ + "crossbeam-channel", + "crossbeam-queue", + "enum_dispatch", + "futures", + "hashbrown 0.14.2", + "num-traits", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-plan", + "polars-row", + "polars-utils", + "rayon", + "smartstring", + "tokio", + "version_check", +] + +[[package]] +name = "polars-plan" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10e32a0958ef854b132bad7f8369cb3237254635d5e864c99505bc0bc1035fbc" +dependencies = [ + "ahash 0.8.6", + "bytemuck", + "once_cell", + "percent-encoding", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-parquet", + "polars-time", + "polars-utils", + "rayon", + "regex", + "smartstring", + "strum_macros", + "version_check", +] + +[[package]] +name = "polars-row" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d135ab81cac2906ba74ea8984c7e6025d081ae5867615bcefb4d84dfdb456dac" +dependencies = [ + "polars-arrow", + "polars-error", + "polars-utils", +] + +[[package]] +name = "polars-sql" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dbd7786849a5e3ad1fde188bf38141632f626e3a57319b0bbf7a5f1d75519e" +dependencies = [ + "polars-arrow", + "polars-core", + "polars-error", + "polars-lazy", + "polars-plan", + "rand 0.8.5", + "serde", + "serde_json", + "sqlparser", +] + +[[package]] +name = "polars-time" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae56f79e9cedd617773c1c8f5ca84a31a8b1d593714959d5f799e7bdd98fe51" +dependencies = [ + "atoi 2.0.0", + "chrono", + "now", + "once_cell", + "polars-arrow", + "polars-core", + "polars-error", + "polars-ops", + "polars-utils", + "regex", + "smartstring", +] + +[[package]] +name = "polars-utils" +version = "0.35.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da6ce68169fe61d46958c8eab7447360f30f2f23f6e24a0ce703a14b0a3cfbfc" +dependencies = [ + "ahash 0.8.6", + "bytemuck", + "hashbrown 0.14.2", + "indexmap 2.1.0", + "num-traits", + "once_cell", + "polars-error", + "rayon", + "smartstring", + "sysinfo", + "version_check", +] + [[package]] name = "polling" version = "2.8.0" @@ -5237,6 +5803,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick_cache" version = "0.4.0" @@ -5977,7 +6553,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "pkg-config", - "quick-xml", + "quick-xml 0.28.2", "rand 0.8.5", "serde", "thiserror", @@ -6479,6 +7055,28 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4b756ac662e92a0e5b360349bea5f0b0784d4be4541eff2972049dfdfd7f862" +[[package]] +name = "snafu" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +dependencies = [ + "doc-comment", + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "snap" version = "1.1.0" @@ -6589,6 +7187,15 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "sqlparser" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" +dependencies = [ + "log", +] + [[package]] name = "sqlx" version = "0.6.3" @@ -6917,6 +7524,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "streaming-decompression" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" +dependencies = [ + "fallible-streaming-iterator", +] + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.8.7" @@ -7401,6 +8029,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "sysinfo" +version = "0.29.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "winapi", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -7439,6 +8081,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-features" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfb5fa503293557c5158bd215fdc225695e567a77e453f5d4452a50a193969bd" + [[package]] name = "tempfile" version = "3.8.1" @@ -8749,6 +9397,9 @@ dependencies = [ "magic-crypt", "mail-send", "mime_guess", + "object_store", + "polars", + "polars-io", "prometheus", "quick_cache", "rand 0.8.5", @@ -9240,6 +9891,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xxhash-rust" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" + [[package]] name = "xz2" version = "0.1.7" @@ -9358,6 +10015,15 @@ dependencies = [ "zstd-safe 6.0.6", ] +[[package]] +name = "zstd" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +dependencies = [ + "zstd-safe 7.0.0", +] + [[package]] name = "zstd-safe" version = "5.0.2+zstd.1.5.2" @@ -9378,6 +10044,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.9+zstd.1.5.5" diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 7184312b9b..661bead073 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -84,3 +84,6 @@ candle-transformers.workspace = true candle-nn.workspace = true aws-sdk-s3 = "0.36.0" aws-config = "0.57.2" +polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv"] } +polars-io = { version = "0.35.2", features = ["csv"] } +object_store = { version = "0.8.0", features = ["aws"] } diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 4c9d1c209f..6ce56c2b97 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.206.0 + version: 1.210.0 title: Windmill API contact: name: Windmill Team @@ -1534,6 +1534,24 @@ paths: properties: deploy_to: type: string + /w/{workspace}/workspaces/is_premium: + get: + summary: get if workspace is premium + operationId: getIsPremium + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: status + content: + application/json: + schema: + type: boolean /w/{workspace}/workspaces/premium_info: get: summary: get premium info @@ -1557,6 +1575,8 @@ paths: type: boolean usage: type: number + seats: + type: number required: - premium /w/{workspace}/workspaces/edit_slack_command: @@ -1670,6 +1690,8 @@ paths: properties: operator: type: boolean + invite_all: + type: boolean responses: '200': description: status @@ -3266,32 +3288,32 @@ paths: id: type: string value: - oneOf: &ref_177 + oneOf: &ref_179 - type: object - properties: &ref_161 + properties: &ref_163 input_transforms: type: object additionalProperties: oneOf: &ref_24 - type: object - properties: &ref_157 + properties: &ref_159 value: {} type: type: string enum: - javascript - required: &ref_158 + required: &ref_160 - expr - type - type: object - properties: &ref_159 + properties: &ref_161 expr: type: string type: type: string enum: - javascript - required: &ref_160 + required: &ref_162 - expr - type discriminator: &ref_25 @@ -3331,13 +3353,13 @@ paths: type: number concurrency_time_window_s: type: number - required: &ref_162 + required: &ref_164 - type - content - language - input_transforms - type: object - properties: &ref_163 + properties: &ref_165 input_transforms: type: object additionalProperties: @@ -3351,12 +3373,12 @@ paths: type: string enum: - script - required: &ref_164 + required: &ref_166 - type - path - input_transforms - type: object - properties: &ref_165 + properties: &ref_167 input_transforms: type: object additionalProperties: @@ -3368,12 +3390,12 @@ paths: type: string enum: - flow - required: &ref_166 + required: &ref_168 - type - path - input_transforms - type: object - properties: &ref_167 + properties: &ref_169 modules: type: array items: @@ -3395,13 +3417,13 @@ paths: type: boolean parallelism: type: integer - required: &ref_168 + required: &ref_170 - modules - iterator - skip_failures - type - type: object - properties: &ref_169 + properties: &ref_171 branches: type: array items: @@ -3432,12 +3454,12 @@ paths: type: string enum: - branchone - required: &ref_170 + required: &ref_172 - branches - default - type - type: object - properties: &ref_171 + properties: &ref_173 branches: type: array items: @@ -3462,28 +3484,28 @@ paths: - branchall parallel: type: boolean - required: &ref_172 - - branches - - type - - type: object - properties: &ref_173 - type: - type: string - enum: - - identity - flow: - type: boolean required: &ref_174 + - branches - type - type: object properties: &ref_175 type: type: string enum: - - graphql + - identity + flow: + type: boolean required: &ref_176 - type - discriminator: &ref_178 + - type: object + properties: &ref_177 + type: + type: string + enum: + - graphql + required: &ref_178 + - type + discriminator: &ref_180 propertyName: type mapping: rawscript: '#/components/schemas/RawScript' @@ -3539,7 +3561,7 @@ paths: type: number retry: type: object - properties: &ref_179 + properties: &ref_181 constant: type: object properties: @@ -5132,6 +5154,8 @@ paths: type: boolean priority: type: integer + dedicated_worker: + type: boolean required: &ref_144 - path - edited_by @@ -5279,6 +5303,8 @@ paths: type: boolean priority: type: integer + dedicated_worker: + type: boolean required: - path - type: object @@ -9465,10 +9491,10 @@ paths: content: application/json: schema: {} - /w/{workspace}/job_helpers/list_stored_datasets: - post: - summary: List the dataset keys available in the worspace datasets storage - operationId: polarsConnectionSettings + /w/{workspace}/job_helpers/list_stored_files: + get: + summary: List the file keys available in the worspace files storage (S3) + operationId: listStoredFiles tags: - helpers parameters: @@ -9478,23 +9504,92 @@ paths: schema: *ref_0 responses: '200': - description: Connection settings + description: List of file keys content: application/json: schema: type: object properties: - dataset_keys: + file_count: + type: integer + windmill_large_files: type: array items: type: object properties: &ref_155 s3: type: string - s3_bucket: - type: string required: &ref_156 - s3 + required: + - file_count + - windmill_large_files + /w/{workspace}/job_helpers/load_file_preview: + get: + summary: Load a preview of the file + operationId: loadFilePreview + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: file_key + in: query + required: true + schema: + type: string + - name: from + in: query + schema: + type: integer + - name: length + in: query + schema: + type: integer + - name: separator + in: query + schema: + type: string + responses: + '200': + description: FilePreview + content: + application/json: + schema: + type: object + properties: &ref_157 + mime_type: + type: string + last_modified: + type: string + format: date-time + size_in_bytes: + type: integer + expires: + type: string + format: date-time + version_id: + type: string + content_preview: + type: object + properties: + msg: + type: string + content: + type: string + content_type: + type: string + enum: + - RawText + - Csv + - Parquet + - Unknown + required: + - content_type + required: &ref_158 + - content_preview components: securitySchemes: bearerAuth: @@ -10100,6 +10195,10 @@ components: type: object properties: *ref_155 required: *ref_156 + WindmillFilePreview: + type: object + properties: *ref_157 + required: *ref_158 S3Resource: type: object properties: @@ -10124,58 +10223,58 @@ components: - useSSL - pathStyle StaticTransform: - type: object - properties: *ref_157 - required: *ref_158 - JavascriptTransform: type: object properties: *ref_159 required: *ref_160 + JavascriptTransform: + type: object + properties: *ref_161 + required: *ref_162 InputTransform: oneOf: *ref_24 discriminator: *ref_25 RawScript: - type: object - properties: *ref_161 - required: *ref_162 - PathScript: type: object properties: *ref_163 required: *ref_164 - PathFlow: + PathScript: type: object properties: *ref_165 required: *ref_166 + PathFlow: + type: object + properties: *ref_167 + required: *ref_168 FlowModule: type: object properties: *ref_26 required: *ref_27 ForloopFlow: - type: object - properties: *ref_167 - required: *ref_168 - BranchOne: type: object properties: *ref_169 required: *ref_170 - BranchAll: + BranchOne: type: object properties: *ref_171 required: *ref_172 - Identity: + BranchAll: type: object properties: *ref_173 required: *ref_174 - Graphql: + Identity: type: object properties: *ref_175 required: *ref_176 + Graphql: + type: object + properties: *ref_177 + required: *ref_178 FlowModuleValue: - oneOf: *ref_177 - discriminator: *ref_178 + oneOf: *ref_179 + discriminator: *ref_180 Retry: type: object - properties: *ref_179 + properties: *ref_181 FlowValue: type: object properties: *ref_48 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index bcd529b2f8..2e31294e6a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -2192,8 +2192,7 @@ paths: /w/{workspace}/resources/get_value_interpolated/{path}: get: - summary: - get resource interpolated (variables and resources are fully unrolled) + summary: get resource interpolated (variables and resources are fully unrolled) operationId: getResourceValueInterpolated tags: - resource @@ -2881,8 +2880,7 @@ paths: schema: type: string - name: first_parent_hash - description: - mask to filter scripts whom first direct parent has exact hash + description: mask to filter scripts whom first direct parent has exact hash in: query schema: type: string @@ -3077,8 +3075,7 @@ paths: /workers/custom_tags: get: - summary: - get all instance custom tags (tags are used to dispatch jobs to + summary: get all instance custom tags (tags are used to dispatch jobs to different worker groups) operationId: getCustomTags tags: @@ -3129,8 +3126,7 @@ paths: /w/{workspace}/scripts/delete/h/{hash}: post: - summary: - delete script by hash (erase content but keep hash, require admin) + summary: delete script by hash (erase content but keep hash, require admin) operationId: deleteScriptByHash tags: - script @@ -3322,16 +3318,14 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: - schedule the script to execute in the number of seconds starting now + description: schedule the script to execute in the number of seconds starting now in: query schema: type: integer - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/NewJobId" - name: invisible_to_owner - description: - make the run invisible to the the script owner (default false) + description: make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -4304,8 +4298,7 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: - schedule the script to execute in the number of seconds starting now + description: schedule the script to execute in the number of seconds starting now in: query schema: type: integer @@ -4313,8 +4306,7 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the flow owner (default false) + description: make the run invisible to the the flow owner (default false) in: query schema: type: boolean @@ -4366,8 +4358,7 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: - schedule the script to execute in the number of seconds starting now + description: schedule the script to execute in the number of seconds starting now in: query schema: type: integer @@ -4375,8 +4366,7 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the flow owner (default false) + description: make the run invisible to the the flow owner (default false) in: query schema: type: boolean @@ -4414,8 +4404,7 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: - schedule the script to execute in the number of seconds starting now + description: schedule the script to execute in the number of seconds starting now in: query schema: type: integer @@ -4423,8 +4412,7 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the script owner (default false) + description: make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -4455,8 +4443,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the script owner (default false) + description: make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -4489,8 +4476,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the script owner (default false) + description: make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -4705,8 +4691,7 @@ paths: - job responses: "200": - description: - the timestamp of the db that can be used to compute the drift + description: the timestamp of the db that can be used to compute the drift content: application/json: schema: @@ -4942,8 +4927,7 @@ paths: /w/{workspace}/jobs/resume_urls/{id}/{resume_id}: get: - summary: - get resume urls given a job_id, resume_id and a nonce to resume a flow + summary: get resume urls given a job_id, resume_id and a nonce to resume a flow operationId: getResumeUrls tags: - job @@ -5494,8 +5478,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - name: only_member_of in: query - description: - only list the groups the user is member of (default false) + description: only list the groups the user is member of (default false) schema: type: boolean responses: @@ -5683,8 +5666,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - name: only_member_of in: query - description: - only list the folders the user is member of (default false) + description: only list the folders the user is member of (default false) schema: type: boolean responses: @@ -6354,26 +6336,64 @@ paths: application/json: schema: {} - /w/{workspace}/job_helpers/list_stored_datasets: - post: - summary: List the dataset keys available in the worspace datasets storage - operationId: polarsConnectionSettings + /w/{workspace}/job_helpers/list_stored_files: + get: + summary: List the file keys available in the worspace files storage (S3) + operationId: listStoredFiles tags: - helpers parameters: - $ref: "#/components/parameters/WorkspaceId" responses: "200": - description: Connection settings + description: List of file keys content: application/json: schema: type: object properties: - dataset_keys: + file_count: + type: integer + windmill_large_files: type: array items: $ref: "#/components/schemas/WindmillLargeFile" + required: + - file_count + - windmill_large_files + + /w/{workspace}/job_helpers/load_file_preview: + get: + summary: Load a preview of the file + operationId: loadFilePreview + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: file_key + in: query + required: true + schema: + type: string + - name: from + in: query + schema: + type: integer + - name: length + in: query + schema: + type: integer + - name: separator + in: query + schema: + type: string + responses: + "200": + description: FilePreview + content: + application/json: + schema: + $ref: "#/components/schemas/WindmillFilePreview" components: securitySchemes: @@ -6467,8 +6487,7 @@ components: type: integer PerPage: name: per_page - description: - number of items to return for a given page (default 30, max 100) + description: number of items to return for a given page (default 30, max 100) in: query schema: type: integer @@ -6602,8 +6621,7 @@ components: type: boolean ArgsFilter: name: args - description: - filter on jobs containing those args as a json subset (@> in postgres) + description: filter on jobs containing those args as a json subset (@> in postgres) in: query schema: type: string @@ -6615,8 +6633,7 @@ components: type: string ResultFilter: name: result - description: - filter on jobs containing those result as a json subset (@> in postgres) + description: filter on jobs containing those result as a json subset (@> in postgres) in: query schema: type: string @@ -8372,11 +8389,39 @@ components: properties: s3: type: string - s3_bucket: - type: string required: - s3 + WindmillFilePreview: + type: object + properties: + mime_type: + type: string + last_modified: + type: string + format: date-time + size_in_bytes: + type: integer + expires: + type: string + format: date-time + version_id: + type: string + content_preview: + type: object + properties: + msg: + type: string + content: + type: string + content_type: + type: string + enum: ["RawText", "Csv", "Parquet", "Unknown"] + required: + - content_type + required: + - content_preview + S3Resource: type: object properties: diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs index 958f8759ba..3fad700a91 100644 --- a/backend/windmill-api/src/job_helpers.rs +++ b/backend/windmill-api/src/job_helpers.rs @@ -1,13 +1,27 @@ +use std::cmp; + use crate::{ db::DB, resources::transform_json_value, users::Tokened, workspaces::LargeFileStorage, }; use aws_sdk_s3::config::{Credentials, Region}; use axum::{ - extract::Path, + extract::{Path, Query}, routing::{get, post}, Extension, Json, Router, }; use hyper::http; +use object_store::ClientConfigKey; +use polars::{ + io::{ + cloud::{AmazonS3ConfigKey, CloudOptions}, + SerReader, + }, + lazy::{ + dsl::col, + frame::{LazyFrame, ScanArgsParquet}, + }, + prelude::CsvReader, +}; use serde::{Deserialize, Serialize}; use tower_http::cors::{Any, CorsLayer}; use windmill_common::{db::UserDB, error}; @@ -31,8 +45,12 @@ pub fn workspaced_service() -> Router { ) .route("/test_connection", get(test_connection).layer(cors.clone())) .route( - "/list_stored_datasets", - get(list_stored_datasets).layer(cors.clone()), + "/list_stored_files", + get(list_stored_files).layer(cors.clone()), + ) + .route( + "/load_file_preview", + get(load_file_preview).layer(cors.clone()), ) } @@ -133,13 +151,13 @@ async fn polars_connection_settings( #[derive(Serialize)] struct ListStoredDatasetsResponse { + file_count: usize, windmill_large_files: Vec, } #[derive(Serialize, Clone)] struct WindmillLargeFile { s3: String, - s3_bucket: Option, } async fn test_connection( @@ -156,7 +174,9 @@ async fn test_connection( )); } - let s3_resource = s3_resource_opt.unwrap(); + let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr( + "No files storage resource defined at the workspace level".to_string(), + ))?; let s3_client = build_s3_client(&s3_resource); s3_client .list_objects() @@ -168,7 +188,7 @@ async fn test_connection( return Ok(Json(())); } -async fn list_stored_datasets( +async fn list_stored_files( authed: ApiAuthed, Extension(user_db): Extension, Extension(db): Extension, @@ -177,7 +197,9 @@ async fn list_stored_datasets( ) -> error::JsonResult { let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?; - let s3_resource = s3_resource_opt.unwrap(); + let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr( + "No files storage resource defined at the workspace level".to_string(), + ))?; let s3_client = build_s3_client(&s3_resource); let mut stored_datasets = Vec::::new(); @@ -206,10 +228,7 @@ async fn list_stored_datasets( .map(|object| object.key()) .map(Option::unwrap) .map(&str::to_string) - .map(|object_key| WindmillLargeFile { - s3: object_key.clone(), - s3_bucket: Some(s3_bucket.clone()), - }) + .map(|object_key| WindmillLargeFile { s3: object_key.clone() }) .collect::>(); stored_datasets.extend(page_datasets.clone()); @@ -219,11 +238,159 @@ async fn list_stored_datasets( } } + #[cfg(not(feature = "enterprise"))] + if stored_datasets.len() > 10 { + return Err(error::Error::ExecutionErr( + "The workspace s3 bucket contains more than 10 files. Consider upgrading to Windmill Enterprise Edition to continue to use this feature, " + .to_string(), + )); + } + return Ok(Json(ListStoredDatasetsResponse { + file_count: stored_datasets.len(), // TODO: for now, no pagination. Add in the future to support large buckets windmill_large_files: stored_datasets, })); } +#[derive(Deserialize)] +struct LoadFilePreviewQuery { + pub file_key: String, + pub from: Option, + pub length: Option, + pub separator: Option, +} + +#[derive(Serialize)] +struct LoadFilePreviewResponse { + pub mime_type: Option, + pub size_in_bytes: Option, + pub last_modified: Option>, + pub expires: Option>, + pub version_id: Option, + pub content_preview: FileContentPreview, +} + +#[derive(Serialize)] +struct FileContentPreview { + pub content: Option, + pub content_type: WindmillContentType, + pub msg: Option, +} + +#[derive(Serialize)] +enum WindmillContentType { + RawText, + Csv, + Parquet, + Unknown, +} + +async fn load_file_preview( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Tokened { token }: Tokened, + Path(w_id): Path, + Query(query): Query, +) -> error::JsonResult { + let file_key = query.file_key.clone(); + let s3_resource_opt = get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?; + + let s3_resource = s3_resource_opt.ok_or(error::Error::InternalErr( + "No files storage resource defined at the workspace level".to_string(), + ))?; + let s3_client = build_s3_client(&s3_resource); + + let s3_bucket = s3_resource.bucket.clone(); + let s3_object_metadata = s3_client + .head_object() + .bucket(&s3_bucket) + .key(&file_key) + .send() + .await + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + + let file_chunk_length = cmp::min( + query.length.unwrap_or(8 * 1024 * 1024) as i64, // no more than 8MB per chunk by default + s3_object_metadata.content_length() - query.from.unwrap_or(0) as i64, + ); + + let object_content_type = s3_object_metadata.content_type(); + let content_type: WindmillContentType; + let content_preview = match object_content_type { + Some("application/json") | Some("application/x-yaml") => { + content_type = WindmillContentType::RawText; + read_s3_text_object_head( + &s3_client, + &s3_bucket, + &file_key, + query.from.unwrap_or(0), + file_chunk_length, // 1KB by default + ) + .await + } + Some("text/csv") => { + content_type = WindmillContentType::Csv; + read_s3_csv_object_head(&s3_client, &s3_bucket, &file_key, query.separator).await + } + Some("application/octet-stream") => { + if file_key.to_lowercase().ends_with(".parquet") { + content_type = WindmillContentType::Parquet; + read_s3_parquet_object_head(&s3_resource, &file_key).await + } else { + content_type = WindmillContentType::Unknown; + Err(error::Error::ExecutionErr( + "Preview is not available for content of type application/octet-stream" + .to_string(), + )) + } + } + Some(mt) if mt.starts_with("text/") => { + content_type = WindmillContentType::RawText; + read_s3_text_object_head( + &s3_client, + &s3_bucket, + &file_key, + query.from.unwrap_or(0), + file_chunk_length, + ) + .await + } + _ => { + content_type = WindmillContentType::Unknown; + Err(error::Error::ExecutionErr( + "Preview is not available for content of type application/octet-stream".to_string(), + )) + } + }; + + let response = LoadFilePreviewResponse { + mime_type: s3_object_metadata.content_type().map(&str::to_string), + size_in_bytes: Some(s3_object_metadata.content_length()), + last_modified: s3_object_metadata + .last_modified() + .map(|dt| chrono::DateTime::from_timestamp(dt.secs(), dt.subsec_nanos())) + .flatten(), + expires: s3_object_metadata + .expires() + .map(|dt| chrono::DateTime::from_timestamp(dt.secs(), dt.subsec_nanos())) + .flatten(), + version_id: s3_object_metadata.version_id().map(&str::to_string), + content_preview: match content_preview { + Ok(content) => { + FileContentPreview { content_type: content_type, content: Some(content), msg: None } + } + Err(err) => FileContentPreview { + content_type: content_type, + content: None, + msg: Some(err.to_string()), + }, + }, + }; + + return Ok(Json(response)); +} + async fn get_workspace_s3_resource<'c>( authed: &ApiAuthed, user_db: &UserDB, @@ -282,18 +449,9 @@ async fn get_workspace_s3_resource<'c>( fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client { let s3_resource = s3_resource_ref.clone(); - let endpoint_with_prefix = if s3_resource.endpoint.starts_with("http://") - || s3_resource.endpoint.starts_with("https://") - { - s3_resource.endpoint.clone() - } else if s3_resource.use_ssl { - format!("https://{}", s3_resource.endpoint) - } else { - format!("http://{}", s3_resource.endpoint) - }; - + let endpoint = render_endpoint(&s3_resource); let mut s3_config_builder = aws_sdk_s3::Config::builder() - .endpoint_url(endpoint_with_prefix) + .endpoint_url(endpoint) .region(Region::new(s3_resource.region)); if s3_resource.access_key.is_some() { s3_config_builder = s3_config_builder.credentials_provider(Credentials::new( @@ -310,3 +468,173 @@ fn build_s3_client(s3_resource_ref: &S3Resource) -> aws_sdk_s3::Client { let s3_config = s3_config_builder.build(); return aws_sdk_s3::Client::from_conf(s3_config); } + +fn build_polars_s3_config(s3_resource_ref: &S3Resource) -> CloudOptions { + let s3_resource = s3_resource_ref.to_owned(); + let mut s3_configs: Vec<(AmazonS3ConfigKey, String)> = vec![ + (AmazonS3ConfigKey::Region, s3_resource.region), + (AmazonS3ConfigKey::Bucket, s3_resource.bucket), + ( + AmazonS3ConfigKey::Endpoint, + render_endpoint(s3_resource_ref), + ), + ( + AmazonS3ConfigKey::Client(ClientConfigKey::AllowHttp), + (!s3_resource.use_ssl).to_string(), + ), + ( + AmazonS3ConfigKey::VirtualHostedStyleRequest, + (!s3_resource.path_style).to_string(), + ), + ]; + if let Some(access_key) = s3_resource.access_key { + s3_configs.push((AmazonS3ConfigKey::AccessKeyId, access_key)); + } + if let Some(secret_key) = s3_resource.secret_key { + s3_configs.push((AmazonS3ConfigKey::SecretAccessKey, secret_key)); + } + return CloudOptions::default().with_aws(s3_configs); +} + +fn render_endpoint(s3_resource: &S3Resource) -> String { + if s3_resource.endpoint.starts_with("http://") || s3_resource.endpoint.starts_with("https://") { + s3_resource.endpoint.clone() + } else if s3_resource.use_ssl { + format!("https://{}", s3_resource.endpoint) + } else { + format!("http://{}", s3_resource.endpoint) + } +} + +async fn read_s3_text_object_head( + s3_client: &aws_sdk_s3::Client, + s3_bucket: &str, + file_key: &str, + from_char: usize, + length: i64, +) -> error::Result { + let s3_object = s3_client + .get_object() + .range(format!("bytes={}-{}", from_char, length).to_string()) + .bucket(s3_bucket) + .key(file_key) + .send() + .await + .map_err(|err| { + tracing::warn!("Error fetching text file from S3: {:?}", err); + error::Error::InternalErr(err.to_string()) + })?; + + let payload = s3_object + .body + .collect() + .await + .map_err(|err| { + tracing::warn!( + "Error reading raw text file {}. Error was: {:?}", + file_key, + err + ); + error::Error::InternalErr("File encoding is not supported".to_string()) + })? + .into_bytes() + .to_vec(); + + let file_header_str = String::from_utf8(payload).map_err(|err| { + tracing::warn!( + "Encoding of file {} unsupported. Error was: {:?}", + file_key, + err + ); + error::Error::InternalErr("File encoding is not supported".to_string()) + })?; + return Ok(file_header_str); +} + +async fn read_s3_parquet_object_head( + s3_resource_ref: &S3Resource, + file_key: &str, +) -> error::Result { + let s3_cloud_config = build_polars_s3_config(s3_resource_ref); + + let args: ScanArgsParquet = ScanArgsParquet { + n_rows: Some(1), + cache: false, + parallel: polars::io::parquet::ParallelStrategy::Auto, + rechunk: false, + row_count: None, + low_memory: false, + use_statistics: false, + hive_partitioning: false, + cloud_options: Some(s3_cloud_config), + }; + + let file_key_clone = file_key.to_string(); + let s3_bucket_clone = s3_resource_ref.bucket.to_string(); + let polars_df_result = tokio::task::spawn_blocking(move || { + let s3_file_key = format!("s3://{}/{}", s3_bucket_clone, file_key_clone); + let lzdf_result = LazyFrame::scan_parquet(s3_file_key, args); + match lzdf_result { + Err(err) => { + tracing::warn!("Error fetching parquet file from S3: {:?}", err); + return Err(error::Error::InternalErr(err.to_string())); + } + Ok(lzdf) => { + let df = lzdf + .select(&[col("*")]) + .limit(10) // for now read only first 10 lines + .collect() + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + return Ok(format!("{:?}", df).to_string()); + } + } + }) + .await + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + + return polars_df_result; +} + +async fn read_s3_csv_object_head( + s3_client: &aws_sdk_s3::Client, + s3_bucket: &str, + file_key: &str, + separator: Option, +) -> error::Result { + let separator_final = if let Some(separator_char) = separator { + if separator_char.len() != 1 { + return Err(error::Error::BadRequest( + "Separator must be a single character".to_string(), + )); + } + separator_char.as_bytes()[0] + } else { + ",".as_bytes()[0] + }; + + let s3_object = s3_client + .get_object() + .bucket(s3_bucket) + .range("bytes=0-33554432".to_string()) // safeguard - do not load CSV files larger than 32MB + .key(file_key) + .send() + .await + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + + // TODO: polars does not seem to support lazy csv reader, unfortunately. We can implement it ourselves if needed + let file_content_bytes = s3_object + .body + .collect() + .await + .map_err(|err| error::Error::InternalErr(err.to_string()))? + .into_bytes(); + let cursor = std::io::Cursor::new(file_content_bytes); + + let csv_df = CsvReader::new(cursor) + .with_n_rows(Some(10)) // for now read only first 10 lines + .with_separator(separator_final) + .finish() + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + + return Ok(format!("{:?}", csv_df).to_string()); +} diff --git a/backend/windmill-api/src/openai.rs b/backend/windmill-api/src/openai.rs index dd3094d2e2..ce9a21937c 100644 --- a/backend/windmill-api/src/openai.rs +++ b/backend/windmill-api/src/openai.rs @@ -126,8 +126,9 @@ async fn proxy( "https://api.openai.com/v1" }; + let url = format!("{}/{}", base_url, openai_path); let mut request = HTTP_CLIENT - .post(base_url.to_string() + "/" + &openai_path) + .post(url) .header("content-type", "application/json") .body(body); diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index ff67e061f6..8c63de55ee 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -1,7 +1,7 @@ + { + rawValue = JSON.stringify(value, null, 2) + editor?.setCode(rawValue) + }} +/> +
@@ -398,6 +411,30 @@ Loading resource types... {:else if inputCat == 'resource-object' && (resourceTypes == undefined || (format.split('-').length > 1 && resourceTypes.includes(format.substring('resource-'.length))))} + {:else if inputCat == 'resource-object' && format.split('-').length > 1 && format.replace('resource-', '') == 's3object'} +
+ { + dispatch('focus') + }} + code={JSON.stringify({ s3: '' }, null, 2)} + bind:value + /> + +
{:else if inputCat == 'object' || inputCat == 'resource-object'} {#if properties && Object.keys(properties).length > 0}
diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte new file mode 100644 index 0000000000..5221dc98e1 --- /dev/null +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -0,0 +1,314 @@ + + + + + { + dispatch('close') + }} + size="1200px" +> + + {#if workspaceSettingsInitialized === false} + +
+

+ The workspace needs to be connected to an S3 storage to use this feature. You can configure it here. +

+
+ + {:else} +
+
+ {#if displayedFileKeys.length === 0} +
+ No files in the workspace S3 bucket +
+ {:else} + +
+ {@const file_info = allFilesByKey[displayedFileKeys[index]]} +
selectItem(index)} + class={`flex flex-row h-full font-semibold text-xs items-center justify-start ${ + selectedFileKey !== undefined && selectedFileKey.s3 === file_info.full_key + ? 'bg-surface-hover' + : '' + } `} + > +
+ {#if file_info.type === 'folder'} + {#if file_info.collapsed}{:else}{/if} + {file_info.display_name} + {:else} + + {file_info.display_name} + {/if} +
+
+
+ {/if} +
+
+ {#if filePreview === undefined} +
+
+
+ {:else} +
+
+ +
+ {/if} + +
+ {#if filePreview !== undefined} +
+ {#if filePreview.content_type === 'Unknown'} + Type of file not supported for preview + {:else if filePreview.content_type === 'Csv'} + Previewing a {filePreview.content_type?.toLowerCase()} file. Change the separator: +
+ +
+ {:else} + Previewing a {filePreview.content_type?.toLowerCase()} file + {/if} +
+ 
{#if !emptyString(filePreview.content_preview)}{filePreview.content_preview}{:else if filePreview.content_type !== undefined}Preview impossible. If it's a CSV file, you can try changing the separator{/if}
+ {/if} +
+
+
+ {/if} + +
+ +
+ + diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 64ba48141f..2022192669 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -49,6 +49,20 @@ export function displayDate(dateString: string | Date | undefined, displaySecond } } +export function displaySize(sizeInBytes: number | undefined): string | undefined { + if (sizeInBytes === undefined) { + return undefined + } + const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'] + let size = sizeInBytes + let unit_idx = 0 + while (unit_idx < units.length - 1 && size > 1024) { + size /= 1024 + unit_idx += 1 + } + return `${size.toFixed(1)}${units[unit_idx]}` +} + export function msToSec(ms: number | undefined, maximumFractionDigits?: number): string { if (ms === undefined) return '?' return (ms / 1000).toLocaleString(undefined, { diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 776fe6621b..97ff90103b 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -295,6 +295,9 @@
Windmill AI
 + +
S3 Storage
+
Delete Workspace